diff --git a/libtomcrypt_VS2008.vcproj b/libtomcrypt_VS2008.vcproj
index 316a646c7..502a19e0a 100644
--- a/libtomcrypt_VS2008.vcproj
+++ b/libtomcrypt_VS2008.vcproj
@@ -1778,6 +1778,10 @@
RelativePath="src\modes\cbc\cbc_start.c"
>
+
+
serpent.k);
+#ifdef LTC_CLEAN_STACK
+ burn_stack(sizeof(ulong32) * 14 + sizeof(int));
+#endif
+ return err;
+}
+
+static LTC_INLINE void s_serpent_accel_ecb_32_bit_load_one(ulong32 *x, const unsigned char *bytes)
+{
+ *x =
+ ((ulong32)(((ulong32)(bytes[0])) << (0 * CHAR_BIT))) |
+ ((ulong32)(((ulong32)(bytes[1])) << (1 * CHAR_BIT))) |
+ ((ulong32)(((ulong32)(bytes[2])) << (2 * CHAR_BIT))) |
+ ((ulong32)(((ulong32)(bytes[3])) << (3 * CHAR_BIT))) |
+ 0;
+}
+
+static LTC_INLINE void s_serpent_accel_ecb_32_bit_store_one(const ulong32 *x, unsigned char *bytes)
+{
+ bytes[0] = ((unsigned char)(((ulong32)((*x)) >> (0 * CHAR_BIT)) & 0xff));
+ bytes[1] = ((unsigned char)(((ulong32)((*x)) >> (1 * CHAR_BIT)) & 0xff));
+ bytes[2] = ((unsigned char)(((ulong32)((*x)) >> (2 * CHAR_BIT)) & 0xff));
+ bytes[3] = ((unsigned char)(((ulong32)((*x)) >> (3 * CHAR_BIT)) & 0xff));
+}
+
+static LTC_INLINE void s_serpent_accel_ecb_32_bit_load_four(ulong32 *pa, ulong32 *pb, ulong32 *pc, ulong32 *pd, const unsigned char *bytes)
+{
+ s_serpent_accel_ecb_32_bit_load_one(pa, &bytes[0 * sizeof(ulong32)]);
+ s_serpent_accel_ecb_32_bit_load_one(pb, &bytes[1 * sizeof(ulong32)]);
+ s_serpent_accel_ecb_32_bit_load_one(pc, &bytes[2 * sizeof(ulong32)]);
+ s_serpent_accel_ecb_32_bit_load_one(pd, &bytes[3 * sizeof(ulong32)]);
+}
+
+static LTC_INLINE void s_serpent_accel_ecb_32_bit_store_four(const ulong32 *pa, const ulong32 *pb, const ulong32 *pc, const ulong32 *pd, unsigned char *bytes)
+{
+ s_serpent_accel_ecb_32_bit_store_one(pa, &bytes[0 * sizeof(ulong32)]);
+ s_serpent_accel_ecb_32_bit_store_one(pb, &bytes[1 * sizeof(ulong32)]);
+ s_serpent_accel_ecb_32_bit_store_one(pc, &bytes[2 * sizeof(ulong32)]);
+ s_serpent_accel_ecb_32_bit_store_one(pd, &bytes[3 * sizeof(ulong32)]);
+}
+
+static LTC_INLINE int s_serpent_accel_ecb_encrypt_32_bit(const unsigned char *pt, unsigned char *ct, unsigned long blocks, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (32 / 32)
+ #define s_do_broadcast(x) x
+ #define s_do_asgn(a, b) a = b
+ #define s_do_or(a, b) a |= b
+ #define s_do_xor(a, b) a ^= b
+ #define s_do_and(a, b) a &= b
+ #define s_do_not(a, b) a =~ b
+ #define s_do_rol(x, i) x = ROL(x, i)
+ #define s_do_shl(a, b, c) a = b << c
+
+ const unsigned char *in;
+ unsigned char *out;
+ const ulong32* k;
+ unsigned long iblock;
ulong32 a, b, c, d, e;
- unsigned int i = 1;
-
- LOAD32L(a, in + 0);
- LOAD32L(b, in + 4);
- LOAD32L(c, in + 8);
- LOAD32L(d, in + 12);
-
- do {
- s_beforeS0(s_kx); s_beforeS0(s_s0); s_afterS0(s_lt);
- s_afterS0(s_kx); s_afterS0(s_s1); s_afterS1(s_lt);
- s_afterS1(s_kx); s_afterS1(s_s2); s_afterS2(s_lt);
- s_afterS2(s_kx); s_afterS2(s_s3); s_afterS3(s_lt);
- s_afterS3(s_kx); s_afterS3(s_s4); s_afterS4(s_lt);
- s_afterS4(s_kx); s_afterS4(s_s5); s_afterS5(s_lt);
- s_afterS5(s_kx); s_afterS5(s_s6); s_afterS6(s_lt);
- s_afterS6(s_kx); s_afterS6(s_s7);
-
- if (i == 4) break;
-
- ++i;
- c = b;
- b = e;
- e = d;
- d = a;
- a = e;
- k += 32;
- s_beforeS0(s_lt);
- } while (1);
-
- s_afterS7(s_kx);
-
- STORE32L(d, out + 0);
- STORE32L(e, out + 4);
- STORE32L(b, out + 8);
- STORE32L(a, out + 12);
+ LTC_ARGCHK(pt);
+ LTC_ARGCHK(ct);
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+ LTC_ARGCHK(skey);
+
+ in = pt;
+ out = ct;
+ k = &skey->serpent.k[0];
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ s_serpent_accel_ecb_32_bit_load_four(&a, &b, &c, &d, in);
+ s_apply_order_enc_00(s_apply_key);
+ s_apply_order_enc_00(s_enc_0); s_apply_order_enc_01(s_apply_lk);
+ s_apply_order_enc_01(s_enc_1); s_apply_order_enc_02(s_apply_lk);
+ s_apply_order_enc_02(s_enc_2); s_apply_order_enc_03(s_apply_lk);
+ s_apply_order_enc_03(s_enc_3); s_apply_order_enc_04(s_apply_lk);
+ s_apply_order_enc_04(s_enc_4); s_apply_order_enc_05(s_apply_lk);
+ s_apply_order_enc_05(s_enc_5); s_apply_order_enc_06(s_apply_lk);
+ s_apply_order_enc_06(s_enc_6); s_apply_order_enc_07(s_apply_lk);
+ s_apply_order_enc_07(s_enc_7); s_apply_order_enc_08(s_apply_lk);
+ s_apply_order_enc_08(s_enc_0); s_apply_order_enc_09(s_apply_lk);
+ s_apply_order_enc_09(s_enc_1); s_apply_order_enc_10(s_apply_lk);
+ s_apply_order_enc_10(s_enc_2); s_apply_order_enc_11(s_apply_lk);
+ s_apply_order_enc_11(s_enc_3); s_apply_order_enc_12(s_apply_lk);
+ s_apply_order_enc_12(s_enc_4); s_apply_order_enc_13(s_apply_lk);
+ s_apply_order_enc_13(s_enc_5); s_apply_order_enc_14(s_apply_lk);
+ s_apply_order_enc_14(s_enc_6); s_apply_order_enc_15(s_apply_lk);
+ s_apply_order_enc_15(s_enc_7); s_apply_order_enc_16(s_apply_lk);
+ s_apply_order_enc_16(s_enc_0); s_apply_order_enc_17(s_apply_lk);
+ s_apply_order_enc_17(s_enc_1); s_apply_order_enc_18(s_apply_lk);
+ s_apply_order_enc_18(s_enc_2); s_apply_order_enc_19(s_apply_lk);
+ s_apply_order_enc_19(s_enc_3); s_apply_order_enc_20(s_apply_lk);
+ s_apply_order_enc_20(s_enc_4); s_apply_order_enc_21(s_apply_lk);
+ s_apply_order_enc_21(s_enc_5); s_apply_order_enc_22(s_apply_lk);
+ s_apply_order_enc_22(s_enc_6); s_apply_order_enc_23(s_apply_lk);
+ s_apply_order_enc_23(s_enc_7); s_apply_order_enc_24(s_apply_lk);
+ s_apply_order_enc_24(s_enc_0); s_apply_order_enc_25(s_apply_lk);
+ s_apply_order_enc_25(s_enc_1); s_apply_order_enc_26(s_apply_lk);
+ s_apply_order_enc_26(s_enc_2); s_apply_order_enc_27(s_apply_lk);
+ s_apply_order_enc_27(s_enc_3); s_apply_order_enc_28(s_apply_lk);
+ s_apply_order_enc_28(s_enc_4); s_apply_order_enc_29(s_apply_lk);
+ s_apply_order_enc_29(s_enc_5); s_apply_order_enc_30(s_apply_lk);
+ s_apply_order_enc_30(s_enc_6); s_apply_order_enc_31(s_apply_lk);
+ s_apply_order_enc_31(s_enc_7); s_apply_order_enc_32(s_apply_key);
+ s_serpent_accel_ecb_32_bit_store_four(&a, &b, &c, &d, out);
+ in += blocks_at_a_time * serpent_block_len;
+ out += blocks_at_a_time * serpent_block_len;
+ }
return CRYPT_OK;
+
+ #undef blocks_at_a_time
+ #undef s_do_broadcast
+ #undef s_do_asgn
+ #undef s_do_or
+ #undef s_do_xor
+ #undef s_do_and
+ #undef s_do_not
+ #undef s_do_rol
+ #undef s_do_shl
}
-static int s_dec_block(const unsigned char *in, unsigned char *out, const ulong32 *k)
+static LTC_INLINE int s_serpent_accel_ecb_decrypt_32_bit(const unsigned char *ct, unsigned char *pt, unsigned long blocks, const symmetric_key *skey)
{
+ #define blocks_at_a_time (32 / 32)
+ #define s_do_broadcast(x) x
+ #define s_do_asgn(a, b) a = b
+ #define s_do_or(a, b) a |= b
+ #define s_do_xor(a, b) a ^= b
+ #define s_do_and(a, b) a &= b
+ #define s_do_not(a, b) a =~ b
+ #define s_do_rol(x, i) x = ROL(x, i)
+ #define s_do_ror(x, i) x = ROR(x, i)
+ #define s_do_shl(a, b, c) a = b << c
+
+ const unsigned char *in;
+ unsigned char *out;
+ const ulong32* k;
+ unsigned long iblock;
ulong32 a, b, c, d, e;
- unsigned int i;
-
- LOAD32L(a, in + 0);
- LOAD32L(b, in + 4);
- LOAD32L(c, in + 8);
- LOAD32L(d, in + 12);
- e = 0; LTC_UNUSED_PARAM(e); /* avoid scan-build warning */
- i = 4;
- k += 96;
-
- s_beforeI7(s_kx);
- goto start;
-
- do {
- c = b;
- b = d;
- d = e;
- k -= 32;
- s_beforeI7(s_ilt);
-start:
- s_beforeI7(s_i7); s_afterI7(s_kx);
- s_afterI7(s_ilt); s_afterI7(s_i6); s_afterI6(s_kx);
- s_afterI6(s_ilt); s_afterI6(s_i5); s_afterI5(s_kx);
- s_afterI5(s_ilt); s_afterI5(s_i4); s_afterI4(s_kx);
- s_afterI4(s_ilt); s_afterI4(s_i3); s_afterI3(s_kx);
- s_afterI3(s_ilt); s_afterI3(s_i2); s_afterI2(s_kx);
- s_afterI2(s_ilt); s_afterI2(s_i1); s_afterI1(s_kx);
- s_afterI1(s_ilt); s_afterI1(s_i0); s_afterI0(s_kx);
- } while (--i != 0);
-
- STORE32L(a, out + 0);
- STORE32L(d, out + 4);
- STORE32L(b, out + 8);
- STORE32L(e, out + 12);
+ LTC_ARGCHK(ct);
+ LTC_ARGCHK(pt);
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+ LTC_ARGCHK(skey);
+
+ in = ct;
+ out = pt;
+ k = &skey->serpent.k[0];
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ s_serpent_accel_ecb_32_bit_load_four(&a, &b, &c, &d, in);
+ s_apply_order_dec_32(s_apply_key);
+ s_apply_order_dec_32(s_dec_7); s_apply_order_dec_31(s_apply_kl);
+ s_apply_order_dec_31(s_dec_6); s_apply_order_dec_30(s_apply_kl);
+ s_apply_order_dec_30(s_dec_5); s_apply_order_dec_29(s_apply_kl);
+ s_apply_order_dec_29(s_dec_4); s_apply_order_dec_28(s_apply_kl);
+ s_apply_order_dec_28(s_dec_3); s_apply_order_dec_27(s_apply_kl);
+ s_apply_order_dec_27(s_dec_2); s_apply_order_dec_26(s_apply_kl);
+ s_apply_order_dec_26(s_dec_1); s_apply_order_dec_25(s_apply_kl);
+ s_apply_order_dec_25(s_dec_0); s_apply_order_dec_24(s_apply_kl);
+ s_apply_order_dec_24(s_dec_7); s_apply_order_dec_23(s_apply_kl);
+ s_apply_order_dec_23(s_dec_6); s_apply_order_dec_22(s_apply_kl);
+ s_apply_order_dec_22(s_dec_5); s_apply_order_dec_21(s_apply_kl);
+ s_apply_order_dec_21(s_dec_4); s_apply_order_dec_20(s_apply_kl);
+ s_apply_order_dec_20(s_dec_3); s_apply_order_dec_19(s_apply_kl);
+ s_apply_order_dec_19(s_dec_2); s_apply_order_dec_18(s_apply_kl);
+ s_apply_order_dec_18(s_dec_1); s_apply_order_dec_17(s_apply_kl);
+ s_apply_order_dec_17(s_dec_0); s_apply_order_dec_16(s_apply_kl);
+ s_apply_order_dec_16(s_dec_7); s_apply_order_dec_15(s_apply_kl);
+ s_apply_order_dec_15(s_dec_6); s_apply_order_dec_14(s_apply_kl);
+ s_apply_order_dec_14(s_dec_5); s_apply_order_dec_13(s_apply_kl);
+ s_apply_order_dec_13(s_dec_4); s_apply_order_dec_12(s_apply_kl);
+ s_apply_order_dec_12(s_dec_3); s_apply_order_dec_11(s_apply_kl);
+ s_apply_order_dec_11(s_dec_2); s_apply_order_dec_10(s_apply_kl);
+ s_apply_order_dec_10(s_dec_1); s_apply_order_dec_09(s_apply_kl);
+ s_apply_order_dec_09(s_dec_0); s_apply_order_dec_08(s_apply_kl);
+ s_apply_order_dec_08(s_dec_7); s_apply_order_dec_07(s_apply_kl);
+ s_apply_order_dec_07(s_dec_6); s_apply_order_dec_06(s_apply_kl);
+ s_apply_order_dec_06(s_dec_5); s_apply_order_dec_05(s_apply_kl);
+ s_apply_order_dec_05(s_dec_4); s_apply_order_dec_04(s_apply_kl);
+ s_apply_order_dec_04(s_dec_3); s_apply_order_dec_03(s_apply_kl);
+ s_apply_order_dec_03(s_dec_2); s_apply_order_dec_02(s_apply_kl);
+ s_apply_order_dec_02(s_dec_1); s_apply_order_dec_01(s_apply_kl);
+ s_apply_order_dec_01(s_dec_0); s_apply_order_dec_00(s_apply_key);
+ s_serpent_accel_ecb_32_bit_store_four(&c, &d, &b, &e, out);
+ in += blocks_at_a_time * serpent_block_len;
+ out += blocks_at_a_time * serpent_block_len;
+ }
return CRYPT_OK;
+
+ #undef blocks_at_a_time
+ #undef s_do_broadcast
+ #undef s_do_asgn
+ #undef s_do_or
+ #undef s_do_xor
+ #undef s_do_and
+ #undef s_do_not
+ #undef s_do_rol
+ #undef s_do_ror
+ #undef s_do_shl
}
-int serpent_setup(const unsigned char *key, int keylen, int num_rounds, symmetric_key *skey)
+#if defined LTC_SERPENT_ACCEL_64_BIT
+
+static LTC_INLINE void s_serpent_accel_ecb_64_bit_load_one(ulong64 *x, const unsigned char *bytes)
{
- int err;
+ *x =
+ ((ulong64)(((ulong64)(bytes[0])) << (0 * CHAR_BIT))) |
+ ((ulong64)(((ulong64)(bytes[1])) << (1 * CHAR_BIT))) |
+ ((ulong64)(((ulong64)(bytes[2])) << (2 * CHAR_BIT))) |
+ ((ulong64)(((ulong64)(bytes[3])) << (3 * CHAR_BIT))) |
+ ((ulong64)(((ulong64)(bytes[4])) << (4 * CHAR_BIT))) |
+ ((ulong64)(((ulong64)(bytes[5])) << (5 * CHAR_BIT))) |
+ ((ulong64)(((ulong64)(bytes[6])) << (6 * CHAR_BIT))) |
+ ((ulong64)(((ulong64)(bytes[7])) << (7 * CHAR_BIT))) |
+ 0;
+}
- LTC_ARGCHK(key != NULL);
- LTC_ARGCHK(skey != NULL);
+static LTC_INLINE void s_serpent_accel_ecb_64_bit_store_one(const ulong64 *x, unsigned char *bytes)
+{
+ bytes[0] = ((unsigned char)(((ulong64)((*x)) >> (0 * CHAR_BIT)) & 0xff));
+ bytes[1] = ((unsigned char)(((ulong64)((*x)) >> (1 * CHAR_BIT)) & 0xff));
+ bytes[2] = ((unsigned char)(((ulong64)((*x)) >> (2 * CHAR_BIT)) & 0xff));
+ bytes[3] = ((unsigned char)(((ulong64)((*x)) >> (3 * CHAR_BIT)) & 0xff));
+ bytes[4] = ((unsigned char)(((ulong64)((*x)) >> (4 * CHAR_BIT)) & 0xff));
+ bytes[5] = ((unsigned char)(((ulong64)((*x)) >> (5 * CHAR_BIT)) & 0xff));
+ bytes[6] = ((unsigned char)(((ulong64)((*x)) >> (6 * CHAR_BIT)) & 0xff));
+ bytes[7] = ((unsigned char)(((ulong64)((*x)) >> (7 * CHAR_BIT)) & 0xff));
+}
- if (num_rounds != 0 && num_rounds != 32) return CRYPT_INVALID_ROUNDS;
- if (keylen != 16 && keylen != 24 && keylen != 32) return CRYPT_INVALID_KEYSIZE;
+static LTC_INLINE void s_serpent_accel_ecb_64_bit_load_four(ulong64 *pa, ulong64 *pb, ulong64 *pc, ulong64 *pd, const unsigned char *bytes)
+{
+ ulong64 ta, tb, tc, td;
+ ulong64 ra, rb, rc, rd;
+ ulong64 xa, xb;
+
+ s_serpent_accel_ecb_64_bit_load_one(&ta, &bytes[0 * sizeof(ulong64)]);
+ s_serpent_accel_ecb_64_bit_load_one(&tb, &bytes[1 * sizeof(ulong64)]);
+ s_serpent_accel_ecb_64_bit_load_one(&tc, &bytes[2 * sizeof(ulong64)]);
+ s_serpent_accel_ecb_64_bit_load_one(&td, &bytes[3 * sizeof(ulong64)]);
+ xa = ta << 32;
+ xa >>= 32;
+ xb = tc << 32;
+ ra = xa | xb;
+ xa = ta >> 32;
+ xb = tc >> 32;
+ xb <<= 32;
+ rb = xa | xb;
+ xa = tb << 32;
+ xa >>= 32;
+ xb = td << 32;
+ rc = xa | xb;
+ xa = tb >> 32;
+ xb = td >> 32;
+ xb <<= 32;
+ rd = xa | xb;
+ *pa = ra;
+ *pb = rb;
+ *pc = rc;
+ *pd = rd;
+}
+
+static LTC_INLINE void s_serpent_accel_ecb_64_bit_store_four(const ulong64 *pa, const ulong64 *pb, const ulong64 *pc, const ulong64 *pd, unsigned char *bytes)
+{
+ ulong64 ta, tb, tc, td;
+ ulong64 ra, rb, rc, rd;
+ ulong64 xa, xb;
+
+ ta = *pa;
+ tb = *pb;
+ tc = *pc;
+ td = *pd;
+ xa = ta << 32;
+ xa >>= 32;
+ xb = tb << 32;
+ ra = xa | xb;
+ xa = tc << 32;
+ xa >>= 32;
+ xb = td << 32;
+ rb = xa | xb;
+ xa = ta >> 32;
+ xb = tb >> 32;
+ xb <<= 32;
+ rc = xa | xb;
+ xa = tc >> 32;
+ xb = td >> 32;
+ xb <<= 32;
+ rd = xa | xb;
+ s_serpent_accel_ecb_64_bit_store_one(&ra, &bytes[0 * sizeof(ulong64)]);
+ s_serpent_accel_ecb_64_bit_store_one(&rb, &bytes[1 * sizeof(ulong64)]);
+ s_serpent_accel_ecb_64_bit_store_one(&rc, &bytes[2 * sizeof(ulong64)]);
+ s_serpent_accel_ecb_64_bit_store_one(&rd, &bytes[3 * sizeof(ulong64)]);
+}
+
+static LTC_INLINE int s_serpent_accel_ecb_encrypt_64_bit(const unsigned char *pt, unsigned char *ct, unsigned long blocks, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (64 / 32)
+ #define s_do_split_general(x, i) ((ulong32)((x) >> ((i) * 32)))
+ #define s_do_split_lo(x) s_do_split_general(x, 0)
+ #define s_do_split_hi(x) s_do_split_general(x, 1)
+ #define s_do_join(lo, hi) ((((ulong64)(hi)) << 32) | ((ulong64)(lo)))
+ #define s_do_broadcast(x) s_do_join(x, x)
+ #define s_do_asgn(a, b) a = b
+ #define s_do_or(a, b) a |= b
+ #define s_do_xor(a, b) a ^= b
+ #define s_do_and(a, b) a &= b
+ #define s_do_not(a, b) a =~ b
+ #define s_do_rol(x, i) x = s_do_join( \
+ ROL(s_do_split_lo(x), i), \
+ ROL(s_do_split_hi(x), i))
+ #define s_do_ror(x, i) x = s_do_join( \
+ ROR(s_do_split_lo(x), i), \
+ ROR(s_do_split_hi(x), i))
+ #define s_do_shl(a, b, c) a = s_do_join( \
+ (s_do_split_lo(b) << c), \
+ (s_do_split_hi(b) << c))
+
+ const unsigned char *in;
+ unsigned char *out;
+ const ulong32* k;
+ unsigned long iblock;
+ ulong64 a, b, c, d, e;
+
+ LTC_ARGCHK(pt);
+ LTC_ARGCHK(ct);
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+ LTC_ARGCHK(skey);
+
+ in = pt;
+ out = ct;
+ k = &skey->serpent.k[0];
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ s_serpent_accel_ecb_64_bit_load_four(&a, &b, &c, &d, in);
+ s_apply_order_enc_00(s_apply_key);
+ s_apply_order_enc_00(s_enc_0); s_apply_order_enc_01(s_apply_lk);
+ s_apply_order_enc_01(s_enc_1); s_apply_order_enc_02(s_apply_lk);
+ s_apply_order_enc_02(s_enc_2); s_apply_order_enc_03(s_apply_lk);
+ s_apply_order_enc_03(s_enc_3); s_apply_order_enc_04(s_apply_lk);
+ s_apply_order_enc_04(s_enc_4); s_apply_order_enc_05(s_apply_lk);
+ s_apply_order_enc_05(s_enc_5); s_apply_order_enc_06(s_apply_lk);
+ s_apply_order_enc_06(s_enc_6); s_apply_order_enc_07(s_apply_lk);
+ s_apply_order_enc_07(s_enc_7); s_apply_order_enc_08(s_apply_lk);
+ s_apply_order_enc_08(s_enc_0); s_apply_order_enc_09(s_apply_lk);
+ s_apply_order_enc_09(s_enc_1); s_apply_order_enc_10(s_apply_lk);
+ s_apply_order_enc_10(s_enc_2); s_apply_order_enc_11(s_apply_lk);
+ s_apply_order_enc_11(s_enc_3); s_apply_order_enc_12(s_apply_lk);
+ s_apply_order_enc_12(s_enc_4); s_apply_order_enc_13(s_apply_lk);
+ s_apply_order_enc_13(s_enc_5); s_apply_order_enc_14(s_apply_lk);
+ s_apply_order_enc_14(s_enc_6); s_apply_order_enc_15(s_apply_lk);
+ s_apply_order_enc_15(s_enc_7); s_apply_order_enc_16(s_apply_lk);
+ s_apply_order_enc_16(s_enc_0); s_apply_order_enc_17(s_apply_lk);
+ s_apply_order_enc_17(s_enc_1); s_apply_order_enc_18(s_apply_lk);
+ s_apply_order_enc_18(s_enc_2); s_apply_order_enc_19(s_apply_lk);
+ s_apply_order_enc_19(s_enc_3); s_apply_order_enc_20(s_apply_lk);
+ s_apply_order_enc_20(s_enc_4); s_apply_order_enc_21(s_apply_lk);
+ s_apply_order_enc_21(s_enc_5); s_apply_order_enc_22(s_apply_lk);
+ s_apply_order_enc_22(s_enc_6); s_apply_order_enc_23(s_apply_lk);
+ s_apply_order_enc_23(s_enc_7); s_apply_order_enc_24(s_apply_lk);
+ s_apply_order_enc_24(s_enc_0); s_apply_order_enc_25(s_apply_lk);
+ s_apply_order_enc_25(s_enc_1); s_apply_order_enc_26(s_apply_lk);
+ s_apply_order_enc_26(s_enc_2); s_apply_order_enc_27(s_apply_lk);
+ s_apply_order_enc_27(s_enc_3); s_apply_order_enc_28(s_apply_lk);
+ s_apply_order_enc_28(s_enc_4); s_apply_order_enc_29(s_apply_lk);
+ s_apply_order_enc_29(s_enc_5); s_apply_order_enc_30(s_apply_lk);
+ s_apply_order_enc_30(s_enc_6); s_apply_order_enc_31(s_apply_lk);
+ s_apply_order_enc_31(s_enc_7); s_apply_order_enc_32(s_apply_key);
+ s_serpent_accel_ecb_64_bit_store_four(&a, &b, &c, &d, out);
+ in += blocks_at_a_time * serpent_block_len;
+ out += blocks_at_a_time * serpent_block_len;
+ }
+ return CRYPT_OK;
+
+ #undef blocks_at_a_time
+ #undef s_do_split_general
+ #undef s_do_split_lo
+ #undef s_do_split_hi
+ #undef s_do_join
+ #undef s_do_broadcast
+ #undef s_do_asgn
+ #undef s_do_or
+ #undef s_do_xor
+ #undef s_do_and
+ #undef s_do_not
+ #undef s_do_rol
+ #undef s_do_ror
+ #undef s_do_shl
+}
+
+static LTC_INLINE int s_serpent_accel_ecb_decrypt_64_bit(const unsigned char *ct, unsigned char *pt, unsigned long blocks, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (64 / 32)
+ #define s_do_split_general(x, i) ((ulong32)((x) >> ((i) * 32)))
+ #define s_do_split_lo(x) s_do_split_general(x, 0)
+ #define s_do_split_hi(x) s_do_split_general(x, 1)
+ #define s_do_join(lo, hi) ((((ulong64)(hi)) << 32) | ((ulong64)(lo)))
+ #define s_do_broadcast(x) s_do_join(x, x)
+ #define s_do_asgn(a, b) a = b
+ #define s_do_or(a, b) a |= b
+ #define s_do_xor(a, b) a ^= b
+ #define s_do_and(a, b) a &= b
+ #define s_do_not(a, b) a =~ b
+ #define s_do_rol(x, i) x = s_do_join( \
+ ROL(s_do_split_lo(x), i), \
+ ROL(s_do_split_hi(x), i))
+ #define s_do_ror(x, i) x = s_do_join( \
+ ROR(s_do_split_lo(x), i), \
+ ROR(s_do_split_hi(x), i))
+ #define s_do_shl(a, b, c) a = s_do_join( \
+ (s_do_split_lo(b) << c), \
+ (s_do_split_hi(b) << c))
+
+ const unsigned char *in;
+ unsigned char *out;
+ const ulong32* k;
+ unsigned long iblock;
+ ulong64 a, b, c, d, e;
+
+ LTC_ARGCHK(ct);
+ LTC_ARGCHK(pt);
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+ LTC_ARGCHK(skey);
+
+ in = ct;
+ out = pt;
+ k = &skey->serpent.k[0];
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ s_serpent_accel_ecb_64_bit_load_four(&a, &b, &c, &d, in);
+ s_apply_order_dec_32(s_apply_key);
+ s_apply_order_dec_32(s_dec_7); s_apply_order_dec_31(s_apply_kl);
+ s_apply_order_dec_31(s_dec_6); s_apply_order_dec_30(s_apply_kl);
+ s_apply_order_dec_30(s_dec_5); s_apply_order_dec_29(s_apply_kl);
+ s_apply_order_dec_29(s_dec_4); s_apply_order_dec_28(s_apply_kl);
+ s_apply_order_dec_28(s_dec_3); s_apply_order_dec_27(s_apply_kl);
+ s_apply_order_dec_27(s_dec_2); s_apply_order_dec_26(s_apply_kl);
+ s_apply_order_dec_26(s_dec_1); s_apply_order_dec_25(s_apply_kl);
+ s_apply_order_dec_25(s_dec_0); s_apply_order_dec_24(s_apply_kl);
+ s_apply_order_dec_24(s_dec_7); s_apply_order_dec_23(s_apply_kl);
+ s_apply_order_dec_23(s_dec_6); s_apply_order_dec_22(s_apply_kl);
+ s_apply_order_dec_22(s_dec_5); s_apply_order_dec_21(s_apply_kl);
+ s_apply_order_dec_21(s_dec_4); s_apply_order_dec_20(s_apply_kl);
+ s_apply_order_dec_20(s_dec_3); s_apply_order_dec_19(s_apply_kl);
+ s_apply_order_dec_19(s_dec_2); s_apply_order_dec_18(s_apply_kl);
+ s_apply_order_dec_18(s_dec_1); s_apply_order_dec_17(s_apply_kl);
+ s_apply_order_dec_17(s_dec_0); s_apply_order_dec_16(s_apply_kl);
+ s_apply_order_dec_16(s_dec_7); s_apply_order_dec_15(s_apply_kl);
+ s_apply_order_dec_15(s_dec_6); s_apply_order_dec_14(s_apply_kl);
+ s_apply_order_dec_14(s_dec_5); s_apply_order_dec_13(s_apply_kl);
+ s_apply_order_dec_13(s_dec_4); s_apply_order_dec_12(s_apply_kl);
+ s_apply_order_dec_12(s_dec_3); s_apply_order_dec_11(s_apply_kl);
+ s_apply_order_dec_11(s_dec_2); s_apply_order_dec_10(s_apply_kl);
+ s_apply_order_dec_10(s_dec_1); s_apply_order_dec_09(s_apply_kl);
+ s_apply_order_dec_09(s_dec_0); s_apply_order_dec_08(s_apply_kl);
+ s_apply_order_dec_08(s_dec_7); s_apply_order_dec_07(s_apply_kl);
+ s_apply_order_dec_07(s_dec_6); s_apply_order_dec_06(s_apply_kl);
+ s_apply_order_dec_06(s_dec_5); s_apply_order_dec_05(s_apply_kl);
+ s_apply_order_dec_05(s_dec_4); s_apply_order_dec_04(s_apply_kl);
+ s_apply_order_dec_04(s_dec_3); s_apply_order_dec_03(s_apply_kl);
+ s_apply_order_dec_03(s_dec_2); s_apply_order_dec_02(s_apply_kl);
+ s_apply_order_dec_02(s_dec_1); s_apply_order_dec_01(s_apply_kl);
+ s_apply_order_dec_01(s_dec_0); s_apply_order_dec_00(s_apply_key);
+ s_serpent_accel_ecb_64_bit_store_four(&c, &d, &b, &e, out);
+ in += blocks_at_a_time * serpent_block_len;
+ out += blocks_at_a_time * serpent_block_len;
+ }
+ return CRYPT_OK;
+
+ #undef blocks_at_a_time
+ #undef s_do_split_general
+ #undef s_do_split_lo
+ #undef s_do_split_hi
+ #undef s_do_join
+ #undef s_do_broadcast
+ #undef s_do_asgn
+ #undef s_do_or
+ #undef s_do_xor
+ #undef s_do_and
+ #undef s_do_not
+ #undef s_do_rol
+ #undef s_do_ror
+ #undef s_do_shl
+}
- err = s_setup_key(key, keylen, 32, skey->serpent.k);
-#ifdef LTC_CLEAN_STACK
- burn_stack(sizeof(ulong32) * 14 + sizeof(int));
#endif
- return err;
+
+#if defined LTC_ARCH_X86
+#if !defined LTC_S_X86_CPUID
+#define LTC_S_X86_CPUID
+#if defined(__GNUC__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-function"
+#endif
+#if defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wunused-function"
+#endif
+#include /* _xgetbv */
+#if defined(__clang__)
+#pragma clang diagnostic pop
+#endif
+#if defined(__GNUC__)
+#pragma GCC diagnostic pop
+#endif
+#if defined _MSC_VER
+#include /* __cpuid */
+#endif /* _MSC_VER */
+static LTC_INLINE ulong64 LTC_XSAVE_TARGET s_x86_xgetbv0(void)
+{
+ return _xgetbv(0);
+}
+static LTC_INLINE void s_x86_cpuid(int* regs, int leaf)
+{
+#if defined _MSC_VER
+ __cpuid(regs, leaf);
+#else /* _MSC_VER */
+ int a, b, c, d;
+ a = leaf;
+ b = c = d = 0;
+ __asm__ volatile ("cpuid"
+ :"=a"(a), "=b"(b), "=c"(c), "=d"(d)
+ :"a"(a), "c"(c)
+ );
+ regs[0] = a;
+ regs[1] = b;
+ regs[2] = c;
+ regs[3] = d;
+#endif /* _MSC_VER */
}
+#endif /* LTC_S_X86_CPUID */
+#endif /* LTC_ARCH_X86 */
+
+#if defined LTC_SERPENT_ACCEL_128_BIT_X86_SSE2
+
+#include /* SSE2 __m128i _mm_and_si128 _mm_cmpeq_epi32 _mm_loadu_si128 _mm_or_si128 _mm_set1_epi32 _mm_slli_epi32 _mm_srli_epi32 _mm_storeu_si128 _mm_unpackhi_epi32 _mm_unpackhi_epi64 _mm_unpacklo_epi32 _mm_unpacklo_epi64 _mm_xor_si128 */
+
+static LTC_INLINE int s_x86_sse2_is_supported(void)
+{
+ static int is_initialized = 0;
+ static int is_supported = 0;
+
+ if (is_initialized == 0) {
+ int regs[4];
+ int sse2;
+
+ s_x86_cpuid(regs, 1);
+ sse2 = ((((unsigned int)(regs[3])) >> 26) & 1u) != 0; /* SSE2, leaf 1, edx, bit 26 */
+ is_supported = sse2;
+ is_initialized = 1;
+ }
+ return is_supported;
+}
+
+static LTC_INLINE void s_serpent_accel_ecb_128_bit_sse2_load_one(__m128i *x, const unsigned char *bytes)
+{
+ *x = _mm_loadu_si128(((const __m128i*)(bytes)));
+}
+
+static LTC_INLINE void s_serpent_accel_ecb_128_bit_sse2_store_one(const __m128i *x, unsigned char *bytes)
+{
+ _mm_storeu_si128(((__m128i*)(bytes)), *x);
+}
+
+static LTC_INLINE void s_serpent_accel_128_bit_sse2_load_four(__m128i *pa, __m128i *pb, __m128i *pc, __m128i *pd, const unsigned char *bytes)
+{
+ __m128i ia, ib, ic, id;
+ __m128i ta, tb, tc, td;
+ __m128i ra, rb, rc, rd;
+
+ s_serpent_accel_ecb_128_bit_sse2_load_one(&ia, &bytes[0 * sizeof(__m128i)]);
+ s_serpent_accel_ecb_128_bit_sse2_load_one(&ib, &bytes[1 * sizeof(__m128i)]);
+ s_serpent_accel_ecb_128_bit_sse2_load_one(&ic, &bytes[2 * sizeof(__m128i)]);
+ s_serpent_accel_ecb_128_bit_sse2_load_one(&id, &bytes[3 * sizeof(__m128i)]);
+ ta = _mm_unpacklo_epi32(ia, ib);
+ tb = _mm_unpacklo_epi32(ic, id);
+ tc = _mm_unpackhi_epi32(ia, ib);
+ td = _mm_unpackhi_epi32(ic, id);
+ ra = _mm_unpacklo_epi64(ta, tb);
+ rb = _mm_unpackhi_epi64(ta, tb);
+ rc = _mm_unpacklo_epi64(tc, td);
+ rd = _mm_unpackhi_epi64(tc, td);
+ *pa = ra;
+ *pb = rb;
+ *pc = rc;
+ *pd = rd;
+}
+
+static LTC_INLINE void s_serpent_accel_128_bit_sse2_store_four(const __m128i *pa, const __m128i *pb, const __m128i *pc, const __m128i *pd, unsigned char *bytes)
+{
+ __m128i ia, ib, ic, id;
+ __m128i ta, tb, tc, td;
+ __m128i ra, rb, rc, rd;
+
+ ia = *pa;
+ ib = *pb;
+ ic = *pc;
+ id = *pd;
+ ta = _mm_unpacklo_epi32(ia, ib);
+ tb = _mm_unpacklo_epi32(ic, id);
+ tc = _mm_unpackhi_epi32(ia, ib);
+ td = _mm_unpackhi_epi32(ic, id);
+ ra = _mm_unpacklo_epi64(ta, tb);
+ rb = _mm_unpackhi_epi64(ta, tb);
+ rc = _mm_unpacklo_epi64(tc, td);
+ rd = _mm_unpackhi_epi64(tc, td);
+ s_serpent_accel_ecb_128_bit_sse2_store_one(&ra, &bytes[0 * sizeof(__m128i)]);
+ s_serpent_accel_ecb_128_bit_sse2_store_one(&rb, &bytes[1 * sizeof(__m128i)]);
+ s_serpent_accel_ecb_128_bit_sse2_store_one(&rc, &bytes[2 * sizeof(__m128i)]);
+ s_serpent_accel_ecb_128_bit_sse2_store_one(&rd, &bytes[3 * sizeof(__m128i)]);
+}
+
+static LTC_INLINE int s_serpent_accel_ecb_encrypt_128_bit_sse2(const unsigned char *pt, unsigned char *ct, unsigned long blocks, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (128 / 32)
+ #define s_do_broadcast(x) _mm_set1_epi32(*((const int *)(&(x))))
+ #define s_do_asgn(a, b) a = b
+ #define s_do_or(a, b) a = _mm_or_si128(a, b)
+ #define s_do_xor(a, b) a = _mm_xor_si128(a, b)
+ #define s_do_and(a, b) a = _mm_and_si128(a, b)
+ #define s_do_not(a, b) a = _mm_xor_si128(b, _mm_cmpeq_epi32(b, b))
+ #define s_do_rol(x, i) x = _mm_xor_si128(_mm_slli_epi32(x, i), _mm_srli_epi32(x, 32 - i))
+ #define s_do_ror(x, i) x = _mm_xor_si128(_mm_srli_epi32(x, i), _mm_slli_epi32(x, 32 - i))
+ #define s_do_shl(a, b, c) a = _mm_slli_epi32(b, c)
+
+ const unsigned char *in;
+ unsigned char *out;
+ const ulong32* k;
+ unsigned long iblock;
+ __m128i a, b, c, d, e;
+
+ LTC_ARGCHK(pt);
+ LTC_ARGCHK(ct);
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+
+ in = pt;
+ out = ct;
+ k = &skey->serpent.k[0];
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ s_serpent_accel_128_bit_sse2_load_four(&a, &b, &c, &d, in);
+ s_apply_order_enc_00(s_apply_key);
+ s_apply_order_enc_00(s_enc_0); s_apply_order_enc_01(s_apply_lk);
+ s_apply_order_enc_01(s_enc_1); s_apply_order_enc_02(s_apply_lk);
+ s_apply_order_enc_02(s_enc_2); s_apply_order_enc_03(s_apply_lk);
+ s_apply_order_enc_03(s_enc_3); s_apply_order_enc_04(s_apply_lk);
+ s_apply_order_enc_04(s_enc_4); s_apply_order_enc_05(s_apply_lk);
+ s_apply_order_enc_05(s_enc_5); s_apply_order_enc_06(s_apply_lk);
+ s_apply_order_enc_06(s_enc_6); s_apply_order_enc_07(s_apply_lk);
+ s_apply_order_enc_07(s_enc_7); s_apply_order_enc_08(s_apply_lk);
+ s_apply_order_enc_08(s_enc_0); s_apply_order_enc_09(s_apply_lk);
+ s_apply_order_enc_09(s_enc_1); s_apply_order_enc_10(s_apply_lk);
+ s_apply_order_enc_10(s_enc_2); s_apply_order_enc_11(s_apply_lk);
+ s_apply_order_enc_11(s_enc_3); s_apply_order_enc_12(s_apply_lk);
+ s_apply_order_enc_12(s_enc_4); s_apply_order_enc_13(s_apply_lk);
+ s_apply_order_enc_13(s_enc_5); s_apply_order_enc_14(s_apply_lk);
+ s_apply_order_enc_14(s_enc_6); s_apply_order_enc_15(s_apply_lk);
+ s_apply_order_enc_15(s_enc_7); s_apply_order_enc_16(s_apply_lk);
+ s_apply_order_enc_16(s_enc_0); s_apply_order_enc_17(s_apply_lk);
+ s_apply_order_enc_17(s_enc_1); s_apply_order_enc_18(s_apply_lk);
+ s_apply_order_enc_18(s_enc_2); s_apply_order_enc_19(s_apply_lk);
+ s_apply_order_enc_19(s_enc_3); s_apply_order_enc_20(s_apply_lk);
+ s_apply_order_enc_20(s_enc_4); s_apply_order_enc_21(s_apply_lk);
+ s_apply_order_enc_21(s_enc_5); s_apply_order_enc_22(s_apply_lk);
+ s_apply_order_enc_22(s_enc_6); s_apply_order_enc_23(s_apply_lk);
+ s_apply_order_enc_23(s_enc_7); s_apply_order_enc_24(s_apply_lk);
+ s_apply_order_enc_24(s_enc_0); s_apply_order_enc_25(s_apply_lk);
+ s_apply_order_enc_25(s_enc_1); s_apply_order_enc_26(s_apply_lk);
+ s_apply_order_enc_26(s_enc_2); s_apply_order_enc_27(s_apply_lk);
+ s_apply_order_enc_27(s_enc_3); s_apply_order_enc_28(s_apply_lk);
+ s_apply_order_enc_28(s_enc_4); s_apply_order_enc_29(s_apply_lk);
+ s_apply_order_enc_29(s_enc_5); s_apply_order_enc_30(s_apply_lk);
+ s_apply_order_enc_30(s_enc_6); s_apply_order_enc_31(s_apply_lk);
+ s_apply_order_enc_31(s_enc_7); s_apply_order_enc_32(s_apply_key);
+ s_serpent_accel_128_bit_sse2_store_four(&a, &b, &c, &d, out);
+ in += blocks_at_a_time * serpent_block_len;
+ out += blocks_at_a_time * serpent_block_len;
+ }
+ return CRYPT_OK;
+
+ #undef blocks_at_a_time
+ #undef s_do_broadcast
+ #undef s_do_asgn
+ #undef s_do_or
+ #undef s_do_xor
+ #undef s_do_and
+ #undef s_do_not
+ #undef s_do_rol
+ #undef s_do_ror
+ #undef s_do_shl
+}
+
+static LTC_INLINE int s_serpent_accel_ecb_decrypt_128_bit_sse2(const unsigned char *ct, unsigned char *pt, unsigned long blocks, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (128 / 32)
+ #define s_do_broadcast(x) _mm_set1_epi32(*((const int *)(&(x))))
+ #define s_do_asgn(a, b) a = b
+ #define s_do_or(a, b) a = _mm_or_si128(a, b)
+ #define s_do_xor(a, b) a = _mm_xor_si128(a, b)
+ #define s_do_and(a, b) a = _mm_and_si128(a, b)
+ #define s_do_not(a, b) a = _mm_xor_si128(b, _mm_cmpeq_epi32(b, b))
+ #define s_do_rol(x, i) x = _mm_xor_si128(_mm_slli_epi32(x, i), _mm_srli_epi32(x, 32 - i))
+ #define s_do_ror(x, i) x = _mm_xor_si128(_mm_srli_epi32(x, i), _mm_slli_epi32(x, 32 - i))
+ #define s_do_shl(a, b, c) a = _mm_slli_epi32(b, c)
+
+ const unsigned char *in;
+ unsigned char *out;
+ const ulong32* k;
+ unsigned long iblock;
+ __m128i a, b, c, d, e;
+
+ LTC_ARGCHK(ct);
+ LTC_ARGCHK(pt);
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+
+ in = ct;
+ out = pt;
+ k = &skey->serpent.k[0];
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ s_serpent_accel_128_bit_sse2_load_four(&a, &b, &c, &d, in);
+ s_apply_order_dec_32(s_apply_key);
+ s_apply_order_dec_32(s_dec_7); s_apply_order_dec_31(s_apply_kl);
+ s_apply_order_dec_31(s_dec_6); s_apply_order_dec_30(s_apply_kl);
+ s_apply_order_dec_30(s_dec_5); s_apply_order_dec_29(s_apply_kl);
+ s_apply_order_dec_29(s_dec_4); s_apply_order_dec_28(s_apply_kl);
+ s_apply_order_dec_28(s_dec_3); s_apply_order_dec_27(s_apply_kl);
+ s_apply_order_dec_27(s_dec_2); s_apply_order_dec_26(s_apply_kl);
+ s_apply_order_dec_26(s_dec_1); s_apply_order_dec_25(s_apply_kl);
+ s_apply_order_dec_25(s_dec_0); s_apply_order_dec_24(s_apply_kl);
+ s_apply_order_dec_24(s_dec_7); s_apply_order_dec_23(s_apply_kl);
+ s_apply_order_dec_23(s_dec_6); s_apply_order_dec_22(s_apply_kl);
+ s_apply_order_dec_22(s_dec_5); s_apply_order_dec_21(s_apply_kl);
+ s_apply_order_dec_21(s_dec_4); s_apply_order_dec_20(s_apply_kl);
+ s_apply_order_dec_20(s_dec_3); s_apply_order_dec_19(s_apply_kl);
+ s_apply_order_dec_19(s_dec_2); s_apply_order_dec_18(s_apply_kl);
+ s_apply_order_dec_18(s_dec_1); s_apply_order_dec_17(s_apply_kl);
+ s_apply_order_dec_17(s_dec_0); s_apply_order_dec_16(s_apply_kl);
+ s_apply_order_dec_16(s_dec_7); s_apply_order_dec_15(s_apply_kl);
+ s_apply_order_dec_15(s_dec_6); s_apply_order_dec_14(s_apply_kl);
+ s_apply_order_dec_14(s_dec_5); s_apply_order_dec_13(s_apply_kl);
+ s_apply_order_dec_13(s_dec_4); s_apply_order_dec_12(s_apply_kl);
+ s_apply_order_dec_12(s_dec_3); s_apply_order_dec_11(s_apply_kl);
+ s_apply_order_dec_11(s_dec_2); s_apply_order_dec_10(s_apply_kl);
+ s_apply_order_dec_10(s_dec_1); s_apply_order_dec_09(s_apply_kl);
+ s_apply_order_dec_09(s_dec_0); s_apply_order_dec_08(s_apply_kl);
+ s_apply_order_dec_08(s_dec_7); s_apply_order_dec_07(s_apply_kl);
+ s_apply_order_dec_07(s_dec_6); s_apply_order_dec_06(s_apply_kl);
+ s_apply_order_dec_06(s_dec_5); s_apply_order_dec_05(s_apply_kl);
+ s_apply_order_dec_05(s_dec_4); s_apply_order_dec_04(s_apply_kl);
+ s_apply_order_dec_04(s_dec_3); s_apply_order_dec_03(s_apply_kl);
+ s_apply_order_dec_03(s_dec_2); s_apply_order_dec_02(s_apply_kl);
+ s_apply_order_dec_02(s_dec_1); s_apply_order_dec_01(s_apply_kl);
+ s_apply_order_dec_01(s_dec_0); s_apply_order_dec_00(s_apply_key);
+ s_serpent_accel_128_bit_sse2_store_four(&c, &d, &b, &e, out);
+ in += blocks_at_a_time * serpent_block_len;
+ out += blocks_at_a_time * serpent_block_len;
+ }
+ return CRYPT_OK;
+
+ #undef blocks_at_a_time
+ #undef s_do_broadcast
+ #undef s_do_asgn
+ #undef s_do_or
+ #undef s_do_xor
+ #undef s_do_and
+ #undef s_do_not
+ #undef s_do_rol
+ #undef s_do_ror
+ #undef s_do_shl
+}
+
+#endif
+
+#if defined LTC_SERPENT_ACCEL_256_BIT_X86_AVX2
+
+#if defined(__GNUC__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-function"
+#endif
+#if defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wunused-function"
+#endif
+#include /* AVX2 __m256i _mm256_and_si256 _mm256_cmpeq_epi32 _mm256_loadu_si256 _mm256_or_si256 _mm256_set1_epi32 _mm256_slli_epi32 _mm256_srli_epi32 _mm256_storeu_si256 _mm256_unpackhi_epi32 _mm256_unpackhi_epi64 _mm256_unpacklo_epi32 _mm256_unpacklo_epi64 _mm256_xor_si256 */
+#if defined(__clang__)
+#pragma clang diagnostic pop
+#endif
+#if defined(__GNUC__)
+#pragma GCC diagnostic pop
+#endif
+
+static LTC_INLINE int s_is_supported_256_bit_avx2(void)
+{
+ static int is_initialized = 0;
+ static int is_supported = 0;
+
+ if (is_initialized == 0) {
+ int regs[4];
+ int osxsave, avx2, ymm;
+ ulong64 xcr0;
+
+ is_initialized = 1;
+ s_x86_cpuid(regs, 0);
+ if (regs[0] >= 7) {
+ s_x86_cpuid(regs, 1);
+ osxsave = ((((unsigned int)(regs[2])) >> 27) & 1u) != 0; /* OSXSAVE, leaf 1, ecx, bit 27 */
+ s_x86_cpuid(regs, 7);
+ avx2 = ((((unsigned int)(regs[1])) >> 5) & 1u) != 0; /* AVX2, leaf 7, ebx, bit 5 */
+ if (osxsave) {
+ xcr0 = s_x86_xgetbv0();
+ ymm = (xcr0 & 0x6) == 0x6; /* 128bit SSE xmm + 256bit AVX ymm */
+ is_supported = osxsave && avx2 && ymm;
+ }
+ }
+ }
+ return is_supported;
+}
+
+static LTC_INLINE void LTC_AVX2_TARGET s_serpent_accel_ecb_256_bit_avx2_load_one(__m256i *x, const unsigned char *bytes)
+{
+ *x = _mm256_loadu_si256(((const __m256i*)(bytes)));
+}
+
+static LTC_INLINE void LTC_AVX2_TARGET s_serpent_accel_ecb_256_bit_avx2_store_one(const __m256i *x, unsigned char *bytes)
+{
+ _mm256_storeu_si256(((__m256i*)(bytes)), *x);
+}
+
+static LTC_INLINE void LTC_AVX2_TARGET s_serpent_accel_256_bit_avx2_load_four(__m256i *pa, __m256i *pb, __m256i *pc, __m256i *pd, const unsigned char *bytes)
+{
+ __m256i ia, ib, ic, id;
+ __m256i ta, tb, tc, td;
+ __m256i ra, rb, rc, rd;
+
+ s_serpent_accel_ecb_256_bit_avx2_load_one(&ia, &bytes[0 * sizeof(__m256i)]);
+ s_serpent_accel_ecb_256_bit_avx2_load_one(&ib, &bytes[1 * sizeof(__m256i)]);
+ s_serpent_accel_ecb_256_bit_avx2_load_one(&ic, &bytes[2 * sizeof(__m256i)]);
+ s_serpent_accel_ecb_256_bit_avx2_load_one(&id, &bytes[3 * sizeof(__m256i)]);
+ ta = _mm256_unpacklo_epi32(ia, ib);
+ tb = _mm256_unpacklo_epi32(ic, id);
+ tc = _mm256_unpackhi_epi32(ia, ib);
+ td = _mm256_unpackhi_epi32(ic, id);
+ ra = _mm256_unpacklo_epi64(ta, tb);
+ rb = _mm256_unpackhi_epi64(ta, tb);
+ rc = _mm256_unpacklo_epi64(tc, td);
+ rd = _mm256_unpackhi_epi64(tc, td);
+ *pa = ra;
+ *pb = rb;
+ *pc = rc;
+ *pd = rd;
+}
+
+static LTC_INLINE void LTC_AVX2_TARGET s_serpent_accel_256_bit_avx2_store_four(const __m256i *pa, const __m256i *pb, const __m256i *pc, const __m256i *pd, unsigned char *bytes)
+{
+ __m256i ia, ib, ic, id;
+ __m256i ta, tb, tc, td;
+ __m256i ra, rb, rc, rd;
+
+ ia = *pa;
+ ib = *pb;
+ ic = *pc;
+ id = *pd;
+ ta = _mm256_unpacklo_epi32(ia, ib);
+ tb = _mm256_unpacklo_epi32(ic, id);
+ tc = _mm256_unpackhi_epi32(ia, ib);
+ td = _mm256_unpackhi_epi32(ic, id);
+ ra = _mm256_unpacklo_epi64(ta, tb);
+ rb = _mm256_unpackhi_epi64(ta, tb);
+ rc = _mm256_unpacklo_epi64(tc, td);
+ rd = _mm256_unpackhi_epi64(tc, td);
+ s_serpent_accel_ecb_256_bit_avx2_store_one(&ra, &bytes[0 * sizeof(__m256i)]);
+ s_serpent_accel_ecb_256_bit_avx2_store_one(&rb, &bytes[1 * sizeof(__m256i)]);
+ s_serpent_accel_ecb_256_bit_avx2_store_one(&rc, &bytes[2 * sizeof(__m256i)]);
+ s_serpent_accel_ecb_256_bit_avx2_store_one(&rd, &bytes[3 * sizeof(__m256i)]);
+}
+
+static LTC_INLINE int LTC_AVX2_TARGET s_serpent_accel_ecb_encrypt_256_bit_avx2(const unsigned char *pt, unsigned char *ct, unsigned long blocks, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (256 / 32)
+ #define s_do_broadcast(x) _mm256_set1_epi32(*((const int *)(&(x))))
+ #define s_do_asgn(a, b) a = b
+ #define s_do_or(a, b) a = _mm256_or_si256(a, b)
+ #define s_do_xor(a, b) a = _mm256_xor_si256(a, b)
+ #define s_do_and(a, b) a = _mm256_and_si256(a, b)
+ #define s_do_not(a, b) a = _mm256_xor_si256(b, _mm256_cmpeq_epi32(b, b))
+ #define s_do_rol(x, i) x = _mm256_or_si256(_mm256_slli_epi32(x, i), _mm256_srli_epi32(x, 32 - i))
+ #define s_do_ror(x, i) x = _mm256_or_si256(_mm256_srli_epi32(x, i), _mm256_slli_epi32(x, 32 - i))
+ #define s_do_shl(a, b, c) a = _mm256_slli_epi32(b, c)
+
+ const unsigned char *in;
+ unsigned char *out;
+ const ulong32* k;
+ unsigned long iblock;
+ __m256i a, b, c, d, e;
+
+ LTC_ARGCHK(pt);
+ LTC_ARGCHK(ct);
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+
+ in = pt;
+ out = ct;
+ k = &skey->serpent.k[0];
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ s_serpent_accel_256_bit_avx2_load_four(&a, &b, &c, &d, in);
+ s_apply_order_enc_00(s_apply_key);
+ s_apply_order_enc_00(s_enc_0); s_apply_order_enc_01(s_apply_lk);
+ s_apply_order_enc_01(s_enc_1); s_apply_order_enc_02(s_apply_lk);
+ s_apply_order_enc_02(s_enc_2); s_apply_order_enc_03(s_apply_lk);
+ s_apply_order_enc_03(s_enc_3); s_apply_order_enc_04(s_apply_lk);
+ s_apply_order_enc_04(s_enc_4); s_apply_order_enc_05(s_apply_lk);
+ s_apply_order_enc_05(s_enc_5); s_apply_order_enc_06(s_apply_lk);
+ s_apply_order_enc_06(s_enc_6); s_apply_order_enc_07(s_apply_lk);
+ s_apply_order_enc_07(s_enc_7); s_apply_order_enc_08(s_apply_lk);
+ s_apply_order_enc_08(s_enc_0); s_apply_order_enc_09(s_apply_lk);
+ s_apply_order_enc_09(s_enc_1); s_apply_order_enc_10(s_apply_lk);
+ s_apply_order_enc_10(s_enc_2); s_apply_order_enc_11(s_apply_lk);
+ s_apply_order_enc_11(s_enc_3); s_apply_order_enc_12(s_apply_lk);
+ s_apply_order_enc_12(s_enc_4); s_apply_order_enc_13(s_apply_lk);
+ s_apply_order_enc_13(s_enc_5); s_apply_order_enc_14(s_apply_lk);
+ s_apply_order_enc_14(s_enc_6); s_apply_order_enc_15(s_apply_lk);
+ s_apply_order_enc_15(s_enc_7); s_apply_order_enc_16(s_apply_lk);
+ s_apply_order_enc_16(s_enc_0); s_apply_order_enc_17(s_apply_lk);
+ s_apply_order_enc_17(s_enc_1); s_apply_order_enc_18(s_apply_lk);
+ s_apply_order_enc_18(s_enc_2); s_apply_order_enc_19(s_apply_lk);
+ s_apply_order_enc_19(s_enc_3); s_apply_order_enc_20(s_apply_lk);
+ s_apply_order_enc_20(s_enc_4); s_apply_order_enc_21(s_apply_lk);
+ s_apply_order_enc_21(s_enc_5); s_apply_order_enc_22(s_apply_lk);
+ s_apply_order_enc_22(s_enc_6); s_apply_order_enc_23(s_apply_lk);
+ s_apply_order_enc_23(s_enc_7); s_apply_order_enc_24(s_apply_lk);
+ s_apply_order_enc_24(s_enc_0); s_apply_order_enc_25(s_apply_lk);
+ s_apply_order_enc_25(s_enc_1); s_apply_order_enc_26(s_apply_lk);
+ s_apply_order_enc_26(s_enc_2); s_apply_order_enc_27(s_apply_lk);
+ s_apply_order_enc_27(s_enc_3); s_apply_order_enc_28(s_apply_lk);
+ s_apply_order_enc_28(s_enc_4); s_apply_order_enc_29(s_apply_lk);
+ s_apply_order_enc_29(s_enc_5); s_apply_order_enc_30(s_apply_lk);
+ s_apply_order_enc_30(s_enc_6); s_apply_order_enc_31(s_apply_lk);
+ s_apply_order_enc_31(s_enc_7); s_apply_order_enc_32(s_apply_key);
+ s_serpent_accel_256_bit_avx2_store_four(&a, &b, &c, &d, out);
+ in += blocks_at_a_time * serpent_block_len;
+ out += blocks_at_a_time * serpent_block_len;
+ }
+ return CRYPT_OK;
+
+ #undef blocks_at_a_time
+ #undef s_do_broadcast
+ #undef s_do_asgn
+ #undef s_do_or
+ #undef s_do_xor
+ #undef s_do_and
+ #undef s_do_not
+ #undef s_do_rol
+ #undef s_do_ror
+ #undef s_do_shl
+}
+
+static LTC_INLINE int LTC_AVX2_TARGET s_serpent_accel_ecb_decrypt_256_bit_avx2(const unsigned char *ct, unsigned char *pt, unsigned long blocks, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (256 / 32)
+ #define s_do_broadcast(x) _mm256_set1_epi32(*((const int *)(&(x))))
+ #define s_do_asgn(a, b) a = b
+ #define s_do_or(a, b) a = _mm256_or_si256(a, b)
+ #define s_do_xor(a, b) a = _mm256_xor_si256(a, b)
+ #define s_do_and(a, b) a = _mm256_and_si256(a, b)
+ #define s_do_not(a, b) a = _mm256_xor_si256(b, _mm256_cmpeq_epi32(b, b))
+ #define s_do_rol(x, i) x = _mm256_or_si256(_mm256_slli_epi32(x, i), _mm256_srli_epi32(x, 32 - i))
+ #define s_do_ror(x, i) x = _mm256_or_si256(_mm256_srli_epi32(x, i), _mm256_slli_epi32(x, 32 - i))
+ #define s_do_shl(a, b, c) a = _mm256_slli_epi32(b, c)
+
+ const unsigned char *in;
+ unsigned char *out;
+ const ulong32* k;
+ unsigned long iblock;
+ __m256i a, b, c, d, e;
+
+ LTC_ARGCHK(ct);
+ LTC_ARGCHK(pt);
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+
+ in = ct;
+ out = pt;
+ k = &skey->serpent.k[0];
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ s_serpent_accel_256_bit_avx2_load_four(&a, &b, &c, &d, in);
+ s_apply_order_dec_32(s_apply_key);
+ s_apply_order_dec_32(s_dec_7); s_apply_order_dec_31(s_apply_kl);
+ s_apply_order_dec_31(s_dec_6); s_apply_order_dec_30(s_apply_kl);
+ s_apply_order_dec_30(s_dec_5); s_apply_order_dec_29(s_apply_kl);
+ s_apply_order_dec_29(s_dec_4); s_apply_order_dec_28(s_apply_kl);
+ s_apply_order_dec_28(s_dec_3); s_apply_order_dec_27(s_apply_kl);
+ s_apply_order_dec_27(s_dec_2); s_apply_order_dec_26(s_apply_kl);
+ s_apply_order_dec_26(s_dec_1); s_apply_order_dec_25(s_apply_kl);
+ s_apply_order_dec_25(s_dec_0); s_apply_order_dec_24(s_apply_kl);
+ s_apply_order_dec_24(s_dec_7); s_apply_order_dec_23(s_apply_kl);
+ s_apply_order_dec_23(s_dec_6); s_apply_order_dec_22(s_apply_kl);
+ s_apply_order_dec_22(s_dec_5); s_apply_order_dec_21(s_apply_kl);
+ s_apply_order_dec_21(s_dec_4); s_apply_order_dec_20(s_apply_kl);
+ s_apply_order_dec_20(s_dec_3); s_apply_order_dec_19(s_apply_kl);
+ s_apply_order_dec_19(s_dec_2); s_apply_order_dec_18(s_apply_kl);
+ s_apply_order_dec_18(s_dec_1); s_apply_order_dec_17(s_apply_kl);
+ s_apply_order_dec_17(s_dec_0); s_apply_order_dec_16(s_apply_kl);
+ s_apply_order_dec_16(s_dec_7); s_apply_order_dec_15(s_apply_kl);
+ s_apply_order_dec_15(s_dec_6); s_apply_order_dec_14(s_apply_kl);
+ s_apply_order_dec_14(s_dec_5); s_apply_order_dec_13(s_apply_kl);
+ s_apply_order_dec_13(s_dec_4); s_apply_order_dec_12(s_apply_kl);
+ s_apply_order_dec_12(s_dec_3); s_apply_order_dec_11(s_apply_kl);
+ s_apply_order_dec_11(s_dec_2); s_apply_order_dec_10(s_apply_kl);
+ s_apply_order_dec_10(s_dec_1); s_apply_order_dec_09(s_apply_kl);
+ s_apply_order_dec_09(s_dec_0); s_apply_order_dec_08(s_apply_kl);
+ s_apply_order_dec_08(s_dec_7); s_apply_order_dec_07(s_apply_kl);
+ s_apply_order_dec_07(s_dec_6); s_apply_order_dec_06(s_apply_kl);
+ s_apply_order_dec_06(s_dec_5); s_apply_order_dec_05(s_apply_kl);
+ s_apply_order_dec_05(s_dec_4); s_apply_order_dec_04(s_apply_kl);
+ s_apply_order_dec_04(s_dec_3); s_apply_order_dec_03(s_apply_kl);
+ s_apply_order_dec_03(s_dec_2); s_apply_order_dec_02(s_apply_kl);
+ s_apply_order_dec_02(s_dec_1); s_apply_order_dec_01(s_apply_kl);
+ s_apply_order_dec_01(s_dec_0); s_apply_order_dec_00(s_apply_key);
+ s_serpent_accel_256_bit_avx2_store_four(&c, &d, &b, &e, out);
+ in += blocks_at_a_time * serpent_block_len;
+ out += blocks_at_a_time * serpent_block_len;
+ }
+ return CRYPT_OK;
+
+ #undef blocks_at_a_time
+ #undef s_do_broadcast
+ #undef s_do_asgn
+ #undef s_do_or
+ #undef s_do_xor
+ #undef s_do_and
+ #undef s_do_not
+ #undef s_do_rol
+ #undef s_do_ror
+ #undef s_do_shl
+}
+
+#endif
+
+#if defined LTC_SERPENT_ACCEL_512_BIT_X86_AVX512F
+
+#if defined(__GNUC__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-function"
+#endif
+#if defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wunused-function"
+#endif
+#include /* AVX512F __m512i _mm512_and_si512 _mm512_cmpeq_epi32 _mm512_loadu_si512 _mm512_or_si512 _mm512_set1_epi32 _mm512_set1_epi8 _mm512_slli_epi32 _mm512_srli_epi32 _mm512_storeu_si512 _mm512_unpackhi_epi32 _mm512_unpackhi_epi64 _mm512_unpacklo_epi32 _mm512_unpacklo_epi64 _mm512_xor_si512 */
+#if defined(__clang__)
+#pragma clang diagnostic pop
+#endif
+#if defined(__GNUC__)
+#pragma GCC diagnostic pop
+#endif
+
+static LTC_INLINE int s_is_supported_512_bit_avx512f(void)
+{
+ static int is_initialized = 0;
+ static int is_supported = 0;
+
+ if (is_initialized == 0) {
+ int regs[4];
+ int osxsave, avx512f, zmm;
+ ulong64 xcr0;
+
+ is_initialized = 1;
+ s_x86_cpuid(regs, 0);
+ if (regs[0] >= 7) {
+ s_x86_cpuid(regs, 1);
+ osxsave = ((((unsigned int)(regs[2])) >> 27) & 1u) != 0; /* OSXSAVE, leaf 1, ecx, bit 27 */
+ s_x86_cpuid(regs, 7);
+ avx512f = ((((unsigned int)(regs[1])) >> 16) & 1u) != 0; /* AVX512F, leaf 7, ebx, bit 16 */
+ if (osxsave) {
+ xcr0 = s_x86_xgetbv0();
+ zmm = (xcr0 & 0xe0) == 0xe0; /* 512bit AVX zmm */
+ is_supported = osxsave && avx512f && zmm;
+ }
+ }
+ }
+ return is_supported;
+}
+
+static LTC_INLINE void LTC_AVX512F_TARGET s_serpent_accel_ecb_512_bit_avx512f_load_one(__m512i *x, const unsigned char *bytes)
+{
+ *x = _mm512_loadu_si512(((const __m512i*)(bytes)));
+}
+
+static LTC_INLINE void LTC_AVX512F_TARGET s_serpent_accel_ecb_512_bit_avx512f_store_one(const __m512i *x, unsigned char *bytes)
+{
+ _mm512_storeu_si512(((__m512i*)(bytes)), *x);
+}
+
+static LTC_INLINE void LTC_AVX512F_TARGET s_serpent_accel_512_bit_avx512f_load_four(__m512i *pa, __m512i *pb, __m512i *pc, __m512i *pd, const unsigned char *bytes)
+{
+ __m512i ia, ib, ic, id;
+ __m512i ta, tb, tc, td;
+ __m512i ra, rb, rc, rd;
+
+ s_serpent_accel_ecb_512_bit_avx512f_load_one(&ia, &bytes[0 * sizeof(__m512i)]);
+ s_serpent_accel_ecb_512_bit_avx512f_load_one(&ib, &bytes[1 * sizeof(__m512i)]);
+ s_serpent_accel_ecb_512_bit_avx512f_load_one(&ic, &bytes[2 * sizeof(__m512i)]);
+ s_serpent_accel_ecb_512_bit_avx512f_load_one(&id, &bytes[3 * sizeof(__m512i)]);
+ ta = _mm512_unpacklo_epi32(ia, ib);
+ tb = _mm512_unpacklo_epi32(ic, id);
+ tc = _mm512_unpackhi_epi32(ia, ib);
+ td = _mm512_unpackhi_epi32(ic, id);
+ ra = _mm512_unpacklo_epi64(ta, tb);
+ rb = _mm512_unpackhi_epi64(ta, tb);
+ rc = _mm512_unpacklo_epi64(tc, td);
+ rd = _mm512_unpackhi_epi64(tc, td);
+ *pa = ra;
+ *pb = rb;
+ *pc = rc;
+ *pd = rd;
+}
+
+static LTC_INLINE void LTC_AVX512F_TARGET s_serpent_accel_512_bit_avx512f_store_four(const __m512i *pa, const __m512i *pb, const __m512i *pc, const __m512i *pd, unsigned char *bytes)
+{
+ __m512i ia, ib, ic, id;
+ __m512i ta, tb, tc, td;
+ __m512i ra, rb, rc, rd;
+
+ ia = *pa;
+ ib = *pb;
+ ic = *pc;
+ id = *pd;
+ ta = _mm512_unpacklo_epi32(ia, ib);
+ tb = _mm512_unpacklo_epi32(ic, id);
+ tc = _mm512_unpackhi_epi32(ia, ib);
+ td = _mm512_unpackhi_epi32(ic, id);
+ ra = _mm512_unpacklo_epi64(ta, tb);
+ rb = _mm512_unpackhi_epi64(ta, tb);
+ rc = _mm512_unpacklo_epi64(tc, td);
+ rd = _mm512_unpackhi_epi64(tc, td);
+ s_serpent_accel_ecb_512_bit_avx512f_store_one(&ra, &bytes[0 * sizeof(__m512i)]);
+ s_serpent_accel_ecb_512_bit_avx512f_store_one(&rb, &bytes[1 * sizeof(__m512i)]);
+ s_serpent_accel_ecb_512_bit_avx512f_store_one(&rc, &bytes[2 * sizeof(__m512i)]);
+ s_serpent_accel_ecb_512_bit_avx512f_store_one(&rd, &bytes[3 * sizeof(__m512i)]);
+}
+
+static LTC_INLINE int LTC_AVX512F_TARGET s_serpent_accel_ecb_encrypt_512_bit_avx512f(const unsigned char *pt, unsigned char *ct, unsigned long blocks, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (512 / 32)
+ #define s_do_broadcast(x) _mm512_set1_epi32(*((const int *)(&(x))))
+ #define s_do_asgn(a, b) a = b
+ #define s_do_or(a, b) a = _mm512_or_si512(a, b)
+ #define s_do_xor(a, b) a = _mm512_xor_si512(a, b)
+ #define s_do_and(a, b) a = _mm512_and_si512(a, b)
+ #define s_do_not(a, b) a = _mm512_xor_si512(b, _mm512_set1_epi8('\xff'))
+ #define s_do_rol(x, i) x = _mm512_or_si512(_mm512_slli_epi32(x, i), _mm512_srli_epi32(x, 32 - i))
+ #define s_do_ror(x, i) x = _mm512_or_si512(_mm512_srli_epi32(x, i), _mm512_slli_epi32(x, 32 - i))
+ #define s_do_shl(a, b, c) a = _mm512_slli_epi32(b, c)
+
+ const unsigned char *in;
+ unsigned char *out;
+ const ulong32* k;
+ unsigned long iblock;
+ __m512i a, b, c, d, e;
+
+ LTC_ARGCHK(pt);
+ LTC_ARGCHK(ct);
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+
+ in = pt;
+ out = ct;
+ k = &skey->serpent.k[0];
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ s_serpent_accel_512_bit_avx512f_load_four(&a, &b, &c, &d, in);
+ s_apply_order_enc_00(s_apply_key);
+ s_apply_order_enc_00(s_enc_0); s_apply_order_enc_01(s_apply_lk);
+ s_apply_order_enc_01(s_enc_1); s_apply_order_enc_02(s_apply_lk);
+ s_apply_order_enc_02(s_enc_2); s_apply_order_enc_03(s_apply_lk);
+ s_apply_order_enc_03(s_enc_3); s_apply_order_enc_04(s_apply_lk);
+ s_apply_order_enc_04(s_enc_4); s_apply_order_enc_05(s_apply_lk);
+ s_apply_order_enc_05(s_enc_5); s_apply_order_enc_06(s_apply_lk);
+ s_apply_order_enc_06(s_enc_6); s_apply_order_enc_07(s_apply_lk);
+ s_apply_order_enc_07(s_enc_7); s_apply_order_enc_08(s_apply_lk);
+ s_apply_order_enc_08(s_enc_0); s_apply_order_enc_09(s_apply_lk);
+ s_apply_order_enc_09(s_enc_1); s_apply_order_enc_10(s_apply_lk);
+ s_apply_order_enc_10(s_enc_2); s_apply_order_enc_11(s_apply_lk);
+ s_apply_order_enc_11(s_enc_3); s_apply_order_enc_12(s_apply_lk);
+ s_apply_order_enc_12(s_enc_4); s_apply_order_enc_13(s_apply_lk);
+ s_apply_order_enc_13(s_enc_5); s_apply_order_enc_14(s_apply_lk);
+ s_apply_order_enc_14(s_enc_6); s_apply_order_enc_15(s_apply_lk);
+ s_apply_order_enc_15(s_enc_7); s_apply_order_enc_16(s_apply_lk);
+ s_apply_order_enc_16(s_enc_0); s_apply_order_enc_17(s_apply_lk);
+ s_apply_order_enc_17(s_enc_1); s_apply_order_enc_18(s_apply_lk);
+ s_apply_order_enc_18(s_enc_2); s_apply_order_enc_19(s_apply_lk);
+ s_apply_order_enc_19(s_enc_3); s_apply_order_enc_20(s_apply_lk);
+ s_apply_order_enc_20(s_enc_4); s_apply_order_enc_21(s_apply_lk);
+ s_apply_order_enc_21(s_enc_5); s_apply_order_enc_22(s_apply_lk);
+ s_apply_order_enc_22(s_enc_6); s_apply_order_enc_23(s_apply_lk);
+ s_apply_order_enc_23(s_enc_7); s_apply_order_enc_24(s_apply_lk);
+ s_apply_order_enc_24(s_enc_0); s_apply_order_enc_25(s_apply_lk);
+ s_apply_order_enc_25(s_enc_1); s_apply_order_enc_26(s_apply_lk);
+ s_apply_order_enc_26(s_enc_2); s_apply_order_enc_27(s_apply_lk);
+ s_apply_order_enc_27(s_enc_3); s_apply_order_enc_28(s_apply_lk);
+ s_apply_order_enc_28(s_enc_4); s_apply_order_enc_29(s_apply_lk);
+ s_apply_order_enc_29(s_enc_5); s_apply_order_enc_30(s_apply_lk);
+ s_apply_order_enc_30(s_enc_6); s_apply_order_enc_31(s_apply_lk);
+ s_apply_order_enc_31(s_enc_7); s_apply_order_enc_32(s_apply_key);
+ s_serpent_accel_512_bit_avx512f_store_four(&a, &b, &c, &d, out);
+ in += blocks_at_a_time * serpent_block_len;
+ out += blocks_at_a_time * serpent_block_len;
+ }
+ return CRYPT_OK;
+
+ #undef blocks_at_a_time
+ #undef s_do_broadcast
+ #undef s_do_asgn
+ #undef s_do_or
+ #undef s_do_xor
+ #undef s_do_and
+ #undef s_do_not
+ #undef s_do_rol
+ #undef s_do_ror
+ #undef s_do_shl
+}
+
+static LTC_INLINE int LTC_AVX512F_TARGET s_serpent_accel_ecb_decrypt_512_bit_avx512f(const unsigned char *ct, unsigned char *pt, unsigned long blocks, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (512 / 32)
+ #define s_do_broadcast(x) _mm512_set1_epi32(*((const int *)(&(x))))
+ #define s_do_asgn(a, b) a = b
+ #define s_do_or(a, b) a = _mm512_or_si512(a, b)
+ #define s_do_xor(a, b) a = _mm512_xor_si512(a, b)
+ #define s_do_and(a, b) a = _mm512_and_si512(a, b)
+ #define s_do_not(a, b) a = _mm512_xor_si512(b, _mm512_set1_epi8('\xff'))
+ #define s_do_rol(x, i) x = _mm512_or_si512(_mm512_slli_epi32(x, i), _mm512_srli_epi32(x, 32 - i))
+ #define s_do_ror(x, i) x = _mm512_or_si512(_mm512_srli_epi32(x, i), _mm512_slli_epi32(x, 32 - i))
+ #define s_do_shl(a, b, c) a = _mm512_slli_epi32(b, c)
+
+ const unsigned char *in;
+ unsigned char *out;
+ const ulong32* k;
+ unsigned long iblock;
+ __m512i a, b, c, d, e;
+
+ LTC_ARGCHK(ct);
+ LTC_ARGCHK(pt);
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+
+ in = ct;
+ out = pt;
+ k = &skey->serpent.k[0];
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ s_serpent_accel_512_bit_avx512f_load_four(&a, &b, &c, &d, in);
+ s_apply_order_dec_32(s_apply_key);
+ s_apply_order_dec_32(s_dec_7); s_apply_order_dec_31(s_apply_kl);
+ s_apply_order_dec_31(s_dec_6); s_apply_order_dec_30(s_apply_kl);
+ s_apply_order_dec_30(s_dec_5); s_apply_order_dec_29(s_apply_kl);
+ s_apply_order_dec_29(s_dec_4); s_apply_order_dec_28(s_apply_kl);
+ s_apply_order_dec_28(s_dec_3); s_apply_order_dec_27(s_apply_kl);
+ s_apply_order_dec_27(s_dec_2); s_apply_order_dec_26(s_apply_kl);
+ s_apply_order_dec_26(s_dec_1); s_apply_order_dec_25(s_apply_kl);
+ s_apply_order_dec_25(s_dec_0); s_apply_order_dec_24(s_apply_kl);
+ s_apply_order_dec_24(s_dec_7); s_apply_order_dec_23(s_apply_kl);
+ s_apply_order_dec_23(s_dec_6); s_apply_order_dec_22(s_apply_kl);
+ s_apply_order_dec_22(s_dec_5); s_apply_order_dec_21(s_apply_kl);
+ s_apply_order_dec_21(s_dec_4); s_apply_order_dec_20(s_apply_kl);
+ s_apply_order_dec_20(s_dec_3); s_apply_order_dec_19(s_apply_kl);
+ s_apply_order_dec_19(s_dec_2); s_apply_order_dec_18(s_apply_kl);
+ s_apply_order_dec_18(s_dec_1); s_apply_order_dec_17(s_apply_kl);
+ s_apply_order_dec_17(s_dec_0); s_apply_order_dec_16(s_apply_kl);
+ s_apply_order_dec_16(s_dec_7); s_apply_order_dec_15(s_apply_kl);
+ s_apply_order_dec_15(s_dec_6); s_apply_order_dec_14(s_apply_kl);
+ s_apply_order_dec_14(s_dec_5); s_apply_order_dec_13(s_apply_kl);
+ s_apply_order_dec_13(s_dec_4); s_apply_order_dec_12(s_apply_kl);
+ s_apply_order_dec_12(s_dec_3); s_apply_order_dec_11(s_apply_kl);
+ s_apply_order_dec_11(s_dec_2); s_apply_order_dec_10(s_apply_kl);
+ s_apply_order_dec_10(s_dec_1); s_apply_order_dec_09(s_apply_kl);
+ s_apply_order_dec_09(s_dec_0); s_apply_order_dec_08(s_apply_kl);
+ s_apply_order_dec_08(s_dec_7); s_apply_order_dec_07(s_apply_kl);
+ s_apply_order_dec_07(s_dec_6); s_apply_order_dec_06(s_apply_kl);
+ s_apply_order_dec_06(s_dec_5); s_apply_order_dec_05(s_apply_kl);
+ s_apply_order_dec_05(s_dec_4); s_apply_order_dec_04(s_apply_kl);
+ s_apply_order_dec_04(s_dec_3); s_apply_order_dec_03(s_apply_kl);
+ s_apply_order_dec_03(s_dec_2); s_apply_order_dec_02(s_apply_kl);
+ s_apply_order_dec_02(s_dec_1); s_apply_order_dec_01(s_apply_kl);
+ s_apply_order_dec_01(s_dec_0); s_apply_order_dec_00(s_apply_key);
+ s_serpent_accel_512_bit_avx512f_store_four(&c, &d, &b, &e, out);
+ in += blocks_at_a_time * serpent_block_len;
+ out += blocks_at_a_time * serpent_block_len;
+ }
+ return CRYPT_OK;
+
+ #undef blocks_at_a_time
+ #undef s_do_broadcast
+ #undef s_do_asgn
+ #undef s_do_or
+ #undef s_do_xor
+ #undef s_do_and
+ #undef s_do_not
+ #undef s_do_rol
+ #undef s_do_ror
+ #undef s_do_shl
+}
+
+#endif
int serpent_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmetric_key *skey)
{
- int err = s_enc_block(pt, ct, skey->serpent.k);
+ int err = s_serpent_accel_ecb_encrypt_32_bit(pt, ct, 1, skey);
#ifdef LTC_CLEAN_STACK
burn_stack(sizeof(ulong32) * 5 + sizeof(int));
#endif
@@ -594,7 +2068,7 @@ int serpent_ecb_encrypt(const unsigned char *pt, unsigned char *ct, const symmet
int serpent_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmetric_key *skey)
{
- int err = s_dec_block(ct, pt, skey->serpent.k);
+ int err = s_serpent_accel_ecb_decrypt_32_bit(ct, pt, 1, skey);
#ifdef LTC_CLEAN_STACK
burn_stack(sizeof(ulong32) * 5 + sizeof(int));
#endif
@@ -617,6 +2091,794 @@ int serpent_keysize(int *keysize)
return CRYPT_OK;
}
+#if defined LTC_SERPENT_ACCEL
+
+static LTC_INLINE void s_serpent_accel_ctr_increment_counter_be(unsigned char *counter)
+{
+ int i;
+ int idx;
+
+ for (i = 0; i != serpent_block_len; ++i) {
+ idx = (serpent_block_len - 1) - i;
+ ++counter[idx];
+ if (counter[idx] != 0x00) {
+ break;
+ }
+ }
+}
+
+static LTC_INLINE void s_serpent_accel_ctr_increment_counter_le(unsigned char *counter)
+{
+ int i;
+ int idx;
+
+ for (i = 0; i != serpent_block_len; ++i) {
+ idx = i;
+ ++counter[idx];
+ if (counter[idx] != 0x00) {
+ break;
+ }
+ }
+}
+
+static LTC_INLINE void s_serpent_accel_ctr_increment_counter_generic(unsigned char *counter, int mode)
+{
+ if (mode == CTR_COUNTER_LITTLE_ENDIAN) {
+ s_serpent_accel_ctr_increment_counter_le(counter);
+ } else {
+ s_serpent_accel_ctr_increment_counter_be(counter);
+ }
+}
+
+static LTC_INLINE int s_serpent_accel_cbc_decrypt_32_bit(const unsigned char *ct, unsigned char *pt, unsigned long blocks, unsigned char *IV, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (32 / 32)
+
+ unsigned long iblock;
+ int err;
+ LTC_ALIGN_MSVC(4) unsigned char pad1[blocks_at_a_time * serpent_block_len] LTC_ALIGN(4);
+ int i;
+ LTC_ALIGN_MSVC(4) unsigned char pad2[blocks_at_a_time * serpent_block_len] LTC_ALIGN(4);
+ int j;
+
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+ LTC_ARGCHK(serpent_block_len % sizeof(LTC_FAST_TYPE) == 0);
+
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ if ((err = s_serpent_accel_ecb_decrypt_32_bit(ct, pad1, blocks_at_a_time, skey)) != CRYPT_OK) {
+ return err;
+ }
+ for (i = 0; i != serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_XOR3(&pad2[i * sizeof(LTC_FAST_TYPE)], &IV[i * sizeof(LTC_FAST_TYPE)], &pad1[i * sizeof(LTC_FAST_TYPE)]);
+ }
+ for (j = 1; j != blocks_at_a_time; ++j) {
+ for (i = 0; i != serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_XOR3(&pad2[j * serpent_block_len + i * sizeof(LTC_FAST_TYPE)], &ct[(j - 1) * serpent_block_len + i * sizeof(LTC_FAST_TYPE)], &pad1[j * serpent_block_len + i * sizeof(LTC_FAST_TYPE)]);
+ }
+ }
+ for (i = 0; i != serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_STORE(&IV[i * sizeof(LTC_FAST_TYPE)], LTC_FAST_LOAD(&ct[(j - 1) * serpent_block_len + i * sizeof(LTC_FAST_TYPE)]));
+ }
+ for (i = 0; i != blocks_at_a_time * serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_STORE(&pt[i * sizeof(LTC_FAST_TYPE)], LTC_FAST_LOAD(&pad2[i * sizeof(LTC_FAST_TYPE)]));
+ }
+ pt += blocks_at_a_time * serpent_block_len;
+ ct += blocks_at_a_time * serpent_block_len;
+ }
+ return CRYPT_OK;
+
+ #undef blocks_at_a_time
+}
+
+#if defined LTC_SERPENT_ACCEL_64_BIT
+
+static LTC_INLINE int s_serpent_accel_cbc_decrypt_64_bit(const unsigned char *ct, unsigned char *pt, unsigned long blocks, unsigned char *IV, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (64 / 32)
+
+ unsigned long iblock;
+ int err;
+ LTC_ALIGN_MSVC(8) unsigned char pad1[blocks_at_a_time * serpent_block_len] LTC_ALIGN(8);
+ int i;
+ LTC_ALIGN_MSVC(8) unsigned char pad2[blocks_at_a_time * serpent_block_len] LTC_ALIGN(8);
+ int j;
+
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+ LTC_ARGCHK(serpent_block_len % sizeof(LTC_FAST_TYPE) == 0);
+
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ if ((err = s_serpent_accel_ecb_decrypt_64_bit(ct, pad1, blocks_at_a_time, skey)) != CRYPT_OK) {
+ return err;
+ }
+ for (i = 0; i != serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_XOR3(&pad2[i * sizeof(LTC_FAST_TYPE)], &IV[i * sizeof(LTC_FAST_TYPE)], &pad1[i * sizeof(LTC_FAST_TYPE)]);
+ }
+ for (j = 1; j != blocks_at_a_time; ++j) {
+ for (i = 0; i != serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_XOR3(&pad2[j * serpent_block_len + i * sizeof(LTC_FAST_TYPE)], &ct[(j - 1) * serpent_block_len + i * sizeof(LTC_FAST_TYPE)], &pad1[j * serpent_block_len + i * sizeof(LTC_FAST_TYPE)]);
+ }
+ }
+ for (i = 0; i != serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_STORE(&IV[i * sizeof(LTC_FAST_TYPE)], LTC_FAST_LOAD(&ct[(j - 1) * serpent_block_len + i * sizeof(LTC_FAST_TYPE)]));
+ }
+ for (i = 0; i != blocks_at_a_time * serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_STORE(&pt[i * sizeof(LTC_FAST_TYPE)], LTC_FAST_LOAD(&pad2[i * sizeof(LTC_FAST_TYPE)]));
+ }
+ pt += blocks_at_a_time * serpent_block_len;
+ ct += blocks_at_a_time * serpent_block_len;
+ }
+ return CRYPT_OK;
+
+ #undef blocks_at_a_time
+}
+
+#endif
+
+#if defined LTC_SERPENT_ACCEL_128_BIT_X86_SSE2
+
+static LTC_INLINE int s_serpent_accel_cbc_decrypt_128_bit_sse2(const unsigned char *ct, unsigned char *pt, unsigned long blocks, unsigned char *IV, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (128 / 32)
+
+ unsigned long iblock;
+ int err;
+ LTC_ALIGN_MSVC(16) unsigned char pad1[blocks_at_a_time * serpent_block_len] LTC_ALIGN(16);
+ int i;
+ LTC_ALIGN_MSVC(16) unsigned char pad2[blocks_at_a_time * serpent_block_len] LTC_ALIGN(16);
+ int j;
+
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+ LTC_ARGCHK(serpent_block_len % sizeof(LTC_FAST_TYPE) == 0);
+
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ if ((err = s_serpent_accel_ecb_decrypt_128_bit_sse2(ct, pad1, blocks_at_a_time, skey)) != CRYPT_OK) {
+ return err;
+ }
+ for (i = 0; i != serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_XOR3(&pad2[i * sizeof(LTC_FAST_TYPE)], &IV[i * sizeof(LTC_FAST_TYPE)], &pad1[i * sizeof(LTC_FAST_TYPE)]);
+ }
+ for (j = 1; j != blocks_at_a_time; ++j) {
+ for (i = 0; i != serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_XOR3(&pad2[j * serpent_block_len + i * sizeof(LTC_FAST_TYPE)], &ct[(j - 1) * serpent_block_len + i * sizeof(LTC_FAST_TYPE)], &pad1[j * serpent_block_len + i * sizeof(LTC_FAST_TYPE)]);
+ }
+ }
+ for (i = 0; i != serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_STORE(&IV[i * sizeof(LTC_FAST_TYPE)], LTC_FAST_LOAD(&ct[(j - 1) * serpent_block_len + i * sizeof(LTC_FAST_TYPE)]));
+ }
+ for (i = 0; i != blocks_at_a_time * serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_STORE(&pt[i * sizeof(LTC_FAST_TYPE)], LTC_FAST_LOAD(&pad2[i * sizeof(LTC_FAST_TYPE)]));
+ }
+ pt += blocks_at_a_time * serpent_block_len;
+ ct += blocks_at_a_time * serpent_block_len;
+ }
+ return CRYPT_OK;
+
+ #undef blocks_at_a_time
+}
+
+#endif
+
+#if defined LTC_SERPENT_ACCEL_256_BIT_X86_AVX2
+
+static LTC_INLINE int s_serpent_accel_cbc_decrypt_256_bit_avx2(const unsigned char *ct, unsigned char *pt, unsigned long blocks, unsigned char *IV, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (256 / 32)
+
+ unsigned long iblock;
+ int err;
+ LTC_ALIGN_MSVC(32) unsigned char pad1[blocks_at_a_time * serpent_block_len] LTC_ALIGN(32);
+ int i;
+ LTC_ALIGN_MSVC(32) unsigned char pad2[blocks_at_a_time * serpent_block_len] LTC_ALIGN(32);
+ int j;
+
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+ LTC_ARGCHK(serpent_block_len % sizeof(LTC_FAST_TYPE) == 0);
+
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ if ((err = s_serpent_accel_ecb_decrypt_256_bit_avx2(ct, pad1, blocks_at_a_time, skey)) != CRYPT_OK) {
+ return err;
+ }
+ for (i = 0; i != serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_XOR3(&pad2[i * sizeof(LTC_FAST_TYPE)], &IV[i * sizeof(LTC_FAST_TYPE)], &pad1[i * sizeof(LTC_FAST_TYPE)]);
+ }
+ for (j = 1; j != blocks_at_a_time; ++j) {
+ for (i = 0; i != serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_XOR3(&pad2[j * serpent_block_len + i * sizeof(LTC_FAST_TYPE)], &ct[(j - 1) * serpent_block_len + i * sizeof(LTC_FAST_TYPE)], &pad1[j * serpent_block_len + i * sizeof(LTC_FAST_TYPE)]);
+ }
+ }
+ for (i = 0; i != serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_STORE(&IV[i * sizeof(LTC_FAST_TYPE)], LTC_FAST_LOAD(&ct[(j - 1) * serpent_block_len + i * sizeof(LTC_FAST_TYPE)]));
+ }
+ for (i = 0; i != blocks_at_a_time * serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_STORE(&pt[i * sizeof(LTC_FAST_TYPE)], LTC_FAST_LOAD(&pad2[i * sizeof(LTC_FAST_TYPE)]));
+ }
+ pt += blocks_at_a_time * serpent_block_len;
+ ct += blocks_at_a_time * serpent_block_len;
+ }
+ return CRYPT_OK;
+
+ #undef blocks_at_a_time
+}
+
+#endif
+
+#if defined LTC_SERPENT_ACCEL_512_BIT_X86_AVX512F
+
+static LTC_INLINE int s_serpent_accel_cbc_decrypt_512_bit_avx512f(const unsigned char *ct, unsigned char *pt, unsigned long blocks, unsigned char *IV, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (512 / 32)
+
+ unsigned long iblock;
+ int err;
+ LTC_ALIGN_MSVC(64) unsigned char pad1[blocks_at_a_time * serpent_block_len] LTC_ALIGN(64);
+ int i;
+ LTC_ALIGN_MSVC(64) unsigned char pad2[blocks_at_a_time * serpent_block_len] LTC_ALIGN(64);
+ int j;
+
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+ LTC_ARGCHK(serpent_block_len % sizeof(LTC_FAST_TYPE) == 0);
+
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ if ((err = s_serpent_accel_ecb_decrypt_512_bit_avx512f(ct, pad1, blocks_at_a_time, skey)) != CRYPT_OK) {
+ return err;
+ }
+ for (i = 0; i != serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_XOR3(&pad2[i * sizeof(LTC_FAST_TYPE)], &IV[i * sizeof(LTC_FAST_TYPE)], &pad1[i * sizeof(LTC_FAST_TYPE)]);
+ }
+ for (j = 1; j != blocks_at_a_time; ++j) {
+ for (i = 0; i != serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_XOR3(&pad2[j * serpent_block_len + i * sizeof(LTC_FAST_TYPE)], &ct[(j - 1) * serpent_block_len + i * sizeof(LTC_FAST_TYPE)], &pad1[j * serpent_block_len + i * sizeof(LTC_FAST_TYPE)]);
+ }
+ }
+ for (i = 0; i != serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_STORE(&IV[i * sizeof(LTC_FAST_TYPE)], LTC_FAST_LOAD(&ct[(j - 1) * serpent_block_len + i * sizeof(LTC_FAST_TYPE)]));
+ }
+ for (i = 0; i != blocks_at_a_time * serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_STORE(&pt[i * sizeof(LTC_FAST_TYPE)], LTC_FAST_LOAD(&pad2[i * sizeof(LTC_FAST_TYPE)]));
+ }
+ pt += blocks_at_a_time * serpent_block_len;
+ ct += blocks_at_a_time * serpent_block_len;
+ }
+ return CRYPT_OK;
+
+ #undef blocks_at_a_time
+}
+
+#endif
+
+static LTC_INLINE int s_serpent_accel_ctr_encrypt_32_bit(const unsigned char *pt, unsigned char *ct, unsigned long blocks, unsigned char *IV, int mode, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (32 / 32)
+
+ unsigned long iblock;
+ int i;
+ LTC_ALIGN_MSVC(4) unsigned char pad[blocks_at_a_time * serpent_block_len] LTC_ALIGN(4);
+ int err;
+
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+ LTC_ARGCHK(serpent_block_len % sizeof(LTC_FAST_TYPE) == 0);
+
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ for (i = 0; i != blocks_at_a_time; ++i) {
+ s_serpent_accel_ctr_increment_counter_generic(IV, mode);
+ XMEMCPY(&pad[i * serpent_block_len], IV, serpent_block_len);
+ }
+ if ((err = s_serpent_accel_ecb_encrypt_32_bit(&pad[0], &pad[0], blocks_at_a_time, skey)) != CRYPT_OK) {
+ return err;
+ }
+ for (i = 0; i != blocks_at_a_time * serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_XOR3(&ct[i * sizeof(LTC_FAST_TYPE)], &pt[i * sizeof(LTC_FAST_TYPE)], &pad[i * sizeof(LTC_FAST_TYPE)]);
+ }
+ pt += blocks_at_a_time * serpent_block_len;
+ ct += blocks_at_a_time * serpent_block_len;
+ }
+ return CRYPT_OK;
+
+ #undef blocks_at_a_time
+}
+
+#if defined LTC_SERPENT_ACCEL_64_BIT
+
+static LTC_INLINE int s_serpent_accel_ctr_encrypt_64_bit(const unsigned char *pt, unsigned char *ct, unsigned long blocks, unsigned char *IV, int mode, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (64 / 32)
+
+ unsigned long iblock;
+ int i;
+ LTC_ALIGN_MSVC(8) unsigned char pad[blocks_at_a_time * serpent_block_len] LTC_ALIGN(8);
+ int err;
+
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+ LTC_ARGCHK(serpent_block_len % sizeof(LTC_FAST_TYPE) == 0);
+
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ for (i = 0; i != blocks_at_a_time; ++i) {
+ s_serpent_accel_ctr_increment_counter_generic(IV, mode);
+ XMEMCPY(&pad[i * serpent_block_len], IV, serpent_block_len);
+ }
+ if ((err = s_serpent_accel_ecb_encrypt_64_bit(&pad[0], &pad[0], blocks_at_a_time, skey)) != CRYPT_OK) {
+ return err;
+ }
+ for (i = 0; i != blocks_at_a_time * serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_XOR3(&ct[i * sizeof(LTC_FAST_TYPE)], &pt[i * sizeof(LTC_FAST_TYPE)], &pad[i * sizeof(LTC_FAST_TYPE)]);
+ }
+ pt += blocks_at_a_time * serpent_block_len;
+ ct += blocks_at_a_time * serpent_block_len;
+ }
+ return CRYPT_OK;
+
+ #undef blocks_at_a_time
+}
+
+#endif
+
+#if defined LTC_SERPENT_ACCEL_128_BIT_X86_SSE2
+
+static LTC_INLINE int s_serpent_accel_ctr_encrypt_128_bit_sse2(const unsigned char *pt, unsigned char *ct, unsigned long blocks, unsigned char *IV, int mode, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (128 / 32)
+
+ unsigned long iblock;
+ int i;
+ LTC_ALIGN_MSVC(16) unsigned char pad[blocks_at_a_time * serpent_block_len] LTC_ALIGN(16);
+ int err;
+
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+ LTC_ARGCHK(serpent_block_len % sizeof(LTC_FAST_TYPE) == 0);
+
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ for (i = 0; i != blocks_at_a_time; ++i) {
+ s_serpent_accel_ctr_increment_counter_generic(IV, mode);
+ XMEMCPY(&pad[i * serpent_block_len], IV, serpent_block_len);
+ }
+ if ((err = s_serpent_accel_ecb_encrypt_128_bit_sse2(&pad[0], &pad[0], blocks_at_a_time, skey)) != CRYPT_OK) {
+ return err;
+ }
+ for (i = 0; i != blocks_at_a_time * serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_XOR3(&ct[i * sizeof(LTC_FAST_TYPE)], &pt[i * sizeof(LTC_FAST_TYPE)], &pad[i * sizeof(LTC_FAST_TYPE)]);
+ }
+ pt += blocks_at_a_time * serpent_block_len;
+ ct += blocks_at_a_time * serpent_block_len;
+ }
+ return CRYPT_OK;
+
+ #undef blocks_at_a_time
+}
+
+#endif
+
+#if defined LTC_SERPENT_ACCEL_256_BIT_X86_AVX2
+
+static LTC_INLINE int s_serpent_accel_ctr_encrypt_256_bit_avx2(const unsigned char *pt, unsigned char *ct, unsigned long blocks, unsigned char *IV, int mode, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (256 / 32)
+
+ unsigned long iblock;
+ int i;
+ LTC_ALIGN_MSVC(32) unsigned char pad[blocks_at_a_time * serpent_block_len] LTC_ALIGN(32);
+ int err;
+
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+ LTC_ARGCHK(serpent_block_len % sizeof(LTC_FAST_TYPE) == 0);
+
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ for (i = 0; i != blocks_at_a_time; ++i) {
+ s_serpent_accel_ctr_increment_counter_generic(IV, mode);
+ XMEMCPY(&pad[i * serpent_block_len], IV, serpent_block_len);
+ }
+ if ((err = s_serpent_accel_ecb_encrypt_256_bit_avx2(&pad[0], &pad[0], blocks_at_a_time, skey)) != CRYPT_OK) {
+ return err;
+ }
+ for (i = 0; i != blocks_at_a_time * serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_XOR3(&ct[i * sizeof(LTC_FAST_TYPE)], &pt[i * sizeof(LTC_FAST_TYPE)], &pad[i * sizeof(LTC_FAST_TYPE)]);
+ }
+ pt += blocks_at_a_time * serpent_block_len;
+ ct += blocks_at_a_time * serpent_block_len;
+ }
+ return CRYPT_OK;
+
+ #undef blocks_at_a_time
+}
+
+#endif
+
+#if defined LTC_SERPENT_ACCEL_512_BIT_X86_AVX512F
+
+static LTC_INLINE int s_serpent_accel_ctr_encrypt_512_bit_avx512f(const unsigned char *pt, unsigned char *ct, unsigned long blocks, unsigned char *IV, int mode, const symmetric_key *skey)
+{
+ #define blocks_at_a_time (512 / 32)
+
+ unsigned long iblock;
+ int i;
+ LTC_ALIGN_MSVC(64) unsigned char pad[blocks_at_a_time * serpent_block_len] LTC_ALIGN(64);
+ int err;
+
+ LTC_ARGCHK(blocks % blocks_at_a_time == 0);
+ LTC_ARGCHK(serpent_block_len % sizeof(LTC_FAST_TYPE) == 0);
+
+ for (iblock = 0; iblock != blocks; iblock += blocks_at_a_time) {
+ for (i = 0; i != blocks_at_a_time; ++i) {
+ s_serpent_accel_ctr_increment_counter_generic(IV, mode);
+ XMEMCPY(&pad[i * serpent_block_len], IV, serpent_block_len);
+ }
+ if ((err = s_serpent_accel_ecb_encrypt_512_bit_avx512f(&pad[0], &pad[0], blocks_at_a_time, skey)) != CRYPT_OK) {
+ return err;
+ }
+ for (i = 0; i != blocks_at_a_time * serpent_block_len / sizeof(LTC_FAST_TYPE); ++i) {
+ LTC_FAST_XOR3(&ct[i * sizeof(LTC_FAST_TYPE)], &pt[i * sizeof(LTC_FAST_TYPE)], &pad[i * sizeof(LTC_FAST_TYPE)]);
+ }
+ pt += blocks_at_a_time * serpent_block_len;
+ ct += blocks_at_a_time * serpent_block_len;
+ }
+ return CRYPT_OK;
+
+ #undef blocks_at_a_time
+}
+
+#endif
+
+int serpent_accel_ecb_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long blocks, const symmetric_key *skey)
+{
+ const unsigned char *in;
+ unsigned char *out;
+ unsigned long rem;
+ unsigned long n;
+ int err;
+
+ in = pt;
+ out = ct;
+ rem = blocks;
+ while (rem != 0) {
+ #if defined LTC_SERPENT_ACCEL_512_BIT_X86_AVX512F
+ if (rem >= (512 / 32) && s_is_supported_512_bit_avx512f()) {
+ n = (rem / (512 / 32)) * (512 / 32);
+ err = s_serpent_accel_ecb_encrypt_512_bit_avx512f(in, out, n, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ } else
+ #endif
+ #if defined LTC_SERPENT_ACCEL_256_BIT_X86_AVX2
+ if (rem >= (256 / 32) && s_is_supported_256_bit_avx2()) {
+ #if defined LTC_SERPENT_ACCEL_512_BIT_X86_AVX512F
+ n = 256 / 32;
+ #else
+ n = (rem / (256 / 32)) * (256 / 32);
+ #endif
+ err = s_serpent_accel_ecb_encrypt_256_bit_avx2(in, out, n, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ } else
+ #endif
+ #if defined LTC_SERPENT_ACCEL_128_BIT_X86_SSE2
+ if (rem >= (128 / 32) && s_x86_sse2_is_supported()) {
+ #if defined LTC_SERPENT_ACCEL_256_BIT_X86_AVX2
+ n = 128 / 32;
+ #else
+ n = (rem / (128 / 32)) * (128 / 32);
+ #endif
+ err = s_serpent_accel_ecb_encrypt_128_bit_sse2(in, out, n, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ } else
+ #endif
+ #if defined LTC_SERPENT_ACCEL_64_BIT
+ if (rem >= (64 / 32)) {
+ #if defined LTC_SERPENT_ACCEL_128_BIT_X86_SSE2
+ n = 64 / 32;
+ #else
+ n = (rem / (64 / 32)) * (64 / 32);
+ #endif
+ err = s_serpent_accel_ecb_encrypt_64_bit(in, out, n, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ } else
+ #endif
+ {
+ #if defined LTC_SERPENT_ACCEL_64_BIT
+ n = 32 / 32;
+ #else
+ n = rem;
+ #endif
+ err = s_serpent_accel_ecb_encrypt_32_bit(in, out, n, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ }
+ }
+ return CRYPT_OK;
+}
+
+int serpent_accel_ecb_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long blocks, const symmetric_key *skey)
+{
+ const unsigned char *in;
+ unsigned char *out;
+ unsigned long rem;
+ unsigned long n;
+ int err;
+
+ in = ct;
+ out = pt;
+ rem = blocks;
+ while (rem != 0) {
+ #if defined LTC_SERPENT_ACCEL_512_BIT_X86_AVX512F
+ if (rem >= (512 / 32) && s_is_supported_512_bit_avx512f()) {
+ n = (rem / (512 / 32)) * (512 / 32);
+ err = s_serpent_accel_ecb_decrypt_512_bit_avx512f(in, out, n, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ } else
+ #endif
+ #if defined LTC_SERPENT_ACCEL_256_BIT_X86_AVX2
+ if (rem >= (256 / 32) && s_is_supported_256_bit_avx2()) {
+ #if defined LTC_SERPENT_ACCEL_512_BIT_X86_AVX512F
+ n = 256 / 32;
+ #else
+ n = (rem / (256 / 32)) * (256 / 32);
+ #endif
+ err = s_serpent_accel_ecb_decrypt_256_bit_avx2(in, out, n, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ } else
+ #endif
+ #if defined LTC_SERPENT_ACCEL_128_BIT_X86_SSE2
+ if (rem >= (128 / 32) && s_x86_sse2_is_supported()) {
+ #if defined LTC_SERPENT_ACCEL_256_BIT_X86_AVX2
+ n = 128 / 32;
+ #else
+ n = (rem / (128 / 32)) * (128 / 32);
+ #endif
+ err = s_serpent_accel_ecb_decrypt_128_bit_sse2(in, out, n, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ } else
+ #endif
+ #if defined LTC_SERPENT_ACCEL_64_BIT
+ if (rem >= (64 / 32)) {
+ #if defined LTC_SERPENT_ACCEL_128_BIT_X86_SSE2
+ n = 64 / 32;
+ #else
+ n = (rem / (64 / 32)) * (64 / 32);
+ #endif
+ err = s_serpent_accel_ecb_decrypt_64_bit(in, out, n, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ } else
+ #endif
+ {
+ #if defined LTC_SERPENT_ACCEL_64_BIT
+ n = 32 / 32;
+ #else
+ n = rem;
+ #endif
+ err = s_serpent_accel_ecb_decrypt_32_bit(in, out, n, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ }
+ }
+ return CRYPT_OK;
+}
+
+int serpent_accel_cbc_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long blocks, unsigned char *IV, const symmetric_key *skey)
+{
+ const unsigned char *in;
+ unsigned char *out;
+ unsigned long rem;
+ unsigned long n;
+ int err;
+
+ in = ct;
+ out = pt;
+ rem = blocks;
+ while (rem != 0) {
+ #if defined LTC_SERPENT_ACCEL_512_BIT_X86_AVX512F
+ if (rem >= (512 / 32) && s_is_supported_512_bit_avx512f()) {
+ n = (rem / (512 / 32)) * (512 / 32);
+ err = s_serpent_accel_cbc_decrypt_512_bit_avx512f(in, out, n, IV, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ } else
+ #endif
+ #if defined LTC_SERPENT_ACCEL_256_BIT_X86_AVX2
+ if (rem >= (256 / 32) && s_is_supported_256_bit_avx2()) {
+ #if defined LTC_SERPENT_ACCEL_512_BIT_X86_AVX512F
+ n = 256 / 32;
+ #else
+ n = (rem / (256 / 32)) * (256 / 32);
+ #endif
+ err = s_serpent_accel_cbc_decrypt_256_bit_avx2(in, out, n, IV, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ } else
+ #endif
+ #if defined LTC_SERPENT_ACCEL_128_BIT_X86_SSE2
+ if (rem >= (128 / 32) && s_x86_sse2_is_supported()) {
+ #if defined LTC_SERPENT_ACCEL_256_BIT_X86_AVX2
+ n = 128 / 32;
+ #else
+ n = (rem / (128 / 32)) * (128 / 32);
+ #endif
+ err = s_serpent_accel_cbc_decrypt_128_bit_sse2(in, out, n, IV, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ } else
+ #endif
+ #if defined LTC_SERPENT_ACCEL_64_BIT
+ if (rem >= (64 / 32)) {
+ #if defined LTC_SERPENT_ACCEL_128_BIT_X86_SSE2
+ n = 64 / 32;
+ #else
+ n = (rem / (64 / 32)) * (64 / 32);
+ #endif
+ err = s_serpent_accel_cbc_decrypt_64_bit(in, out, n, IV, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ } else
+ #endif
+ {
+ #if defined LTC_SERPENT_ACCEL_64_BIT
+ n = 32 / 32;
+ #else
+ n = rem;
+ #endif
+ err = s_serpent_accel_cbc_decrypt_32_bit(in, out, n, IV, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ }
+ }
+ return CRYPT_OK;
+}
+
+int serpent_accel_ctr_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long blocks, unsigned char *IV, int mode, const symmetric_key *skey)
+{
+ const unsigned char *in;
+ unsigned char *out;
+ unsigned long rem;
+ unsigned long n;
+ int err;
+
+ in = pt;
+ out = ct;
+ rem = blocks;
+ while (rem != 0) {
+ #if defined LTC_SERPENT_ACCEL_512_BIT_X86_AVX512F
+ if (rem >= (512 / 32) && s_is_supported_512_bit_avx512f()) {
+ n = (rem / (512 / 32)) * (512 / 32);
+ err = s_serpent_accel_ctr_encrypt_512_bit_avx512f(in, out, n, IV, mode, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ } else
+ #endif
+ #if defined LTC_SERPENT_ACCEL_256_BIT_X86_AVX2
+ if (rem >= (256 / 32) && s_is_supported_256_bit_avx2()) {
+ #if defined LTC_SERPENT_ACCEL_512_BIT_X86_AVX512F
+ n = 256 / 32;
+ #else
+ n = (rem / (256 / 32)) * (256 / 32);
+ #endif
+ err = s_serpent_accel_ctr_encrypt_256_bit_avx2(in, out, n, IV, mode, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ } else
+ #endif
+ #if defined LTC_SERPENT_ACCEL_128_BIT_X86_SSE2
+ if (rem >= (128 / 32) && s_x86_sse2_is_supported()) {
+ #if defined LTC_SERPENT_ACCEL_256_BIT_X86_AVX2
+ n = 128 / 32;
+ #else
+ n = (rem / (128 / 32)) * (128 / 32);
+ #endif
+ err = s_serpent_accel_ctr_encrypt_128_bit_sse2(in, out, n, IV, mode, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ } else
+ #endif
+ #if defined LTC_SERPENT_ACCEL_64_BIT
+ if (rem >= (64 / 32)) {
+ #if defined LTC_SERPENT_ACCEL_128_BIT_X86_SSE2
+ n = 64 / 32;
+ #else
+ n = (rem / (64 / 32)) * (64 / 32);
+ #endif
+ err = s_serpent_accel_ctr_encrypt_64_bit(in, out, n, IV, mode, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ } else
+ #endif
+ {
+ #if defined LTC_SERPENT_ACCEL_64_BIT
+ n = 32 / 32;
+ #else
+ n = rem;
+ #endif
+ err = s_serpent_accel_ctr_encrypt_32_bit(in, out, n, IV, mode, skey);
+ if (err != CRYPT_OK) {
+ return err;
+ }
+ out += n * serpent_block_len;
+ in += n * serpent_block_len;
+ rem -= n;
+ }
+ }
+ return CRYPT_OK;
+}
+
+#endif
+
int serpent_test(void)
{
#ifndef LTC_TEST
@@ -755,5 +3017,6 @@ int serpent_test(void)
#undef s_lk
#undef s_sk
#undef s_setup_key
+#undef serpent_block_len
#endif
diff --git a/src/headers/tomcrypt_cipher.h b/src/headers/tomcrypt_cipher.h
index 0cf7feec2..31b91d370 100644
--- a/src/headers/tomcrypt_cipher.h
+++ b/src/headers/tomcrypt_cipher.h
@@ -895,6 +895,10 @@ int serpent_ecb_decrypt(const unsigned char *ct, unsigned char *pt, const symmet
int serpent_test(void);
void serpent_done(symmetric_key *skey);
int serpent_keysize(int *keysize);
+int serpent_accel_ecb_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long blocks, const symmetric_key *skey);
+int serpent_accel_ecb_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long blocks, const symmetric_key *skey);
+int serpent_accel_cbc_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long blocks, unsigned char *IV, const symmetric_key *skey);
+int serpent_accel_ctr_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long blocks, unsigned char *IV, int mode, const symmetric_key *skey);
extern const struct ltc_cipher_descriptor serpent_desc;
#endif
@@ -956,6 +960,7 @@ int cbc_decrypt(const unsigned char *ct, unsigned char *pt, unsigned long len, s
int cbc_getiv(unsigned char *IV, unsigned long *len, const symmetric_CBC *cbc);
int cbc_setiv(const unsigned char *IV, unsigned long len, symmetric_CBC *cbc);
int cbc_done(symmetric_CBC *cbc);
+int cbc_test(void);
#endif
#ifdef LTC_CTR_MODE
diff --git a/src/headers/tomcrypt_private.h b/src/headers/tomcrypt_private.h
index 3a30526c8..1f2ca814b 100644
--- a/src/headers/tomcrypt_private.h
+++ b/src/headers/tomcrypt_private.h
@@ -269,6 +269,9 @@ int func_name (hash_state * md, const unsigned char *in, unsigned long inlen)
}
+#define LTC_XSAVE_TARGET LTC_ATTRIBUTE((__target__("xsave")))
+#define LTC_AVX2_TARGET LTC_ATTRIBUTE((__target__("avx2")))
+#define LTC_AVX512F_TARGET LTC_ATTRIBUTE((__target__("avx512f")))
#define LTC_SHA_TARGET LTC_ATTRIBUTE((__target__("sse2,ssse3,sse4.1,sha")))
#define LTC_SHA512_TARGET LTC_ATTRIBUTE((__target__("avx2,sha512")))
diff --git a/src/modes/cbc/cbc_test.c b/src/modes/cbc/cbc_test.c
new file mode 100644
index 000000000..124142266
--- /dev/null
+++ b/src/modes/cbc/cbc_test.c
@@ -0,0 +1,106 @@
+/* LibTomCrypt, modular cryptographic library -- Tom St Denis */
+/* SPDX-License-Identifier: Unlicense */
+#include "tomcrypt_private.h"
+
+/**
+ @file cbc_test.c
+ CBC implementation
+*/
+
+#ifdef LTC_CBC_MODE
+
+static LTC_INLINE int s_cbc_test_1(void)
+{
+#ifdef LTC_NO_TEST
+ return CRYPT_NOP;
+#else
+ #define LTC_ALIGN_BUF2(buf, align) ((void*)(((((ltc_uintptr)(buf)) + ((align) - 1)) / (align)) * (align)))
+ #define buf_cap (4 * 1024) /* allocate big buffer, for example 4kB */
+ #define buf_alg (1 * 1024) /* align the buffer to ridiculously strict alignment, for example 1kB */
+ #define buf_len (buf_cap - buf_alg) /* in worst case, the buffer will be only 3kB big */
+
+ unsigned char *ct;
+ unsigned char ct1_storage[buf_cap];
+ unsigned char *pt1;
+ unsigned char pt_storage[buf_cap];
+ unsigned char *pt2;
+ unsigned char ct2_storage[buf_cap];
+ int idx;
+ int n;
+ int i;
+ unsigned char iv[MAXBLOCKSIZE];
+ unsigned char key[4 * MAXBLOCKSIZE]; /* todo this is only guesstimate */
+ int block_len;
+ int err;
+ symmetric_CBC cbc;
+
+ ct = (unsigned char*)LTC_ALIGN_BUF2(ct1_storage, buf_alg);
+ pt1 = (unsigned char*)LTC_ALIGN_BUF2(pt_storage, buf_alg);
+ pt2 = (unsigned char*)LTC_ALIGN_BUF2(ct2_storage, buf_alg);
+ idx = 0;
+ for(;;) {
+ if (cipher_is_valid(idx) != CRYPT_OK) {
+ break;
+ }
+ n = buf_len;
+ for (i = 0; i != n; ++i) {
+ ct[i] = rand() & 0xff;
+ }
+ n = buf_len;
+ for (i = 0; i != n; ++i) {
+ pt1[i] = rand() & 0xff;
+ }
+ n = buf_len;
+ for (i = 0; i != n; ++i) {
+ pt2[i] = rand() & 0xff;
+ }
+ n = LTC_ARRAY_SIZE(iv);
+ for (i = 0; i != n; ++i) {
+ iv[i] = rand() & 0xff;
+ }
+ n = LTC_ARRAY_SIZE(key);
+ for (i = 0; i != n; ++i) {
+ key[i] = rand() & 0xff;
+ }
+ block_len = cipher_descriptor[idx].block_length;
+ LTC_ARGCHK(block_len >= 2);
+ LTC_ARGCHK(block_len % 2 == 0);
+ LTC_ARGCHK((int)LTC_ARRAY_SIZE(key) >= cipher_descriptor[idx].max_key_length);
+ LTC_ARGCHK(buf_len % block_len == 0);
+
+ /* decrypt random data by random key, do it piece-wise, so any acceleration might be hindered */
+ if ((err = cbc_start(idx, iv, key, cipher_descriptor[idx].max_key_length, 0, &cbc)) != CRYPT_OK) { return err; }
+ n = buf_len / block_len;
+ for (i = 0; i != n; ++i) {
+ if ((err = cbc_decrypt(ct + i * block_len, pt1 + i * block_len, block_len, &cbc)) != CRYPT_OK) { return err; }
+ }
+ if ((err = cbc_done(&cbc)) != CRYPT_OK) { return err; }
+
+ /* decrypt random data by random key, do it all at once, so any acceleration might be used */
+ if ((err = cbc_start(idx, iv, key, cipher_descriptor[idx].max_key_length, 0, &cbc)) != CRYPT_OK) { return err; }
+ if ((err = cbc_decrypt(ct, pt2, buf_len, &cbc)) != CRYPT_OK) { return err; }
+ if ((err = cbc_done(&cbc)) != CRYPT_OK) { return err; }
+
+ /* compare both non-accelerated and accelerated plain texts */
+ if (ltc_compare_testvector(pt2, buf_len, pt1, buf_len, "CBC", idx)) { return CRYPT_FAIL_TESTVECTOR; }
+ ++idx;
+ }
+ return CRYPT_OK;
+
+ #undef LTC_ALIGN_BUF2
+ #undef buf_cap
+ #undef buf_alg
+ #undef buf_len
+#endif
+}
+
+
+int cbc_test(void)
+{
+ int err;
+
+ err = s_cbc_test_1(); if (err != CRYPT_OK){ return err; }
+ return CRYPT_OK;
+}
+
+#endif
diff --git a/src/modes/ctr/ctr_test.c b/src/modes/ctr/ctr_test.c
index 0f7f03f1a..c0cadeff8 100644
--- a/src/modes/ctr/ctr_test.c
+++ b/src/modes/ctr/ctr_test.c
@@ -9,7 +9,7 @@
#ifdef LTC_CTR_MODE
-int ctr_test(void)
+static LTC_INLINE int s_ctr_test_1(void)
{
#ifdef LTC_NO_TEST
return CRYPT_NOP;
@@ -67,7 +67,185 @@ int ctr_test(void)
#endif
}
+static LTC_INLINE int s_ctr_test_2(void)
+{
+#ifdef LTC_NO_TEST
+ return CRYPT_NOP;
+#else
+ #define LTC_ALIGN_BUF2(buf, align) ((void*)(((((ltc_uintptr)(buf)) + ((align) - 1)) / (align)) * (align)))
+ #define buf_cap (4 * 1024) /* allocate big buffer, for example 4kB */
+ #define buf_alg (1 * 1024) /* align the buffer to ridiculously strict alignment, for example 1kB */
+ #define buf_len (buf_cap - buf_alg) /* in worst case, the buffer will be only 3kB big */
+
+ unsigned char *pt;
+ unsigned char pt_storage[buf_cap];
+ unsigned char *ct1;
+ unsigned char ct1_storage[buf_cap];
+ unsigned char *ct2;
+ unsigned char ct2_storage[buf_cap];
+ int idx;
+ int n;
+ int i;
+ unsigned char iv[MAXBLOCKSIZE];
+ unsigned char key[4 * MAXBLOCKSIZE]; /* todo this is only guesstimate */
+ int block_len;
+ int err;
+ symmetric_CTR ctr;
+
+ pt = (unsigned char*)LTC_ALIGN_BUF2(pt_storage, buf_alg);
+ ct1 = (unsigned char*)LTC_ALIGN_BUF2(ct1_storage, buf_alg);
+ ct2 = (unsigned char*)LTC_ALIGN_BUF2(ct2_storage, buf_alg);
+ idx = 0;
+ for(;;) {
+ if (cipher_is_valid(idx) != CRYPT_OK) {
+ break;
+ }
+ n = buf_len;
+ for (i = 0; i != n; ++i) {
+ pt[i] = rand() & 0xff;
+ }
+ n = buf_len;
+ for (i = 0; i != n; ++i) {
+ ct1[i] = rand() & 0xff;
+ }
+ n = buf_len;
+ for (i = 0; i != n; ++i) {
+ ct2[i] = rand() & 0xff;
+ }
+ n = LTC_ARRAY_SIZE(iv);
+ for (i = 0; i != n; ++i) {
+ iv[i] = rand() & 0xff;
+ }
+ n = LTC_ARRAY_SIZE(key);
+ for (i = 0; i != n; ++i) {
+ key[i] = rand() & 0xff;
+ }
+ block_len = cipher_descriptor[idx].block_length;
+ LTC_ARGCHK(block_len >= 2);
+ LTC_ARGCHK(block_len % 2 == 0);
+ LTC_ARGCHK((int)LTC_ARRAY_SIZE(key) >= cipher_descriptor[idx].max_key_length);
+ LTC_ARGCHK(buf_len % block_len == 0);
+
+ /* encrypt random data by random key, do it piece-wise, so any acceleration might be hindered */
+ if ((err = ctr_start(idx, iv, key, cipher_descriptor[idx].max_key_length, 0, CTR_COUNTER_BIG_ENDIAN | LTC_CTR_RFC3686, &ctr)) != CRYPT_OK) { return err; }
+ n = buf_len / block_len;
+ for (i = 0; i != n; ++i) {
+ if ((err = ctr_encrypt(pt + i * block_len + (block_len / 2) * 0, ct1 + i * block_len + (block_len / 2) * 0, block_len / 2, &ctr)) != CRYPT_OK) { return err; }
+ if ((err = ctr_encrypt(pt + i * block_len + (block_len / 2) * 1, ct1 + i * block_len + (block_len / 2) * 1, block_len / 2, &ctr)) != CRYPT_OK) { return err; }
+ }
+ if ((err = ctr_done(&ctr)) != CRYPT_OK) { return err; }
+
+ /* encrypt random data by random key, do it all at once, so any acceleration might be used */
+ if ((err = ctr_start(idx, iv, key, cipher_descriptor[idx].max_key_length, 0, CTR_COUNTER_BIG_ENDIAN | LTC_CTR_RFC3686, &ctr)) != CRYPT_OK) { return err; }
+ if ((err = ctr_encrypt(pt, ct2, buf_len, &ctr)) != CRYPT_OK) { return err; }
+ if ((err = ctr_done(&ctr)) != CRYPT_OK) { return err; }
+
+ /* compare both non-accelerated and accelerated cipher texts */
+ if (ltc_compare_testvector(ct2, buf_len, ct1, buf_len, "CTR", idx)) { return CRYPT_FAIL_TESTVECTOR; }
+ ++idx;
+ }
+ return CRYPT_OK;
+
+ #undef LTC_ALIGN_BUF2
+ #undef buf_cap
+ #undef buf_alg
+ #undef buf_len
#endif
+}
+static LTC_INLINE int s_ctr_test_3(void)
+{
+#ifdef LTC_NO_TEST
+ return CRYPT_NOP;
+#else
+ #define LTC_ALIGN_BUF2(buf, align) ((void*)(((((ltc_uintptr)(buf)) + ((align) - 1)) / (align)) * (align)))
+ #define buf_cap (4 * 1024) /* allocate big buffer, for example 4kB */
+ #define buf_alg (1 * 1024) /* align the buffer to ridiculously strict alignment, for example 1kB */
+ #define buf_len (buf_cap - buf_alg) /* in worst case, the buffer will be only 3kB big */
+ unsigned char *pt1;
+ unsigned char pt1_storage[buf_cap];
+ unsigned char *ct;
+ unsigned char ct_storage[buf_cap];
+ unsigned char *pt2;
+ unsigned char pt2_storage[buf_cap];
+ int idx;
+ int n;
+ int i;
+ unsigned char iv[MAXBLOCKSIZE];
+ unsigned char key[4 * MAXBLOCKSIZE]; /* todo this is only guesstimate */
+ int block_len;
+ int err;
+ symmetric_CTR ctr;
+
+ pt1 = (unsigned char*)LTC_ALIGN_BUF2(pt1_storage, buf_alg);
+ ct = (unsigned char*)LTC_ALIGN_BUF2(ct_storage, buf_alg);
+ pt2 = (unsigned char*)LTC_ALIGN_BUF2(pt2_storage, buf_alg);
+ idx = 0;
+ for(;;) {
+ if (cipher_is_valid(idx) != CRYPT_OK) {
+ break;
+ }
+ n = buf_len;
+ for (i = 0; i != n; ++i) {
+ pt1[i] = rand() & 0xff;
+ }
+ n = buf_len;
+ for (i = 0; i != n; ++i) {
+ ct[i] = rand() & 0xff;
+ }
+ n = buf_len;
+ for (i = 0; i != n; ++i) {
+ pt2[i] = rand() & 0xff;
+ }
+ n = LTC_ARRAY_SIZE(iv);
+ for (i = 0; i != n; ++i) {
+ iv[i] = rand() & 0xff;
+ }
+ n = LTC_ARRAY_SIZE(key);
+ for (i = 0; i != n; ++i) {
+ key[i] = rand() & 0xff;
+ }
+ block_len = cipher_descriptor[idx].block_length;
+ LTC_ARGCHK(block_len >= 2);
+ LTC_ARGCHK(block_len % 2 == 0);
+ LTC_ARGCHK((int)LTC_ARRAY_SIZE(key) >= cipher_descriptor[idx].max_key_length);
+ LTC_ARGCHK(buf_len % block_len == 0);
+
+ if ((err = ctr_start(idx, iv, key, cipher_descriptor[idx].max_key_length, 0, CTR_COUNTER_BIG_ENDIAN | LTC_CTR_RFC3686, &ctr)) != CRYPT_OK) { return err; }
+ n = buf_len / block_len;
+ for (i = 0; i != n; ++i) {
+ if ((err = ctr_encrypt(pt1 + i * block_len + (block_len / 2) * 0, ct + i * block_len + (block_len / 2) * 0, block_len / 2, &ctr)) != CRYPT_OK) { return err; }
+ if ((err = ctr_encrypt(pt1 + i * block_len + (block_len / 2) * 1, ct + i * block_len + (block_len / 2) * 1, block_len / 2, &ctr)) != CRYPT_OK) { return err; }
+ }
+ if ((err = ctr_done(&ctr)) != CRYPT_OK) { return err; }
+
+ if ((err = ctr_start(idx, iv, key, cipher_descriptor[idx].max_key_length, 0, CTR_COUNTER_BIG_ENDIAN | LTC_CTR_RFC3686, &ctr)) != CRYPT_OK) { return err; }
+ if ((err = ctr_decrypt(ct, pt2, buf_len, &ctr)) != CRYPT_OK) { return err; }
+ if ((err = ctr_done(&ctr)) != CRYPT_OK) { return err; }
+
+ /* test (possibly) accelerated decryption */
+ if (ltc_compare_testvector(pt2, buf_len, pt1, buf_len, "CTR", idx)) { return CRYPT_FAIL_TESTVECTOR; }
+ ++idx;
+ }
+ return CRYPT_OK;
+
+ #undef LTC_ALIGN_BUF2
+ #undef buf_cap
+ #undef buf_alg
+ #undef buf_len
+#endif
+}
+
+
+int ctr_test(void)
+{
+ int err;
+ err = s_ctr_test_1(); if (err != CRYPT_OK){ return err; }
+ err = s_ctr_test_2(); if (err != CRYPT_OK){ return err; }
+ err = s_ctr_test_3(); if (err != CRYPT_OK){ return err; }
+ return CRYPT_OK;
+}
+
+#endif
diff --git a/tests/modes_test.c b/tests/modes_test.c
index 9b16dd673..1df239868 100644
--- a/tests/modes_test.c
+++ b/tests/modes_test.c
@@ -131,10 +131,14 @@ int modes_test(void)
COMPARE_TESTVECTOR(tmp, 64, pt, 64, "OFB", 0);
#endif
-#if defined(LTC_CTR_MODE) && defined(LTC_RIJNDAEL)
+#if defined(LTC_CTR_MODE)
DO(ctr_test());
#endif
+#if defined(LTC_CBC_MODE)
+ DO(cbc_test());
+#endif
+
#ifdef LTC_XTS_MODE
DO(xts_test());
#endif