aesni: Avoid loading AES/GHASH round keys into local variables
The performance impact is not measurable, as the compiler loads these variables in xmm registers in unrolled loops anyway. However, we avoid loading these sensitive keys onto the stack. This happens for larger key schedules, where the register count is insufficient. If that key material is not on the stack, we can avoid to wipe it explicitly after crypto operations.
This commit is contained in:
Martin Willi
@@ -70,22 +70,10 @@ struct private_aesni_cbc_t {
|
||||
static void encrypt_cbc128(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
u_char *iv, u_char *out)
|
||||
{
|
||||
__m128i k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10;
|
||||
__m128i t, fb, *bi, *bo;
|
||||
__m128i *ks, t, fb, *bi, *bo;
|
||||
int i;
|
||||
|
||||
k0 = key->schedule[0];
|
||||
k1 = key->schedule[1];
|
||||
k2 = key->schedule[2];
|
||||
k3 = key->schedule[3];
|
||||
k4 = key->schedule[4];
|
||||
k5 = key->schedule[5];
|
||||
k6 = key->schedule[6];
|
||||
k7 = key->schedule[7];
|
||||
k8 = key->schedule[8];
|
||||
k9 = key->schedule[9];
|
||||
k10 = key->schedule[10];
|
||||
|
||||
ks = key->schedule;
|
||||
bi = (__m128i*)in;
|
||||
bo = (__m128i*)out;
|
||||
|
||||
@@ -94,19 +82,19 @@ static void encrypt_cbc128(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
{
|
||||
t = _mm_loadu_si128(bi + i);
|
||||
fb = _mm_xor_si128(t, fb);
|
||||
fb = _mm_xor_si128(fb, k0);
|
||||
fb = _mm_xor_si128(fb, ks[0]);
|
||||
|
||||
fb = _mm_aesenc_si128(fb, k1);
|
||||
fb = _mm_aesenc_si128(fb, k2);
|
||||
fb = _mm_aesenc_si128(fb, k3);
|
||||
fb = _mm_aesenc_si128(fb, k4);
|
||||
fb = _mm_aesenc_si128(fb, k5);
|
||||
fb = _mm_aesenc_si128(fb, k6);
|
||||
fb = _mm_aesenc_si128(fb, k7);
|
||||
fb = _mm_aesenc_si128(fb, k8);
|
||||
fb = _mm_aesenc_si128(fb, k9);
|
||||
fb = _mm_aesenc_si128(fb, ks[1]);
|
||||
fb = _mm_aesenc_si128(fb, ks[2]);
|
||||
fb = _mm_aesenc_si128(fb, ks[3]);
|
||||
fb = _mm_aesenc_si128(fb, ks[4]);
|
||||
fb = _mm_aesenc_si128(fb, ks[5]);
|
||||
fb = _mm_aesenc_si128(fb, ks[6]);
|
||||
fb = _mm_aesenc_si128(fb, ks[7]);
|
||||
fb = _mm_aesenc_si128(fb, ks[8]);
|
||||
fb = _mm_aesenc_si128(fb, ks[9]);
|
||||
|
||||
fb = _mm_aesenclast_si128(fb, k10);
|
||||
fb = _mm_aesenclast_si128(fb, ks[10]);
|
||||
_mm_storeu_si128(bo + i, fb);
|
||||
}
|
||||
}
|
||||
@@ -117,24 +105,12 @@ static void encrypt_cbc128(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
static void decrypt_cbc128(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
u_char *iv, u_char *out)
|
||||
{
|
||||
__m128i k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10;
|
||||
__m128i last, *bi, *bo;
|
||||
__m128i *ks, last, *bi, *bo;
|
||||
__m128i t1, t2, t3, t4;
|
||||
__m128i f1, f2, f3, f4;
|
||||
u_int i, pblocks;
|
||||
|
||||
k0 = key->schedule[0];
|
||||
k1 = key->schedule[1];
|
||||
k2 = key->schedule[2];
|
||||
k3 = key->schedule[3];
|
||||
k4 = key->schedule[4];
|
||||
k5 = key->schedule[5];
|
||||
k6 = key->schedule[6];
|
||||
k7 = key->schedule[7];
|
||||
k8 = key->schedule[8];
|
||||
k9 = key->schedule[9];
|
||||
k10 = key->schedule[10];
|
||||
|
||||
ks = key->schedule;
|
||||
bi = (__m128i*)in;
|
||||
bo = (__m128i*)out;
|
||||
pblocks = blocks - (blocks % CBC_DECRYPT_PARALLELISM);
|
||||
@@ -153,52 +129,52 @@ static void decrypt_cbc128(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
f4 = t3;
|
||||
last = t4;
|
||||
|
||||
t1 = _mm_xor_si128(t1, k0);
|
||||
t2 = _mm_xor_si128(t2, k0);
|
||||
t3 = _mm_xor_si128(t3, k0);
|
||||
t4 = _mm_xor_si128(t4, k0);
|
||||
t1 = _mm_xor_si128(t1, ks[0]);
|
||||
t2 = _mm_xor_si128(t2, ks[0]);
|
||||
t3 = _mm_xor_si128(t3, ks[0]);
|
||||
t4 = _mm_xor_si128(t4, ks[0]);
|
||||
|
||||
t1 = _mm_aesdec_si128(t1, k1);
|
||||
t2 = _mm_aesdec_si128(t2, k1);
|
||||
t3 = _mm_aesdec_si128(t3, k1);
|
||||
t4 = _mm_aesdec_si128(t4, k1);
|
||||
t1 = _mm_aesdec_si128(t1, k2);
|
||||
t2 = _mm_aesdec_si128(t2, k2);
|
||||
t3 = _mm_aesdec_si128(t3, k2);
|
||||
t4 = _mm_aesdec_si128(t4, k2);
|
||||
t1 = _mm_aesdec_si128(t1, k3);
|
||||
t2 = _mm_aesdec_si128(t2, k3);
|
||||
t3 = _mm_aesdec_si128(t3, k3);
|
||||
t4 = _mm_aesdec_si128(t4, k3);
|
||||
t1 = _mm_aesdec_si128(t1, k4);
|
||||
t2 = _mm_aesdec_si128(t2, k4);
|
||||
t3 = _mm_aesdec_si128(t3, k4);
|
||||
t4 = _mm_aesdec_si128(t4, k4);
|
||||
t1 = _mm_aesdec_si128(t1, k5);
|
||||
t2 = _mm_aesdec_si128(t2, k5);
|
||||
t3 = _mm_aesdec_si128(t3, k5);
|
||||
t4 = _mm_aesdec_si128(t4, k5);
|
||||
t1 = _mm_aesdec_si128(t1, k6);
|
||||
t2 = _mm_aesdec_si128(t2, k6);
|
||||
t3 = _mm_aesdec_si128(t3, k6);
|
||||
t4 = _mm_aesdec_si128(t4, k6);
|
||||
t1 = _mm_aesdec_si128(t1, k7);
|
||||
t2 = _mm_aesdec_si128(t2, k7);
|
||||
t3 = _mm_aesdec_si128(t3, k7);
|
||||
t4 = _mm_aesdec_si128(t4, k7);
|
||||
t1 = _mm_aesdec_si128(t1, k8);
|
||||
t2 = _mm_aesdec_si128(t2, k8);
|
||||
t3 = _mm_aesdec_si128(t3, k8);
|
||||
t4 = _mm_aesdec_si128(t4, k8);
|
||||
t1 = _mm_aesdec_si128(t1, k9);
|
||||
t2 = _mm_aesdec_si128(t2, k9);
|
||||
t3 = _mm_aesdec_si128(t3, k9);
|
||||
t4 = _mm_aesdec_si128(t4, k9);
|
||||
t1 = _mm_aesdec_si128(t1, ks[1]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[1]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[1]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[1]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[2]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[2]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[2]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[2]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[3]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[3]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[3]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[3]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[4]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[4]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[4]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[4]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[5]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[5]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[5]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[5]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[6]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[6]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[6]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[6]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[7]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[7]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[7]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[7]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[8]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[8]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[8]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[8]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[9]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[9]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[9]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[9]);
|
||||
|
||||
t1 = _mm_aesdeclast_si128(t1, k10);
|
||||
t2 = _mm_aesdeclast_si128(t2, k10);
|
||||
t3 = _mm_aesdeclast_si128(t3, k10);
|
||||
t4 = _mm_aesdeclast_si128(t4, k10);
|
||||
t1 = _mm_aesdeclast_si128(t1, ks[10]);
|
||||
t2 = _mm_aesdeclast_si128(t2, ks[10]);
|
||||
t3 = _mm_aesdeclast_si128(t3, ks[10]);
|
||||
t4 = _mm_aesdeclast_si128(t4, ks[10]);
|
||||
t1 = _mm_xor_si128(t1, f1);
|
||||
t2 = _mm_xor_si128(t2, f2);
|
||||
t3 = _mm_xor_si128(t3, f3);
|
||||
@@ -213,19 +189,19 @@ static void decrypt_cbc128(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
for (i = pblocks; i < blocks; i++)
|
||||
{
|
||||
last = _mm_loadu_si128(bi + i);
|
||||
t1 = _mm_xor_si128(last, k0);
|
||||
t1 = _mm_xor_si128(last, ks[0]);
|
||||
|
||||
t1 = _mm_aesdec_si128(t1, k1);
|
||||
t1 = _mm_aesdec_si128(t1, k2);
|
||||
t1 = _mm_aesdec_si128(t1, k3);
|
||||
t1 = _mm_aesdec_si128(t1, k4);
|
||||
t1 = _mm_aesdec_si128(t1, k5);
|
||||
t1 = _mm_aesdec_si128(t1, k6);
|
||||
t1 = _mm_aesdec_si128(t1, k7);
|
||||
t1 = _mm_aesdec_si128(t1, k8);
|
||||
t1 = _mm_aesdec_si128(t1, k9);
|
||||
t1 = _mm_aesdec_si128(t1, ks[1]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[2]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[3]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[4]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[5]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[6]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[7]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[8]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[9]);
|
||||
|
||||
t1 = _mm_aesdeclast_si128(t1, k10);
|
||||
t1 = _mm_aesdeclast_si128(t1, ks[10]);
|
||||
t1 = _mm_xor_si128(t1, f1);
|
||||
_mm_storeu_si128(bo + i, t1);
|
||||
f1 = last;
|
||||
@@ -238,24 +214,10 @@ static void decrypt_cbc128(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
static void encrypt_cbc192(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
u_char *iv, u_char *out)
|
||||
{
|
||||
__m128i k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12;
|
||||
__m128i t, fb, *bi, *bo;
|
||||
__m128i *ks, t, fb, *bi, *bo;
|
||||
int i;
|
||||
|
||||
k0 = key->schedule[0];
|
||||
k1 = key->schedule[1];
|
||||
k2 = key->schedule[2];
|
||||
k3 = key->schedule[3];
|
||||
k4 = key->schedule[4];
|
||||
k5 = key->schedule[5];
|
||||
k6 = key->schedule[6];
|
||||
k7 = key->schedule[7];
|
||||
k8 = key->schedule[8];
|
||||
k9 = key->schedule[9];
|
||||
k10 = key->schedule[10];
|
||||
k11 = key->schedule[11];
|
||||
k12 = key->schedule[12];
|
||||
|
||||
ks = key->schedule;
|
||||
bi = (__m128i*)in;
|
||||
bo = (__m128i*)out;
|
||||
|
||||
@@ -264,21 +226,21 @@ static void encrypt_cbc192(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
{
|
||||
t = _mm_loadu_si128(bi + i);
|
||||
fb = _mm_xor_si128(t, fb);
|
||||
fb = _mm_xor_si128(fb, k0);
|
||||
fb = _mm_xor_si128(fb, ks[0]);
|
||||
|
||||
fb = _mm_aesenc_si128(fb, k1);
|
||||
fb = _mm_aesenc_si128(fb, k2);
|
||||
fb = _mm_aesenc_si128(fb, k3);
|
||||
fb = _mm_aesenc_si128(fb, k4);
|
||||
fb = _mm_aesenc_si128(fb, k5);
|
||||
fb = _mm_aesenc_si128(fb, k6);
|
||||
fb = _mm_aesenc_si128(fb, k7);
|
||||
fb = _mm_aesenc_si128(fb, k8);
|
||||
fb = _mm_aesenc_si128(fb, k9);
|
||||
fb = _mm_aesenc_si128(fb, k10);
|
||||
fb = _mm_aesenc_si128(fb, k11);
|
||||
fb = _mm_aesenc_si128(fb, ks[1]);
|
||||
fb = _mm_aesenc_si128(fb, ks[2]);
|
||||
fb = _mm_aesenc_si128(fb, ks[3]);
|
||||
fb = _mm_aesenc_si128(fb, ks[4]);
|
||||
fb = _mm_aesenc_si128(fb, ks[5]);
|
||||
fb = _mm_aesenc_si128(fb, ks[6]);
|
||||
fb = _mm_aesenc_si128(fb, ks[7]);
|
||||
fb = _mm_aesenc_si128(fb, ks[8]);
|
||||
fb = _mm_aesenc_si128(fb, ks[9]);
|
||||
fb = _mm_aesenc_si128(fb, ks[10]);
|
||||
fb = _mm_aesenc_si128(fb, ks[11]);
|
||||
|
||||
fb = _mm_aesenclast_si128(fb, k12);
|
||||
fb = _mm_aesenclast_si128(fb, ks[12]);
|
||||
_mm_storeu_si128(bo + i, fb);
|
||||
}
|
||||
}
|
||||
@@ -289,26 +251,12 @@ static void encrypt_cbc192(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
static void decrypt_cbc192(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
u_char *iv, u_char *out)
|
||||
{
|
||||
__m128i k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12;
|
||||
__m128i last, *bi, *bo;
|
||||
__m128i *ks, last, *bi, *bo;
|
||||
__m128i t1, t2, t3, t4;
|
||||
__m128i f1, f2, f3, f4;
|
||||
u_int i, pblocks;
|
||||
|
||||
k0 = key->schedule[0];
|
||||
k1 = key->schedule[1];
|
||||
k2 = key->schedule[2];
|
||||
k3 = key->schedule[3];
|
||||
k4 = key->schedule[4];
|
||||
k5 = key->schedule[5];
|
||||
k6 = key->schedule[6];
|
||||
k7 = key->schedule[7];
|
||||
k8 = key->schedule[8];
|
||||
k9 = key->schedule[9];
|
||||
k10 = key->schedule[10];
|
||||
k11 = key->schedule[11];
|
||||
k12 = key->schedule[12];
|
||||
|
||||
ks = key->schedule;
|
||||
bi = (__m128i*)in;
|
||||
bo = (__m128i*)out;
|
||||
pblocks = blocks - (blocks % CBC_DECRYPT_PARALLELISM);
|
||||
@@ -327,60 +275,60 @@ static void decrypt_cbc192(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
f4 = t3;
|
||||
last = t4;
|
||||
|
||||
t1 = _mm_xor_si128(t1, k0);
|
||||
t2 = _mm_xor_si128(t2, k0);
|
||||
t3 = _mm_xor_si128(t3, k0);
|
||||
t4 = _mm_xor_si128(t4, k0);
|
||||
t1 = _mm_xor_si128(t1, ks[0]);
|
||||
t2 = _mm_xor_si128(t2, ks[0]);
|
||||
t3 = _mm_xor_si128(t3, ks[0]);
|
||||
t4 = _mm_xor_si128(t4, ks[0]);
|
||||
|
||||
t1 = _mm_aesdec_si128(t1, k1);
|
||||
t2 = _mm_aesdec_si128(t2, k1);
|
||||
t3 = _mm_aesdec_si128(t3, k1);
|
||||
t4 = _mm_aesdec_si128(t4, k1);
|
||||
t1 = _mm_aesdec_si128(t1, k2);
|
||||
t2 = _mm_aesdec_si128(t2, k2);
|
||||
t3 = _mm_aesdec_si128(t3, k2);
|
||||
t4 = _mm_aesdec_si128(t4, k2);
|
||||
t1 = _mm_aesdec_si128(t1, k3);
|
||||
t2 = _mm_aesdec_si128(t2, k3);
|
||||
t3 = _mm_aesdec_si128(t3, k3);
|
||||
t4 = _mm_aesdec_si128(t4, k3);
|
||||
t1 = _mm_aesdec_si128(t1, k4);
|
||||
t2 = _mm_aesdec_si128(t2, k4);
|
||||
t3 = _mm_aesdec_si128(t3, k4);
|
||||
t4 = _mm_aesdec_si128(t4, k4);
|
||||
t1 = _mm_aesdec_si128(t1, k5);
|
||||
t2 = _mm_aesdec_si128(t2, k5);
|
||||
t3 = _mm_aesdec_si128(t3, k5);
|
||||
t4 = _mm_aesdec_si128(t4, k5);
|
||||
t1 = _mm_aesdec_si128(t1, k6);
|
||||
t2 = _mm_aesdec_si128(t2, k6);
|
||||
t3 = _mm_aesdec_si128(t3, k6);
|
||||
t4 = _mm_aesdec_si128(t4, k6);
|
||||
t1 = _mm_aesdec_si128(t1, k7);
|
||||
t2 = _mm_aesdec_si128(t2, k7);
|
||||
t3 = _mm_aesdec_si128(t3, k7);
|
||||
t4 = _mm_aesdec_si128(t4, k7);
|
||||
t1 = _mm_aesdec_si128(t1, k8);
|
||||
t2 = _mm_aesdec_si128(t2, k8);
|
||||
t3 = _mm_aesdec_si128(t3, k8);
|
||||
t4 = _mm_aesdec_si128(t4, k8);
|
||||
t1 = _mm_aesdec_si128(t1, k9);
|
||||
t2 = _mm_aesdec_si128(t2, k9);
|
||||
t3 = _mm_aesdec_si128(t3, k9);
|
||||
t4 = _mm_aesdec_si128(t4, k9);
|
||||
t1 = _mm_aesdec_si128(t1, k10);
|
||||
t2 = _mm_aesdec_si128(t2, k10);
|
||||
t3 = _mm_aesdec_si128(t3, k10);
|
||||
t4 = _mm_aesdec_si128(t4, k10);
|
||||
t1 = _mm_aesdec_si128(t1, k11);
|
||||
t2 = _mm_aesdec_si128(t2, k11);
|
||||
t3 = _mm_aesdec_si128(t3, k11);
|
||||
t4 = _mm_aesdec_si128(t4, k11);
|
||||
t1 = _mm_aesdec_si128(t1, ks[1]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[1]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[1]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[1]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[2]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[2]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[2]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[2]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[3]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[3]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[3]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[3]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[4]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[4]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[4]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[4]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[5]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[5]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[5]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[5]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[6]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[6]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[6]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[6]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[7]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[7]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[7]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[7]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[8]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[8]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[8]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[8]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[9]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[9]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[9]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[9]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[10]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[10]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[10]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[10]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[11]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[11]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[11]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[11]);
|
||||
|
||||
t1 = _mm_aesdeclast_si128(t1, k12);
|
||||
t2 = _mm_aesdeclast_si128(t2, k12);
|
||||
t3 = _mm_aesdeclast_si128(t3, k12);
|
||||
t4 = _mm_aesdeclast_si128(t4, k12);
|
||||
t1 = _mm_aesdeclast_si128(t1, ks[12]);
|
||||
t2 = _mm_aesdeclast_si128(t2, ks[12]);
|
||||
t3 = _mm_aesdeclast_si128(t3, ks[12]);
|
||||
t4 = _mm_aesdeclast_si128(t4, ks[12]);
|
||||
t1 = _mm_xor_si128(t1, f1);
|
||||
t2 = _mm_xor_si128(t2, f2);
|
||||
t3 = _mm_xor_si128(t3, f3);
|
||||
@@ -395,21 +343,21 @@ static void decrypt_cbc192(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
for (i = pblocks; i < blocks; i++)
|
||||
{
|
||||
last = _mm_loadu_si128(bi + i);
|
||||
t1 = _mm_xor_si128(last, k0);
|
||||
t1 = _mm_xor_si128(last, ks[0]);
|
||||
|
||||
t1 = _mm_aesdec_si128(t1, k1);
|
||||
t1 = _mm_aesdec_si128(t1, k2);
|
||||
t1 = _mm_aesdec_si128(t1, k3);
|
||||
t1 = _mm_aesdec_si128(t1, k4);
|
||||
t1 = _mm_aesdec_si128(t1, k5);
|
||||
t1 = _mm_aesdec_si128(t1, k6);
|
||||
t1 = _mm_aesdec_si128(t1, k7);
|
||||
t1 = _mm_aesdec_si128(t1, k8);
|
||||
t1 = _mm_aesdec_si128(t1, k9);
|
||||
t1 = _mm_aesdec_si128(t1, k10);
|
||||
t1 = _mm_aesdec_si128(t1, k11);
|
||||
t1 = _mm_aesdec_si128(t1, ks[1]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[2]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[3]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[4]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[5]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[6]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[7]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[8]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[9]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[10]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[11]);
|
||||
|
||||
t1 = _mm_aesdeclast_si128(t1, k12);
|
||||
t1 = _mm_aesdeclast_si128(t1, ks[12]);
|
||||
t1 = _mm_xor_si128(t1, f1);
|
||||
_mm_storeu_si128(bo + i, t1);
|
||||
f1 = last;
|
||||
@@ -422,26 +370,10 @@ static void decrypt_cbc192(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
static void encrypt_cbc256(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
u_char *iv, u_char *out)
|
||||
{
|
||||
__m128i k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12, k13, k14;
|
||||
__m128i t, fb, *bi, *bo;
|
||||
__m128i *ks, t, fb, *bi, *bo;
|
||||
int i;
|
||||
|
||||
k0 = key->schedule[0];
|
||||
k1 = key->schedule[1];
|
||||
k2 = key->schedule[2];
|
||||
k3 = key->schedule[3];
|
||||
k4 = key->schedule[4];
|
||||
k5 = key->schedule[5];
|
||||
k6 = key->schedule[6];
|
||||
k7 = key->schedule[7];
|
||||
k8 = key->schedule[8];
|
||||
k9 = key->schedule[9];
|
||||
k10 = key->schedule[10];
|
||||
k11 = key->schedule[11];
|
||||
k12 = key->schedule[12];
|
||||
k13 = key->schedule[13];
|
||||
k14 = key->schedule[14];
|
||||
|
||||
ks = key->schedule;
|
||||
bi = (__m128i*)in;
|
||||
bo = (__m128i*)out;
|
||||
|
||||
@@ -450,23 +382,23 @@ static void encrypt_cbc256(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
{
|
||||
t = _mm_loadu_si128(bi + i);
|
||||
fb = _mm_xor_si128(t, fb);
|
||||
fb = _mm_xor_si128(fb, k0);
|
||||
fb = _mm_xor_si128(fb, ks[0]);
|
||||
|
||||
fb = _mm_aesenc_si128(fb, k1);
|
||||
fb = _mm_aesenc_si128(fb, k2);
|
||||
fb = _mm_aesenc_si128(fb, k3);
|
||||
fb = _mm_aesenc_si128(fb, k4);
|
||||
fb = _mm_aesenc_si128(fb, k5);
|
||||
fb = _mm_aesenc_si128(fb, k6);
|
||||
fb = _mm_aesenc_si128(fb, k7);
|
||||
fb = _mm_aesenc_si128(fb, k8);
|
||||
fb = _mm_aesenc_si128(fb, k9);
|
||||
fb = _mm_aesenc_si128(fb, k10);
|
||||
fb = _mm_aesenc_si128(fb, k11);
|
||||
fb = _mm_aesenc_si128(fb, k12);
|
||||
fb = _mm_aesenc_si128(fb, k13);
|
||||
fb = _mm_aesenc_si128(fb, ks[1]);
|
||||
fb = _mm_aesenc_si128(fb, ks[2]);
|
||||
fb = _mm_aesenc_si128(fb, ks[3]);
|
||||
fb = _mm_aesenc_si128(fb, ks[4]);
|
||||
fb = _mm_aesenc_si128(fb, ks[5]);
|
||||
fb = _mm_aesenc_si128(fb, ks[6]);
|
||||
fb = _mm_aesenc_si128(fb, ks[7]);
|
||||
fb = _mm_aesenc_si128(fb, ks[8]);
|
||||
fb = _mm_aesenc_si128(fb, ks[9]);
|
||||
fb = _mm_aesenc_si128(fb, ks[10]);
|
||||
fb = _mm_aesenc_si128(fb, ks[11]);
|
||||
fb = _mm_aesenc_si128(fb, ks[12]);
|
||||
fb = _mm_aesenc_si128(fb, ks[13]);
|
||||
|
||||
fb = _mm_aesenclast_si128(fb, k14);
|
||||
fb = _mm_aesenclast_si128(fb, ks[14]);
|
||||
_mm_storeu_si128(bo + i, fb);
|
||||
}
|
||||
}
|
||||
@@ -477,28 +409,12 @@ static void encrypt_cbc256(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
static void decrypt_cbc256(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
u_char *iv, u_char *out)
|
||||
{
|
||||
__m128i k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12, k13, k14;
|
||||
__m128i last, *bi, *bo;
|
||||
__m128i *ks, last, *bi, *bo;
|
||||
__m128i t1, t2, t3, t4;
|
||||
__m128i f1, f2, f3, f4;
|
||||
u_int i, pblocks;
|
||||
|
||||
k0 = key->schedule[0];
|
||||
k1 = key->schedule[1];
|
||||
k2 = key->schedule[2];
|
||||
k3 = key->schedule[3];
|
||||
k4 = key->schedule[4];
|
||||
k5 = key->schedule[5];
|
||||
k6 = key->schedule[6];
|
||||
k7 = key->schedule[7];
|
||||
k8 = key->schedule[8];
|
||||
k9 = key->schedule[9];
|
||||
k10 = key->schedule[10];
|
||||
k11 = key->schedule[11];
|
||||
k12 = key->schedule[12];
|
||||
k13 = key->schedule[13];
|
||||
k14 = key->schedule[14];
|
||||
|
||||
ks = key->schedule;
|
||||
bi = (__m128i*)in;
|
||||
bo = (__m128i*)out;
|
||||
pblocks = blocks - (blocks % CBC_DECRYPT_PARALLELISM);
|
||||
@@ -517,68 +433,68 @@ static void decrypt_cbc256(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
f4 = t3;
|
||||
last = t4;
|
||||
|
||||
t1 = _mm_xor_si128(t1, k0);
|
||||
t2 = _mm_xor_si128(t2, k0);
|
||||
t3 = _mm_xor_si128(t3, k0);
|
||||
t4 = _mm_xor_si128(t4, k0);
|
||||
t1 = _mm_xor_si128(t1, ks[0]);
|
||||
t2 = _mm_xor_si128(t2, ks[0]);
|
||||
t3 = _mm_xor_si128(t3, ks[0]);
|
||||
t4 = _mm_xor_si128(t4, ks[0]);
|
||||
|
||||
t1 = _mm_aesdec_si128(t1, k1);
|
||||
t2 = _mm_aesdec_si128(t2, k1);
|
||||
t3 = _mm_aesdec_si128(t3, k1);
|
||||
t4 = _mm_aesdec_si128(t4, k1);
|
||||
t1 = _mm_aesdec_si128(t1, k2);
|
||||
t2 = _mm_aesdec_si128(t2, k2);
|
||||
t3 = _mm_aesdec_si128(t3, k2);
|
||||
t4 = _mm_aesdec_si128(t4, k2);
|
||||
t1 = _mm_aesdec_si128(t1, k3);
|
||||
t2 = _mm_aesdec_si128(t2, k3);
|
||||
t3 = _mm_aesdec_si128(t3, k3);
|
||||
t4 = _mm_aesdec_si128(t4, k3);
|
||||
t1 = _mm_aesdec_si128(t1, k4);
|
||||
t2 = _mm_aesdec_si128(t2, k4);
|
||||
t3 = _mm_aesdec_si128(t3, k4);
|
||||
t4 = _mm_aesdec_si128(t4, k4);
|
||||
t1 = _mm_aesdec_si128(t1, k5);
|
||||
t2 = _mm_aesdec_si128(t2, k5);
|
||||
t3 = _mm_aesdec_si128(t3, k5);
|
||||
t4 = _mm_aesdec_si128(t4, k5);
|
||||
t1 = _mm_aesdec_si128(t1, k6);
|
||||
t2 = _mm_aesdec_si128(t2, k6);
|
||||
t3 = _mm_aesdec_si128(t3, k6);
|
||||
t4 = _mm_aesdec_si128(t4, k6);
|
||||
t1 = _mm_aesdec_si128(t1, k7);
|
||||
t2 = _mm_aesdec_si128(t2, k7);
|
||||
t3 = _mm_aesdec_si128(t3, k7);
|
||||
t4 = _mm_aesdec_si128(t4, k7);
|
||||
t1 = _mm_aesdec_si128(t1, k8);
|
||||
t2 = _mm_aesdec_si128(t2, k8);
|
||||
t3 = _mm_aesdec_si128(t3, k8);
|
||||
t4 = _mm_aesdec_si128(t4, k8);
|
||||
t1 = _mm_aesdec_si128(t1, k9);
|
||||
t2 = _mm_aesdec_si128(t2, k9);
|
||||
t3 = _mm_aesdec_si128(t3, k9);
|
||||
t4 = _mm_aesdec_si128(t4, k9);
|
||||
t1 = _mm_aesdec_si128(t1, k10);
|
||||
t2 = _mm_aesdec_si128(t2, k10);
|
||||
t3 = _mm_aesdec_si128(t3, k10);
|
||||
t4 = _mm_aesdec_si128(t4, k10);
|
||||
t1 = _mm_aesdec_si128(t1, k11);
|
||||
t2 = _mm_aesdec_si128(t2, k11);
|
||||
t3 = _mm_aesdec_si128(t3, k11);
|
||||
t4 = _mm_aesdec_si128(t4, k11);
|
||||
t1 = _mm_aesdec_si128(t1, k12);
|
||||
t2 = _mm_aesdec_si128(t2, k12);
|
||||
t3 = _mm_aesdec_si128(t3, k12);
|
||||
t4 = _mm_aesdec_si128(t4, k12);
|
||||
t1 = _mm_aesdec_si128(t1, k13);
|
||||
t2 = _mm_aesdec_si128(t2, k13);
|
||||
t3 = _mm_aesdec_si128(t3, k13);
|
||||
t4 = _mm_aesdec_si128(t4, k13);
|
||||
t1 = _mm_aesdec_si128(t1, ks[1]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[1]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[1]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[1]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[2]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[2]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[2]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[2]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[3]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[3]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[3]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[3]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[4]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[4]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[4]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[4]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[5]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[5]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[5]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[5]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[6]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[6]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[6]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[6]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[7]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[7]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[7]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[7]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[8]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[8]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[8]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[8]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[9]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[9]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[9]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[9]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[10]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[10]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[10]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[10]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[11]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[11]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[11]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[11]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[12]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[12]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[12]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[12]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[13]);
|
||||
t2 = _mm_aesdec_si128(t2, ks[13]);
|
||||
t3 = _mm_aesdec_si128(t3, ks[13]);
|
||||
t4 = _mm_aesdec_si128(t4, ks[13]);
|
||||
|
||||
t1 = _mm_aesdeclast_si128(t1, k14);
|
||||
t2 = _mm_aesdeclast_si128(t2, k14);
|
||||
t3 = _mm_aesdeclast_si128(t3, k14);
|
||||
t4 = _mm_aesdeclast_si128(t4, k14);
|
||||
t1 = _mm_aesdeclast_si128(t1, ks[14]);
|
||||
t2 = _mm_aesdeclast_si128(t2, ks[14]);
|
||||
t3 = _mm_aesdeclast_si128(t3, ks[14]);
|
||||
t4 = _mm_aesdeclast_si128(t4, ks[14]);
|
||||
t1 = _mm_xor_si128(t1, f1);
|
||||
t2 = _mm_xor_si128(t2, f2);
|
||||
t3 = _mm_xor_si128(t3, f3);
|
||||
@@ -593,23 +509,23 @@ static void decrypt_cbc256(aesni_key_t *key, u_int blocks, u_char *in,
|
||||
for (i = pblocks; i < blocks; i++)
|
||||
{
|
||||
last = _mm_loadu_si128(bi + i);
|
||||
t1 = _mm_xor_si128(last, k0);
|
||||
t1 = _mm_xor_si128(last, ks[0]);
|
||||
|
||||
t1 = _mm_aesdec_si128(t1, k1);
|
||||
t1 = _mm_aesdec_si128(t1, k2);
|
||||
t1 = _mm_aesdec_si128(t1, k3);
|
||||
t1 = _mm_aesdec_si128(t1, k4);
|
||||
t1 = _mm_aesdec_si128(t1, k5);
|
||||
t1 = _mm_aesdec_si128(t1, k6);
|
||||
t1 = _mm_aesdec_si128(t1, k7);
|
||||
t1 = _mm_aesdec_si128(t1, k8);
|
||||
t1 = _mm_aesdec_si128(t1, k9);
|
||||
t1 = _mm_aesdec_si128(t1, k10);
|
||||
t1 = _mm_aesdec_si128(t1, k11);
|
||||
t1 = _mm_aesdec_si128(t1, k12);
|
||||
t1 = _mm_aesdec_si128(t1, k13);
|
||||
t1 = _mm_aesdec_si128(t1, ks[1]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[2]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[3]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[4]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[5]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[6]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[7]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[8]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[9]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[10]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[11]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[12]);
|
||||
t1 = _mm_aesdec_si128(t1, ks[13]);
|
||||
|
||||
t1 = _mm_aesdeclast_si128(t1, k14);
|
||||
t1 = _mm_aesdeclast_si128(t1, ks[14]);
|
||||
t1 = _mm_xor_si128(t1, f1);
|
||||
_mm_storeu_si128(bo + i, t1);
|
||||
f1 = last;
|
||||
|
||||
Reference in New Issue
Block a user