denozord/strongswan-ext
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

aesni: Use 4-way parallel en/decryption in GCM

Browse Source 
Increases overall performance by ~25%.
This commit is contained in:
Martin Willi
2015-04-15 11:35:28 +02:00
parent 677649cfb5
commit 58c44cdd00
1 changed files with 629 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libstrongswan/plugins/aesni/aesni_gcm.c
+629 -126
View File
@@ -24,6 +24,11 @@
#define IV_SIZE 8 #define IV_SIZE 8
#define SALT_SIZE (NONCE_SIZE - IV_SIZE) #define SALT_SIZE (NONCE_SIZE - IV_SIZE)
/**
* Parallel pipelining
*/
#define GCM_CRYPT_PARALLELISM 4
typedef struct private_aesni_gcm_t private_aesni_gcm_t; typedef struct private_aesni_gcm_t private_aesni_gcm_t;
/** /**
@@ -304,12 +309,15 @@ static void encrypt_gcm128(private_aesni_gcm_t *this,
size_t alen, u_char *assoc, u_char *icv) size_t alen, u_char *assoc, u_char *icv)
{ {
__m128i k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10; __m128i k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10;
__m128i d, t, y, j, cb, *bi, *bo; __m128i d1, d2, d3, d4, t1, t2, t3, t4;
u_int blocks, rem, i; __m128i y, j, cb, *bi, *bo;
u_int blocks, pblocks, rem, i;
j = create_j(this, iv); j = create_j(this, iv);
cb = increment_be(j);
y = icv_header(this, assoc, alen); y = icv_header(this, assoc, alen);
blocks = len / AES_BLOCK_SIZE; blocks = len / AES_BLOCK_SIZE;
pblocks = blocks - (blocks % GCM_CRYPT_PARALLELISM);
rem = len % AES_BLOCK_SIZE; rem = len % AES_BLOCK_SIZE;
bi = (__m128i*)in; bi = (__m128i*)in;
bo = (__m128i*)out; bo = (__m128i*)out;
@@ -326,27 +334,99 @@ static void encrypt_gcm128(private_aesni_gcm_t *this,
k9 = this->key->schedule[9]; k9 = this->key->schedule[9];
k10 = this->key->schedule[10]; k10 = this->key->schedule[10];
cb = increment_be(j); for (i = 0; i < pblocks; i += GCM_CRYPT_PARALLELISM)
for (i = 0; i < blocks; i++)
{ {
d = _mm_loadu_si128(bi + i); d1 = _mm_loadu_si128(bi + i + 0);
d2 = _mm_loadu_si128(bi + i + 1);
d3 = _mm_loadu_si128(bi + i + 2);
d4 = _mm_loadu_si128(bi + i + 3);
t = _mm_xor_si128(cb, k0); t1 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k1); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k2); t2 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k3); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k4); t3 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k5); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k6); t4 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k7); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k8);
t = _mm_aesenc_si128(t, k9);
t = _mm_aesenclast_si128(t, k10);
t = _mm_xor_si128(t, d); t1 = _mm_aesenc_si128(t1, k1);
_mm_storeu_si128(bo + i, t); t2 = _mm_aesenc_si128(t2, k1);
t3 = _mm_aesenc_si128(t3, k1);
t4 = _mm_aesenc_si128(t4, k1);
t1 = _mm_aesenc_si128(t1, k2);
t2 = _mm_aesenc_si128(t2, k2);
t3 = _mm_aesenc_si128(t3, k2);
t4 = _mm_aesenc_si128(t4, k2);
t1 = _mm_aesenc_si128(t1, k3);
t2 = _mm_aesenc_si128(t2, k3);
t3 = _mm_aesenc_si128(t3, k3);
t4 = _mm_aesenc_si128(t4, k3);
t1 = _mm_aesenc_si128(t1, k4);
t2 = _mm_aesenc_si128(t2, k4);
t3 = _mm_aesenc_si128(t3, k4);
t4 = _mm_aesenc_si128(t4, k4);
t1 = _mm_aesenc_si128(t1, k5);
t2 = _mm_aesenc_si128(t2, k5);
t3 = _mm_aesenc_si128(t3, k5);
t4 = _mm_aesenc_si128(t4, k5);
t1 = _mm_aesenc_si128(t1, k6);
t2 = _mm_aesenc_si128(t2, k6);
t3 = _mm_aesenc_si128(t3, k6);
t4 = _mm_aesenc_si128(t4, k6);
t1 = _mm_aesenc_si128(t1, k7);
t2 = _mm_aesenc_si128(t2, k7);
t3 = _mm_aesenc_si128(t3, k7);
t4 = _mm_aesenc_si128(t4, k7);
t1 = _mm_aesenc_si128(t1, k8);
t2 = _mm_aesenc_si128(t2, k8);
t3 = _mm_aesenc_si128(t3, k8);
t4 = _mm_aesenc_si128(t4, k8);
t1 = _mm_aesenc_si128(t1, k9);
t2 = _mm_aesenc_si128(t2, k9);
t3 = _mm_aesenc_si128(t3, k9);
t4 = _mm_aesenc_si128(t4, k9);
y = ghash(this->h, y, t); t1 = _mm_aesenclast_si128(t1, k10);
t2 = _mm_aesenclast_si128(t2, k10);
t3 = _mm_aesenclast_si128(t3, k10);
t4 = _mm_aesenclast_si128(t4, k10);
t1 = _mm_xor_si128(t1, d1);
t2 = _mm_xor_si128(t2, d2);
t3 = _mm_xor_si128(t3, d3);
t4 = _mm_xor_si128(t4, d4);
_mm_storeu_si128(bo + i + 0, t1);
_mm_storeu_si128(bo + i + 1, t2);
_mm_storeu_si128(bo + i + 2, t3);
_mm_storeu_si128(bo + i + 3, t4);
y = ghash(this->h, y, t1);
y = ghash(this->h, y, t2);
y = ghash(this->h, y, t3);
y = ghash(this->h, y, t4);
}
for (i = pblocks; i < blocks; i++)
{
d1 = _mm_loadu_si128(bi + i);
t1 = _mm_xor_si128(cb, k0);
t1 = _mm_aesenc_si128(t1, k1);
t1 = _mm_aesenc_si128(t1, k2);
t1 = _mm_aesenc_si128(t1, k3);
t1 = _mm_aesenc_si128(t1, k4);
t1 = _mm_aesenc_si128(t1, k5);
t1 = _mm_aesenc_si128(t1, k6);
t1 = _mm_aesenc_si128(t1, k7);
t1 = _mm_aesenc_si128(t1, k8);
t1 = _mm_aesenc_si128(t1, k9);
t1 = _mm_aesenclast_si128(t1, k10);
t1 = _mm_xor_si128(t1, d1);
_mm_storeu_si128(bo + i, t1);
y = ghash(this->h, y, t1);
cb = increment_be(cb); cb = increment_be(cb);
} }
@@ -367,12 +447,15 @@ static void decrypt_gcm128(private_aesni_gcm_t *this,
size_t alen, u_char *assoc, u_char *icv) size_t alen, u_char *assoc, u_char *icv)
{ {
__m128i k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10; __m128i k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10;
__m128i d, t, y, j, cb, *bi, *bo; __m128i d1, d2, d3, d4, t1, t2, t3, t4;
u_int blocks, rem, i; __m128i y, j, cb, *bi, *bo;
u_int blocks, pblocks, rem, i;
j = create_j(this, iv); j = create_j(this, iv);
cb = increment_be(j);
y = icv_header(this, assoc, alen); y = icv_header(this, assoc, alen);
blocks = len / AES_BLOCK_SIZE; blocks = len / AES_BLOCK_SIZE;
pblocks = blocks - (blocks % GCM_CRYPT_PARALLELISM);
rem = len % AES_BLOCK_SIZE; rem = len % AES_BLOCK_SIZE;
bi = (__m128i*)in; bi = (__m128i*)in;
bo = (__m128i*)out; bo = (__m128i*)out;
@@ -389,27 +472,99 @@ static void decrypt_gcm128(private_aesni_gcm_t *this,
k9 = this->key->schedule[9]; k9 = this->key->schedule[9];
k10 = this->key->schedule[10]; k10 = this->key->schedule[10];
cb = increment_be(j); for (i = 0; i < pblocks; i += GCM_CRYPT_PARALLELISM)
for (i = 0; i < blocks; i++)
{ {
d = _mm_loadu_si128(bi + i); d1 = _mm_loadu_si128(bi + i + 0);
d2 = _mm_loadu_si128(bi + i + 1);
d3 = _mm_loadu_si128(bi + i + 2);
d4 = _mm_loadu_si128(bi + i + 3);
y = ghash(this->h, y, d); y = ghash(this->h, y, d1);
y = ghash(this->h, y, d2);
y = ghash(this->h, y, d3);
y = ghash(this->h, y, d4);
t = _mm_xor_si128(cb, k0); t1 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k1); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k2); t2 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k3); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k4); t3 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k5); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k6); t4 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k7); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k8);
t = _mm_aesenc_si128(t, k9);
t = _mm_aesenclast_si128(t, k10);
t = _mm_xor_si128(t, d); t1 = _mm_aesenc_si128(t1, k1);
_mm_storeu_si128(bo + i, t); t2 = _mm_aesenc_si128(t2, k1);
t3 = _mm_aesenc_si128(t3, k1);
t4 = _mm_aesenc_si128(t4, k1);
t1 = _mm_aesenc_si128(t1, k2);
t2 = _mm_aesenc_si128(t2, k2);
t3 = _mm_aesenc_si128(t3, k2);
t4 = _mm_aesenc_si128(t4, k2);
t1 = _mm_aesenc_si128(t1, k3);
t2 = _mm_aesenc_si128(t2, k3);
t3 = _mm_aesenc_si128(t3, k3);
t4 = _mm_aesenc_si128(t4, k3);
t1 = _mm_aesenc_si128(t1, k4);
t2 = _mm_aesenc_si128(t2, k4);
t3 = _mm_aesenc_si128(t3, k4);
t4 = _mm_aesenc_si128(t4, k4);
t1 = _mm_aesenc_si128(t1, k5);
t2 = _mm_aesenc_si128(t2, k5);
t3 = _mm_aesenc_si128(t3, k5);
t4 = _mm_aesenc_si128(t4, k5);
t1 = _mm_aesenc_si128(t1, k6);
t2 = _mm_aesenc_si128(t2, k6);
t3 = _mm_aesenc_si128(t3, k6);
t4 = _mm_aesenc_si128(t4, k6);
t1 = _mm_aesenc_si128(t1, k7);
t2 = _mm_aesenc_si128(t2, k7);
t3 = _mm_aesenc_si128(t3, k7);
t4 = _mm_aesenc_si128(t4, k7);
t1 = _mm_aesenc_si128(t1, k8);
t2 = _mm_aesenc_si128(t2, k8);
t3 = _mm_aesenc_si128(t3, k8);
t4 = _mm_aesenc_si128(t4, k8);
t1 = _mm_aesenc_si128(t1, k9);
t2 = _mm_aesenc_si128(t2, k9);
t3 = _mm_aesenc_si128(t3, k9);
t4 = _mm_aesenc_si128(t4, k9);
t1 = _mm_aesenclast_si128(t1, k10);
t2 = _mm_aesenclast_si128(t2, k10);
t3 = _mm_aesenclast_si128(t3, k10);
t4 = _mm_aesenclast_si128(t4, k10);
t1 = _mm_xor_si128(t1, d1);
t2 = _mm_xor_si128(t2, d2);
t3 = _mm_xor_si128(t3, d3);
t4 = _mm_xor_si128(t4, d4);
_mm_storeu_si128(bo + i + 0, t1);
_mm_storeu_si128(bo + i + 1, t2);
_mm_storeu_si128(bo + i + 2, t3);
_mm_storeu_si128(bo + i + 3, t4);
}
for (i = pblocks; i < blocks; i++)
{
d1 = _mm_loadu_si128(bi + i);
y = ghash(this->h, y, d1);
t1 = _mm_xor_si128(cb, k0);
t1 = _mm_aesenc_si128(t1, k1);
t1 = _mm_aesenc_si128(t1, k2);
t1 = _mm_aesenc_si128(t1, k3);
t1 = _mm_aesenc_si128(t1, k4);
t1 = _mm_aesenc_si128(t1, k5);
t1 = _mm_aesenc_si128(t1, k6);
t1 = _mm_aesenc_si128(t1, k7);
t1 = _mm_aesenc_si128(t1, k8);
t1 = _mm_aesenc_si128(t1, k9);
t1 = _mm_aesenclast_si128(t1, k10);
t1 = _mm_xor_si128(t1, d1);
_mm_storeu_si128(bo + i, t1);
cb = increment_be(cb); cb = increment_be(cb);
} }
@@ -430,12 +585,15 @@ static void encrypt_gcm192(private_aesni_gcm_t *this,
size_t alen, u_char *assoc, u_char *icv) size_t alen, u_char *assoc, u_char *icv)
{ {
__m128i k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12; __m128i k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12;
__m128i d, t, y, j, cb, *bi, *bo; __m128i d1, d2, d3, d4, t1, t2, t3, t4;
u_int blocks, rem, i; __m128i y, j, cb, *bi, *bo;
u_int blocks, pblocks, rem, i;
j = create_j(this, iv); j = create_j(this, iv);
cb = increment_be(j);
y = icv_header(this, assoc, alen); y = icv_header(this, assoc, alen);
blocks = len / AES_BLOCK_SIZE; blocks = len / AES_BLOCK_SIZE;
pblocks = blocks - (blocks % GCM_CRYPT_PARALLELISM);
rem = len % AES_BLOCK_SIZE; rem = len % AES_BLOCK_SIZE;
bi = (__m128i*)in; bi = (__m128i*)in;
bo = (__m128i*)out; bo = (__m128i*)out;
@@ -454,29 +612,109 @@ static void encrypt_gcm192(private_aesni_gcm_t *this,
k11 = this->key->schedule[11]; k11 = this->key->schedule[11];
k12 = this->key->schedule[12]; k12 = this->key->schedule[12];
cb = increment_be(j); for (i = 0; i < pblocks; i += GCM_CRYPT_PARALLELISM)
for (i = 0; i < blocks; i++)
{ {
d = _mm_loadu_si128(bi + i); d1 = _mm_loadu_si128(bi + i + 0);
d2 = _mm_loadu_si128(bi + i + 1);
d3 = _mm_loadu_si128(bi + i + 2);
d4 = _mm_loadu_si128(bi + i + 3);
t = _mm_xor_si128(cb, k0); t1 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k1); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k2); t2 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k3); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k4); t3 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k5); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k6); t4 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k7); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k8);
t = _mm_aesenc_si128(t, k9);
t = _mm_aesenc_si128(t, k10);
t = _mm_aesenc_si128(t, k11);
t = _mm_aesenclast_si128(t, k12);
t = _mm_xor_si128(t, d); t1 = _mm_aesenc_si128(t1, k1);
_mm_storeu_si128(bo + i, t); t2 = _mm_aesenc_si128(t2, k1);
t3 = _mm_aesenc_si128(t3, k1);
t4 = _mm_aesenc_si128(t4, k1);
t1 = _mm_aesenc_si128(t1, k2);
t2 = _mm_aesenc_si128(t2, k2);
t3 = _mm_aesenc_si128(t3, k2);
t4 = _mm_aesenc_si128(t4, k2);
t1 = _mm_aesenc_si128(t1, k3);
t2 = _mm_aesenc_si128(t2, k3);
t3 = _mm_aesenc_si128(t3, k3);
t4 = _mm_aesenc_si128(t4, k3);
t1 = _mm_aesenc_si128(t1, k4);
t2 = _mm_aesenc_si128(t2, k4);
t3 = _mm_aesenc_si128(t3, k4);
t4 = _mm_aesenc_si128(t4, k4);
t1 = _mm_aesenc_si128(t1, k5);
t2 = _mm_aesenc_si128(t2, k5);
t3 = _mm_aesenc_si128(t3, k5);
t4 = _mm_aesenc_si128(t4, k5);
t1 = _mm_aesenc_si128(t1, k6);
t2 = _mm_aesenc_si128(t2, k6);
t3 = _mm_aesenc_si128(t3, k6);
t4 = _mm_aesenc_si128(t4, k6);
t1 = _mm_aesenc_si128(t1, k7);
t2 = _mm_aesenc_si128(t2, k7);
t3 = _mm_aesenc_si128(t3, k7);
t4 = _mm_aesenc_si128(t4, k7);
t1 = _mm_aesenc_si128(t1, k8);
t2 = _mm_aesenc_si128(t2, k8);
t3 = _mm_aesenc_si128(t3, k8);
t4 = _mm_aesenc_si128(t4, k8);
t1 = _mm_aesenc_si128(t1, k9);
t2 = _mm_aesenc_si128(t2, k9);
t3 = _mm_aesenc_si128(t3, k9);
t4 = _mm_aesenc_si128(t4, k9);
t1 = _mm_aesenc_si128(t1, k10);
t2 = _mm_aesenc_si128(t2, k10);
t3 = _mm_aesenc_si128(t3, k10);
t4 = _mm_aesenc_si128(t4, k10);
t1 = _mm_aesenc_si128(t1, k11);
t2 = _mm_aesenc_si128(t2, k11);
t3 = _mm_aesenc_si128(t3, k11);
t4 = _mm_aesenc_si128(t4, k11);
y = ghash(this->h, y, t); t1 = _mm_aesenclast_si128(t1, k12);
t2 = _mm_aesenclast_si128(t2, k12);
t3 = _mm_aesenclast_si128(t3, k12);
t4 = _mm_aesenclast_si128(t4, k12);
t1 = _mm_xor_si128(t1, d1);
t2 = _mm_xor_si128(t2, d2);
t3 = _mm_xor_si128(t3, d3);
t4 = _mm_xor_si128(t4, d4);
_mm_storeu_si128(bo + i + 0, t1);
_mm_storeu_si128(bo + i + 1, t2);
_mm_storeu_si128(bo + i + 2, t3);
_mm_storeu_si128(bo + i + 3, t4);
y = ghash(this->h, y, t1);
y = ghash(this->h, y, t2);
y = ghash(this->h, y, t3);
y = ghash(this->h, y, t4);
}
for (i = pblocks; i < blocks; i++)
{
d1 = _mm_loadu_si128(bi + i);
t1 = _mm_xor_si128(cb, k0);
t1 = _mm_aesenc_si128(t1, k1);
t1 = _mm_aesenc_si128(t1, k2);
t1 = _mm_aesenc_si128(t1, k3);
t1 = _mm_aesenc_si128(t1, k4);
t1 = _mm_aesenc_si128(t1, k5);
t1 = _mm_aesenc_si128(t1, k6);
t1 = _mm_aesenc_si128(t1, k7);
t1 = _mm_aesenc_si128(t1, k8);
t1 = _mm_aesenc_si128(t1, k9);
t1 = _mm_aesenc_si128(t1, k10);
t1 = _mm_aesenc_si128(t1, k11);
t1 = _mm_aesenclast_si128(t1, k12);
t1 = _mm_xor_si128(t1, d1);
_mm_storeu_si128(bo + i, t1);
y = ghash(this->h, y, t1);
cb = increment_be(cb); cb = increment_be(cb);
} }
@@ -497,12 +735,15 @@ static void decrypt_gcm192(private_aesni_gcm_t *this,
size_t alen, u_char *assoc, u_char *icv) size_t alen, u_char *assoc, u_char *icv)
{ {
__m128i k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12; __m128i k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12;
__m128i d, t, y, j, cb, *bi, *bo; __m128i d1, d2, d3, d4, t1, t2, t3, t4;
u_int blocks, rem, i; __m128i y, j, cb, *bi, *bo;
u_int blocks, pblocks, rem, i;
j = create_j(this, iv); j = create_j(this, iv);
cb = increment_be(j);
y = icv_header(this, assoc, alen); y = icv_header(this, assoc, alen);
blocks = len / AES_BLOCK_SIZE; blocks = len / AES_BLOCK_SIZE;
pblocks = blocks - (blocks % GCM_CRYPT_PARALLELISM);
rem = len % AES_BLOCK_SIZE; rem = len % AES_BLOCK_SIZE;
bi = (__m128i*)in; bi = (__m128i*)in;
bo = (__m128i*)out; bo = (__m128i*)out;
@@ -521,29 +762,109 @@ static void decrypt_gcm192(private_aesni_gcm_t *this,
k11 = this->key->schedule[11]; k11 = this->key->schedule[11];
k12 = this->key->schedule[12]; k12 = this->key->schedule[12];
cb = increment_be(j); for (i = 0; i < pblocks; i += GCM_CRYPT_PARALLELISM)
for (i = 0; i < blocks; i++)
{ {
d = _mm_loadu_si128(bi + i); d1 = _mm_loadu_si128(bi + i + 0);
d2 = _mm_loadu_si128(bi + i + 1);
d3 = _mm_loadu_si128(bi + i + 2);
d4 = _mm_loadu_si128(bi + i + 3);
y = ghash(this->h, y, d); y = ghash(this->h, y, d1);
y = ghash(this->h, y, d2);
y = ghash(this->h, y, d3);
y = ghash(this->h, y, d4);
t = _mm_xor_si128(cb, k0); t1 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k1); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k2); t2 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k3); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k4); t3 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k5); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k6); t4 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k7); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k8);
t = _mm_aesenc_si128(t, k9);
t = _mm_aesenc_si128(t, k10);
t = _mm_aesenc_si128(t, k11);
t = _mm_aesenclast_si128(t, k12);
t = _mm_xor_si128(t, d); t1 = _mm_aesenc_si128(t1, k1);
_mm_storeu_si128(bo + i, t); t2 = _mm_aesenc_si128(t2, k1);
t3 = _mm_aesenc_si128(t3, k1);
t4 = _mm_aesenc_si128(t4, k1);
t1 = _mm_aesenc_si128(t1, k2);
t2 = _mm_aesenc_si128(t2, k2);
t3 = _mm_aesenc_si128(t3, k2);
t4 = _mm_aesenc_si128(t4, k2);
t1 = _mm_aesenc_si128(t1, k3);
t2 = _mm_aesenc_si128(t2, k3);
t3 = _mm_aesenc_si128(t3, k3);
t4 = _mm_aesenc_si128(t4, k3);
t1 = _mm_aesenc_si128(t1, k4);
t2 = _mm_aesenc_si128(t2, k4);
t3 = _mm_aesenc_si128(t3, k4);
t4 = _mm_aesenc_si128(t4, k4);
t1 = _mm_aesenc_si128(t1, k5);
t2 = _mm_aesenc_si128(t2, k5);
t3 = _mm_aesenc_si128(t3, k5);
t4 = _mm_aesenc_si128(t4, k5);
t1 = _mm_aesenc_si128(t1, k6);
t2 = _mm_aesenc_si128(t2, k6);
t3 = _mm_aesenc_si128(t3, k6);
t4 = _mm_aesenc_si128(t4, k6);
t1 = _mm_aesenc_si128(t1, k7);
t2 = _mm_aesenc_si128(t2, k7);
t3 = _mm_aesenc_si128(t3, k7);
t4 = _mm_aesenc_si128(t4, k7);
t1 = _mm_aesenc_si128(t1, k8);
t2 = _mm_aesenc_si128(t2, k8);
t3 = _mm_aesenc_si128(t3, k8);
t4 = _mm_aesenc_si128(t4, k8);
t1 = _mm_aesenc_si128(t1, k9);
t2 = _mm_aesenc_si128(t2, k9);
t3 = _mm_aesenc_si128(t3, k9);
t4 = _mm_aesenc_si128(t4, k9);
t1 = _mm_aesenc_si128(t1, k10);
t2 = _mm_aesenc_si128(t2, k10);
t3 = _mm_aesenc_si128(t3, k10);
t4 = _mm_aesenc_si128(t4, k10);
t1 = _mm_aesenc_si128(t1, k11);
t2 = _mm_aesenc_si128(t2, k11);
t3 = _mm_aesenc_si128(t3, k11);
t4 = _mm_aesenc_si128(t4, k11);
t1 = _mm_aesenclast_si128(t1, k12);
t2 = _mm_aesenclast_si128(t2, k12);
t3 = _mm_aesenclast_si128(t3, k12);
t4 = _mm_aesenclast_si128(t4, k12);
t1 = _mm_xor_si128(t1, d1);
t2 = _mm_xor_si128(t2, d2);
t3 = _mm_xor_si128(t3, d3);
t4 = _mm_xor_si128(t4, d4);
_mm_storeu_si128(bo + i + 0, t1);
_mm_storeu_si128(bo + i + 1, t2);
_mm_storeu_si128(bo + i + 2, t3);
_mm_storeu_si128(bo + i + 3, t4);
}
for (i = pblocks; i < blocks; i++)
{
d1 = _mm_loadu_si128(bi + i);
y = ghash(this->h, y, d1);
t1 = _mm_xor_si128(cb, k0);
t1 = _mm_aesenc_si128(t1, k1);
t1 = _mm_aesenc_si128(t1, k2);
t1 = _mm_aesenc_si128(t1, k3);
t1 = _mm_aesenc_si128(t1, k4);
t1 = _mm_aesenc_si128(t1, k5);
t1 = _mm_aesenc_si128(t1, k6);
t1 = _mm_aesenc_si128(t1, k7);
t1 = _mm_aesenc_si128(t1, k8);
t1 = _mm_aesenc_si128(t1, k9);
t1 = _mm_aesenc_si128(t1, k10);
t1 = _mm_aesenc_si128(t1, k11);
t1 = _mm_aesenclast_si128(t1, k12);
t1 = _mm_xor_si128(t1, d1);
_mm_storeu_si128(bo + i, t1);
cb = increment_be(cb); cb = increment_be(cb);
} }
@@ -564,12 +885,15 @@ static void encrypt_gcm256(private_aesni_gcm_t *this,
size_t alen, u_char *assoc, u_char *icv) size_t alen, u_char *assoc, u_char *icv)
{ {
__m128i k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12, k13, k14; __m128i k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12, k13, k14;
__m128i d, t, y, j, cb, *bi, *bo; __m128i d1, d2, d3, d4, t1, t2, t3, t4;
u_int blocks, rem, i; __m128i y, j, cb, *bi, *bo;
u_int blocks, pblocks, rem, i;
j = create_j(this, iv); j = create_j(this, iv);
cb = increment_be(j);
y = icv_header(this, assoc, alen); y = icv_header(this, assoc, alen);
blocks = len / AES_BLOCK_SIZE; blocks = len / AES_BLOCK_SIZE;
pblocks = blocks - (blocks % GCM_CRYPT_PARALLELISM);
rem = len % AES_BLOCK_SIZE; rem = len % AES_BLOCK_SIZE;
bi = (__m128i*)in; bi = (__m128i*)in;
bo = (__m128i*)out; bo = (__m128i*)out;
@@ -590,31 +914,119 @@ static void encrypt_gcm256(private_aesni_gcm_t *this,
k13 = this->key->schedule[13]; k13 = this->key->schedule[13];
k14 = this->key->schedule[14]; k14 = this->key->schedule[14];
cb = increment_be(j); for (i = 0; i < pblocks; i += GCM_CRYPT_PARALLELISM)
for (i = 0; i < blocks; i++)
{ {
d = _mm_loadu_si128(bi + i); d1 = _mm_loadu_si128(bi + i + 0);
d2 = _mm_loadu_si128(bi + i + 1);
d3 = _mm_loadu_si128(bi + i + 2);
d4 = _mm_loadu_si128(bi + i + 3);
t = _mm_xor_si128(cb, k0); t1 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k1); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k2); t2 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k3); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k4); t3 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k5); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k6); t4 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k7); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k8);
t = _mm_aesenc_si128(t, k9);
t = _mm_aesenc_si128(t, k10);
t = _mm_aesenc_si128(t, k11);
t = _mm_aesenc_si128(t, k12);
t = _mm_aesenc_si128(t, k13);
t = _mm_aesenclast_si128(t, k14);
t = _mm_xor_si128(t, d); t1 = _mm_aesenc_si128(t1, k1);
_mm_storeu_si128(bo + i, t); t2 = _mm_aesenc_si128(t2, k1);
t3 = _mm_aesenc_si128(t3, k1);
t4 = _mm_aesenc_si128(t4, k1);
t1 = _mm_aesenc_si128(t1, k2);
t2 = _mm_aesenc_si128(t2, k2);
t3 = _mm_aesenc_si128(t3, k2);
t4 = _mm_aesenc_si128(t4, k2);
t1 = _mm_aesenc_si128(t1, k3);
t2 = _mm_aesenc_si128(t2, k3);
t3 = _mm_aesenc_si128(t3, k3);
t4 = _mm_aesenc_si128(t4, k3);
t1 = _mm_aesenc_si128(t1, k4);
t2 = _mm_aesenc_si128(t2, k4);
t3 = _mm_aesenc_si128(t3, k4);
t4 = _mm_aesenc_si128(t4, k4);
t1 = _mm_aesenc_si128(t1, k5);
t2 = _mm_aesenc_si128(t2, k5);
t3 = _mm_aesenc_si128(t3, k5);
t4 = _mm_aesenc_si128(t4, k5);
t1 = _mm_aesenc_si128(t1, k6);
t2 = _mm_aesenc_si128(t2, k6);
t3 = _mm_aesenc_si128(t3, k6);
t4 = _mm_aesenc_si128(t4, k6);
t1 = _mm_aesenc_si128(t1, k7);
t2 = _mm_aesenc_si128(t2, k7);
t3 = _mm_aesenc_si128(t3, k7);
t4 = _mm_aesenc_si128(t4, k7);
t1 = _mm_aesenc_si128(t1, k8);
t2 = _mm_aesenc_si128(t2, k8);
t3 = _mm_aesenc_si128(t3, k8);
t4 = _mm_aesenc_si128(t4, k8);
t1 = _mm_aesenc_si128(t1, k9);
t2 = _mm_aesenc_si128(t2, k9);
t3 = _mm_aesenc_si128(t3, k9);
t4 = _mm_aesenc_si128(t4, k9);
t1 = _mm_aesenc_si128(t1, k10);
t2 = _mm_aesenc_si128(t2, k10);
t3 = _mm_aesenc_si128(t3, k10);
t4 = _mm_aesenc_si128(t4, k10);
t1 = _mm_aesenc_si128(t1, k11);
t2 = _mm_aesenc_si128(t2, k11);
t3 = _mm_aesenc_si128(t3, k11);
t4 = _mm_aesenc_si128(t4, k11);
t1 = _mm_aesenc_si128(t1, k12);
t2 = _mm_aesenc_si128(t2, k12);
t3 = _mm_aesenc_si128(t3, k12);
t4 = _mm_aesenc_si128(t4, k12);
t1 = _mm_aesenc_si128(t1, k13);
t2 = _mm_aesenc_si128(t2, k13);
t3 = _mm_aesenc_si128(t3, k13);
t4 = _mm_aesenc_si128(t4, k13);
y = ghash(this->h, y, t); t1 = _mm_aesenclast_si128(t1, k14);
t2 = _mm_aesenclast_si128(t2, k14);
t3 = _mm_aesenclast_si128(t3, k14);
t4 = _mm_aesenclast_si128(t4, k14);
t1 = _mm_xor_si128(t1, d1);
t2 = _mm_xor_si128(t2, d2);
t3 = _mm_xor_si128(t3, d3);
t4 = _mm_xor_si128(t4, d4);
_mm_storeu_si128(bo + i + 0, t1);
_mm_storeu_si128(bo + i + 1, t2);
_mm_storeu_si128(bo + i + 2, t3);
_mm_storeu_si128(bo + i + 3, t4);
y = ghash(this->h, y, t1);
y = ghash(this->h, y, t2);
y = ghash(this->h, y, t3);
y = ghash(this->h, y, t4);
}
for (i = pblocks; i < blocks; i++)
{
d1 = _mm_loadu_si128(bi + i);
t1 = _mm_xor_si128(cb, k0);
t1 = _mm_aesenc_si128(t1, k1);
t1 = _mm_aesenc_si128(t1, k2);
t1 = _mm_aesenc_si128(t1, k3);
t1 = _mm_aesenc_si128(t1, k4);
t1 = _mm_aesenc_si128(t1, k5);
t1 = _mm_aesenc_si128(t1, k6);
t1 = _mm_aesenc_si128(t1, k7);
t1 = _mm_aesenc_si128(t1, k8);
t1 = _mm_aesenc_si128(t1, k9);
t1 = _mm_aesenc_si128(t1, k10);
t1 = _mm_aesenc_si128(t1, k11);
t1 = _mm_aesenc_si128(t1, k12);
t1 = _mm_aesenc_si128(t1, k13);
t1 = _mm_aesenclast_si128(t1, k14);
t1 = _mm_xor_si128(t1, d1);
_mm_storeu_si128(bo + i, t1);
y = ghash(this->h, y, t1);
cb = increment_be(cb); cb = increment_be(cb);
} }
@@ -635,12 +1047,15 @@ static void decrypt_gcm256(private_aesni_gcm_t *this,
size_t alen, u_char *assoc, u_char *icv) size_t alen, u_char *assoc, u_char *icv)
{ {
__m128i k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12, k13, k14; __m128i k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12, k13, k14;
__m128i d, t, y, j, cb, *bi, *bo; __m128i d1, d2, d3, d4, t1, t2, t3, t4;
u_int blocks, rem, i; __m128i y, j, cb, *bi, *bo;
u_int blocks, pblocks, rem, i;
j = create_j(this, iv); j = create_j(this, iv);
cb = increment_be(j);
y = icv_header(this, assoc, alen); y = icv_header(this, assoc, alen);
blocks = len / AES_BLOCK_SIZE; blocks = len / AES_BLOCK_SIZE;
pblocks = blocks - (blocks % GCM_CRYPT_PARALLELISM);
rem = len % AES_BLOCK_SIZE; rem = len % AES_BLOCK_SIZE;
bi = (__m128i*)in; bi = (__m128i*)in;
bo = (__m128i*)out; bo = (__m128i*)out;
@@ -661,31 +1076,119 @@ static void decrypt_gcm256(private_aesni_gcm_t *this,
k13 = this->key->schedule[13]; k13 = this->key->schedule[13];
k14 = this->key->schedule[14]; k14 = this->key->schedule[14];
cb = increment_be(j); for (i = 0; i < pblocks; i += GCM_CRYPT_PARALLELISM)
for (i = 0; i < blocks; i++)
{ {
d = _mm_loadu_si128(bi + i); d1 = _mm_loadu_si128(bi + i + 0);
d2 = _mm_loadu_si128(bi + i + 1);
d3 = _mm_loadu_si128(bi + i + 2);
d4 = _mm_loadu_si128(bi + i + 3);
y = ghash(this->h, y, d); y = ghash(this->h, y, d1);
y = ghash(this->h, y, d2);
y = ghash(this->h, y, d3);
y = ghash(this->h, y, d4);
t = _mm_xor_si128(cb, k0); t1 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k1); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k2); t2 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k3); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k4); t3 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k5); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k6); t4 = _mm_xor_si128(cb, k0);
t = _mm_aesenc_si128(t, k7); cb = increment_be(cb);
t = _mm_aesenc_si128(t, k8);
t = _mm_aesenc_si128(t, k9);
t = _mm_aesenc_si128(t, k10);
t = _mm_aesenc_si128(t, k11);
t = _mm_aesenc_si128(t, k12);
t = _mm_aesenc_si128(t, k13);
t = _mm_aesenclast_si128(t, k14);
t = _mm_xor_si128(t, d); t1 = _mm_aesenc_si128(t1, k1);
_mm_storeu_si128(bo + i, t); t2 = _mm_aesenc_si128(t2, k1);
t3 = _mm_aesenc_si128(t3, k1);
t4 = _mm_aesenc_si128(t4, k1);
t1 = _mm_aesenc_si128(t1, k2);
t2 = _mm_aesenc_si128(t2, k2);
t3 = _mm_aesenc_si128(t3, k2);
t4 = _mm_aesenc_si128(t4, k2);
t1 = _mm_aesenc_si128(t1, k3);
t2 = _mm_aesenc_si128(t2, k3);
t3 = _mm_aesenc_si128(t3, k3);
t4 = _mm_aesenc_si128(t4, k3);
t1 = _mm_aesenc_si128(t1, k4);
t2 = _mm_aesenc_si128(t2, k4);
t3 = _mm_aesenc_si128(t3, k4);
t4 = _mm_aesenc_si128(t4, k4);
t1 = _mm_aesenc_si128(t1, k5);
t2 = _mm_aesenc_si128(t2, k5);
t3 = _mm_aesenc_si128(t3, k5);
t4 = _mm_aesenc_si128(t4, k5);
t1 = _mm_aesenc_si128(t1, k6);
t2 = _mm_aesenc_si128(t2, k6);
t3 = _mm_aesenc_si128(t3, k6);
t4 = _mm_aesenc_si128(t4, k6);
t1 = _mm_aesenc_si128(t1, k7);
t2 = _mm_aesenc_si128(t2, k7);
t3 = _mm_aesenc_si128(t3, k7);
t4 = _mm_aesenc_si128(t4, k7);
t1 = _mm_aesenc_si128(t1, k8);
t2 = _mm_aesenc_si128(t2, k8);
t3 = _mm_aesenc_si128(t3, k8);
t4 = _mm_aesenc_si128(t4, k8);
t1 = _mm_aesenc_si128(t1, k9);
t2 = _mm_aesenc_si128(t2, k9);
t3 = _mm_aesenc_si128(t3, k9);
t4 = _mm_aesenc_si128(t4, k9);
t1 = _mm_aesenc_si128(t1, k10);
t2 = _mm_aesenc_si128(t2, k10);
t3 = _mm_aesenc_si128(t3, k10);
t4 = _mm_aesenc_si128(t4, k10);
t1 = _mm_aesenc_si128(t1, k11);
t2 = _mm_aesenc_si128(t2, k11);
t3 = _mm_aesenc_si128(t3, k11);
t4 = _mm_aesenc_si128(t4, k11);
t1 = _mm_aesenc_si128(t1, k12);
t2 = _mm_aesenc_si128(t2, k12);
t3 = _mm_aesenc_si128(t3, k12);
t4 = _mm_aesenc_si128(t4, k12);
t1 = _mm_aesenc_si128(t1, k13);
t2 = _mm_aesenc_si128(t2, k13);
t3 = _mm_aesenc_si128(t3, k13);
t4 = _mm_aesenc_si128(t4, k13);
t1 = _mm_aesenclast_si128(t1, k14);
t2 = _mm_aesenclast_si128(t2, k14);
t3 = _mm_aesenclast_si128(t3, k14);
t4 = _mm_aesenclast_si128(t4, k14);
t1 = _mm_xor_si128(t1, d1);
t2 = _mm_xor_si128(t2, d2);
t3 = _mm_xor_si128(t3, d3);
t4 = _mm_xor_si128(t4, d4);
_mm_storeu_si128(bo + i + 0, t1);
_mm_storeu_si128(bo + i + 1, t2);
_mm_storeu_si128(bo + i + 2, t3);
_mm_storeu_si128(bo + i + 3, t4);
}
for (i = pblocks; i < blocks; i++)
{
d1 = _mm_loadu_si128(bi + i);
y = ghash(this->h, y, d1);
t1 = _mm_xor_si128(cb, k0);
t1 = _mm_aesenc_si128(t1, k1);
t1 = _mm_aesenc_si128(t1, k2);
t1 = _mm_aesenc_si128(t1, k3);
t1 = _mm_aesenc_si128(t1, k4);
t1 = _mm_aesenc_si128(t1, k5);
t1 = _mm_aesenc_si128(t1, k6);
t1 = _mm_aesenc_si128(t1, k7);
t1 = _mm_aesenc_si128(t1, k8);
t1 = _mm_aesenc_si128(t1, k9);
t1 = _mm_aesenc_si128(t1, k10);
t1 = _mm_aesenc_si128(t1, k11);
t1 = _mm_aesenc_si128(t1, k12);
t1 = _mm_aesenc_si128(t1, k13);
t1 = _mm_aesenclast_si128(t1, k14);
t1 = _mm_xor_si128(t1, d1);
_mm_storeu_si128(bo + i, t1);
cb = increment_be(cb); cb = increment_be(cb);
} }