added AVX support

speck.c

@@ -6,7 +6,214 @@
 #include <stdlib.h>
 #include <stdint.h>
 
-#if defined (__SSE4_2__)// SSE support ----------------------------------------------------
+
+#if defined (__AVX2__)	// AVX support ----------------------------------------------------
+
+#include <immintrin.h>
+
+#define u32 uint32_t
+#define u64 uint64_t
+#define u256 __m256i
+
+#define LCS(x,r) (((x)<<r)|((x)>>(64-r)))
+#define RCS(x,r) (((x)>>r)|((x)<<(64-r)))
+
+#define XOR _mm256_xor_si256
+#define AND _mm256_and_si256
+#define ADD _mm256_add_epi64
+#define SL  _mm256_slli_epi64
+#define SR  _mm256_srli_epi64
+
+#define _q SET(0x3,0x1,0x2,0x0)
+#define _four SET(0x4,0x4,0x4,0x4)
+
+#define SET _mm256_set_epi64x
+#define SET1(X,c) (X=SET(c,c,c,c))
+#define SET4(X,c) (X=SET(c,c,c,c), X=ADD(X,_q))
+
+#define LOW  _mm256_unpacklo_epi64
+#define HIGH _mm256_unpackhi_epi64
+#define LD(ip) _mm256_loadu_si256((__m256i *)(ip))
+#define ST(ip,X) _mm256_storeu_si256((__m256i *)(ip),X)
+#define STORE(out,X,Y) (ST(out,LOW(Y,X)), ST(out+32,HIGH(Y,X)))
+#define STORE_ALT(out,X,Y) (ST(out,LOW(X,Y)), ST(out+32,HIGH(X,Y)))
+#define XOR_STORE(in,out,X,Y) (ST(out,XOR(LD(in),LOW(Y,X))), ST(out+32,XOR(LD(in+32),HIGH(Y,X))))
+#define XOR_STORE_ALT(in,out,X,Y) (ST(out,XOR(LD(in),LOW(X,Y))), ST(out+32,XOR(LD(in+32),HIGH(X,Y))))
+
+#define SHFL _mm256_shuffle_epi8
+#define R8 SET(0x080f0e0d0c0b0a09LL,0x0007060504030201LL,0x080f0e0d0c0b0a09LL,0x0007060504030201LL)
+#define L8 SET(0x0e0d0c0b0a09080fLL,0x0605040302010007LL,0x0e0d0c0b0a09080fLL,0x0605040302010007LL)
+#define ROL8(X)  (SHFL(X,L8))
+#define ROR8(X)  (SHFL(X,R8))
+#define ROL(X,r) (XOR(SL(X,r),SR(X,(64-r))))
+#define ROR(X,r) (XOR(SR(X,r),SL(X,(64-r))))
+
+#define numrounds   34
+#define numkeywords 4
+
+#define R(X,Y,k) (X=XOR(ADD(ROR8(X),Y),k), Y=XOR(ROL(Y,3),X))
+
+#define Rx4(X,Y,k)  (R(X[0],Y[0],k))
+#define Rx8(X,Y,k)  (R(X[0],Y[0],k), R(X[1],Y[1],k))
+#define Rx12(X,Y,k) (R(X[0],Y[0],k), R(X[1],Y[1],k), R(X[2],Y[2],k))
+
+#define Rx16(X,Y,k) (X[0]=ROR8(X[0]), X[0]=ADD(X[0],Y[0]), X[1]=ROR8(X[1]), X[1]=ADD(X[1],Y[1]), \
+		     X[2]=ROR8(X[2]), X[2]=ADD(X[2],Y[2]), X[3]=ROR8(X[3]), X[3]=ADD(X[3],Y[3]), \
+		     X[0]=XOR(X[0],k), X[1]=XOR(X[1],k), X[2]=XOR(X[2],k), X[3]=XOR(X[3],k), \
+		     Z[0]=Y[0], Z[1]=Y[1], Z[2]=Y[2], Z[3]=Y[3],	\
+		     Z[0]=SL(Z[0],3),  Y[0]=SR(Y[0],61), Z[1]=SL(Z[1],3), Y[1]=SR(Y[1],61), \
+		     Z[2]=SL(Z[2],3),  Y[2]=SR(Y[2],61), Z[3]=SL(Z[3],3), Y[3]=SR(Y[3],61), \
+		     Y[0]=XOR(Y[0],Z[0]), Y[1]=XOR(Y[1],Z[1]), Y[2]=XOR(Y[2],Z[2]), Y[3]=XOR(Y[3],Z[3]), \
+		     Y[0]=XOR(X[0],Y[0]), Y[1]=XOR(X[1],Y[1]), Y[2]=XOR(X[2],Y[2]), Y[3]=XOR(X[3],Y[3]))
+
+#define Rx2(x,y,k) (x[0]=RCS(x[0],8), x[1]=RCS(x[1],8), x[0]+=y[0], x[1]+=y[1],	\
+                    x[0]^=k, x[1]^=k, y[0]=LCS(y[0],3), y[1]=LCS(y[1],3), y[0]^=x[0], y[1]^=x[1])
+
+#define Rx1(x,y,k) (x[0]=RCS(x[0],8), x[0]+=y[0], x[0]^=k, y[0]=LCS(y[0],3), y[0]^=x[0])
+
+#define Rx1b(x,y,k) (x=RCS(x,8), x+=y, x^=k, y=LCS(y,3), y^=x)
+
+#define Enc(X,Y,k,n) (Rx##n(X,Y,k[0]),  Rx##n(X,Y,k[1]),  Rx##n(X,Y,k[2]),  Rx##n(X,Y,k[3]),  Rx##n(X,Y,k[4]),  Rx##n(X,Y,k[5]),  Rx##n(X,Y,k[6]),  Rx##n(X,Y,k[7]), \
+		      Rx##n(X,Y,k[8]),  Rx##n(X,Y,k[9]),  Rx##n(X,Y,k[10]), Rx##n(X,Y,k[11]), Rx##n(X,Y,k[12]), Rx##n(X,Y,k[13]), Rx##n(X,Y,k[14]), Rx##n(X,Y,k[15]), \
+		      Rx##n(X,Y,k[16]), Rx##n(X,Y,k[17]), Rx##n(X,Y,k[18]), Rx##n(X,Y,k[19]), Rx##n(X,Y,k[20]), Rx##n(X,Y,k[21]), Rx##n(X,Y,k[22]), Rx##n(X,Y,k[23]), \
+		      Rx##n(X,Y,k[24]), Rx##n(X,Y,k[25]), Rx##n(X,Y,k[26]), Rx##n(X,Y,k[27]), Rx##n(X,Y,k[28]), Rx##n(X,Y,k[29]), Rx##n(X,Y,k[30]), Rx##n(X,Y,k[31]), \
+		      Rx##n(X,Y,k[32]), Rx##n(X,Y,k[33]))
+
+#define RK(X,Y,k,key,i)   (SET1(k[i],Y), key[i]=Y, X=RCS(X,8), X+=Y, X^=i, Y=LCS(Y,3), Y^=X)
+
+#define EK(A,B,C,D,k,key) (RK(B,A,k,key,0),  RK(C,A,k,key,1),  RK(D,A,k,key,2),  RK(B,A,k,key,3),  RK(C,A,k,key,4),  RK(D,A,k,key,5),  RK(B,A,k,key,6), \
+			   RK(C,A,k,key,7),  RK(D,A,k,key,8),  RK(B,A,k,key,9),  RK(C,A,k,key,10), RK(D,A,k,key,11), RK(B,A,k,key,12), RK(C,A,k,key,13), \
+			   RK(D,A,k,key,14), RK(B,A,k,key,15), RK(C,A,k,key,16), RK(D,A,k,key,17), RK(B,A,k,key,18), RK(C,A,k,key,19), RK(D,A,k,key,20), \
+			   RK(B,A,k,key,21), RK(C,A,k,key,22), RK(D,A,k,key,23), RK(B,A,k,key,24), RK(C,A,k,key,25), RK(D,A,k,key,26), RK(B,A,k,key,27), \
+			   RK(C,A,k,key,28), RK(D,A,k,key,29), RK(B,A,k,key,30), RK(C,A,k,key,31), RK(D,A,k,key,32), RK(B,A,k,key,33))
+
+typedef struct {
+	u256 rk[34];
+	u64 key[34];
+} speck_context_t;
+
+
+static int Encrypt_Xor(unsigned char *out, const unsigned char *in, u64 nonce[], speck_context_t *ctx, int numbytes)
+{
+  u64  x[2],y[2];
+  u256 X[4],Y[4],Z[4];
+
+  if (numbytes==16){
+    x[0]=nonce[1]; y[0]=nonce[0]; nonce[0]++;
+    Enc(x,y,ctx->key,1);
+    ((u64 *)out)[1]=x[0]; ((u64 *)out)[0]=y[0];
+
+    return 0;
+  }
+
+  if (numbytes==32){
+    x[0]=nonce[1]; y[0]=nonce[0]; nonce[0]++;
+    x[1]=nonce[1]; y[1]=nonce[0]; nonce[0]++;
+    Enc(x,y,ctx->key,2);
+    ((u64 *)out)[1]=x[0]^((u64 *)in)[1]; ((u64 *)out)[0]=y[0]^((u64 *)in)[0];
+    ((u64 *)out)[3]=x[1]^((u64 *)in)[3]; ((u64 *)out)[2]=y[1]^((u64 *)in)[2];
+
+    return 0;
+  }
+
+  SET1(X[0],nonce[1]); SET4(Y[0],nonce[0]);
+
+  if (numbytes==64) Enc(X,Y,ctx->rk,4);
+  else{
+    X[1]=X[0];
+    Y[1]=ADD(Y[0],_four);
+    if (numbytes==128) Enc(X,Y,ctx->rk,8);
+    else{
+      X[2]=X[0];
+      Y[2]=ADD(Y[1],_four);
+      if (numbytes==192) Enc(X,Y,ctx->rk,12);
+      else{
+	X[3]=X[0];
+	Y[3]=ADD(Y[2],_four);
+	Enc(X,Y,ctx->rk,16);
+      }
+    }
+  }
+
+  nonce[0]+=(numbytes>>4);
+
+  XOR_STORE(in,out,X[0],Y[0]);
+  if (numbytes>=128) XOR_STORE(in+64,out+64,X[1],Y[1]);
+  if (numbytes>=192) XOR_STORE(in+128,out+128,X[2],Y[2]);
+  if (numbytes>=256) XOR_STORE(in+192,out+192,X[3],Y[3]);
+
+  return 0;
+}
+
+
+int speck_ctr( unsigned char *out, const unsigned char *in, unsigned long long inlen,
+	       const unsigned char *n, speck_context_t *ctx) {
+
+  int i;
+  u64 nonce[2];
+  unsigned char block[16];
+  u64 * const block64 = (u64 *)block;
+
+  if (!inlen) return 0;
+
+  nonce[0]=((u64 *)n)[0];
+  nonce[1]=((u64 *)n)[1];
+
+  while (inlen>=256){
+    Encrypt_Xor(out,in,nonce,ctx,256);
+    in+=256; inlen-=256; out+=256;
+  }
+
+  if (inlen>=192){
+    Encrypt_Xor(out,in,nonce,ctx,192);
+    in+=192; inlen-=192; out+=192;
+  }
+
+  if (inlen>=128){
+    Encrypt_Xor(out,in,nonce,ctx,128);
+    in+=128; inlen-=128; out+=128;
+  }
+
+  if (inlen>=64){
+    Encrypt_Xor(out,in,nonce,ctx,64);
+    in+=64; inlen-=64; out+=64;
+  }
+
+  if (inlen>=32){
+    Encrypt_Xor(out,in,nonce,ctx,32);
+    in+=32; inlen-=32; out+=32;
+  }
+
+  if (inlen>=16){
+    Encrypt_Xor(block,in,nonce,ctx,16);
+    ((u64 *)out)[0]=block64[0]^((u64 *)in)[0];
+    ((u64 *)out)[1]=block64[1]^((u64 *)in)[1];
+    in+=16; inlen-=16; out+=16;
+  }
+
+  if (inlen>0){
+    Encrypt_Xor(block,in,nonce,ctx,16);
+    for (i=0;i<inlen;i++) out[i]=block[i]^in[i];
+  }
+
+  return 0;
+}
+
+
+int speck_expand_key (const unsigned char *k, speck_context_t *ctx) {
+
+	u64 K[4];
+	size_t i;
+	for(i = 0; i < numkeywords; i++)
+		K[i] = ((u64 *)k)[i];
+
+	EK (K[0], K[1], K[2], K[3], ctx->rk, ctx->key);
+
+	return 0;
+}
+
+
+#elif defined (__SSE4_2__) // SSE support -------------------------------------------------
 
 #include <smmintrin.h>
 
@@ -274,7 +481,7 @@ typedef struct {
 } speck_context_t;
 
 
-int Encrypt_Xor(unsigned char *out, const unsigned char *in, u64 nonce[], speck_context_t *ctx, int numbytes)
+static int Encrypt_Xor(unsigned char *out, const unsigned char *in, u64 nonce[], speck_context_t *ctx, int numbytes)
 {
 
   u64  x[2],y[2];
@@ -288,7 +495,6 @@ int Encrypt_Xor(unsigned char *out, const unsigned char *in, u64 nonce[], speck_
     return 0;
   }
 
-
   SET1(X[0],nonce[1]); SET2(Y[0],nonce[0]);
 
   if (numbytes==32) Enc(X,Y,ctx->rk,2);
@@ -331,10 +537,9 @@ int speck_ctr (unsigned char *out, const unsigned char *in, unsigned long long i
 	       const unsigned char *n, speck_context_t *ctx) {
 
   int i;
-  u64 nonce[2],K[4],key[34],A,B,C,D,x,y;
+  u64 nonce[2];
   unsigned char block[16];
   u64 *const block64=(u64 *)block;
-  u128 rk[34];
 
   if (!inlen) return 0;
 
@@ -460,7 +665,7 @@ int speck_expand_key (const unsigned char *k, speck_context_t *ctx) {
 }
 
 
-#endif // SSE, NEON, plain C
+#endif // AVX, SSE, NEON, plain C
 
 
 int speck_test () {
@@ -487,10 +692,10 @@ int speck_test () {
 	speck_ctr (pt, pt, 16, iv, &ctx);
 
 	u64 i;
-//fprintf (stderr, "rk00: %016llx\n",  ctx.key[0]);
+// fprintf (stderr, "rk00: %016llx\n",  ctx.key[0]);
-//fprintf (stderr, "rk33: %016llx\n",  ctx.key[33]);
+// fprintf (stderr, "rk33: %016llx\n",  ctx.key[33]);
-//fprintf (stderr, "out : %016lx\n", *(uint64_t*)pt);
+// fprintf (stderr, "out : %016lx\n", *(uint64_t*)pt);
-//fprintf (stderr, "mem : " ); for (i=0; i < 16; i++) fprintf (stderr, "%02x ", pt[i]); fprintf (stderr, "\n");
+// fprintf (stderr, "mem : " ); for (i=0; i < 16; i++) fprintf (stderr, "%02x ", pt[i]); fprintf (stderr, "\n");
 
 	int ret = 1;
 	for (i=0; i < 16; i++)

speck.h

@@ -1,7 +1,16 @@
 
 #define u64 uint64_t
 
-#if defined (__SSE4_2__)
+#if defined (__AVX2__)
+ #define SPECK_ALIGNED_CTX	32
+ #include <immintrin.h>
+ #define u256 __m256i
+typedef struct {
+        u256 rk[34];
+        u64 key[34];
+} speck_context_t;
+#elif defined (__SSE4_2__)
+ #define SPECK_ALIGNED_CTX	16
  #include <immintrin.h>
  #define u128 __m128i
 typedef struct {

transform_speck.c

@@ -41,8 +41,11 @@ static int transop_deinit_speck(n2n_trans_op_t *arg) {
   transop_speck_t *priv = (transop_speck_t *)arg->priv;
 
   if(priv)
-    free(priv);
+#if defined (SPECK_ALIGNED_CTX)
-
+    _mm_free (priv);
+#else
+    free (priv);
+#endif
   return 0;
 }
 
@@ -163,7 +166,7 @@ static int setup_speck_key(transop_speck_t *priv, const uint8_t *key, ssize_t ke
   uint8_t key_mat_buf[32] = { 0x00 };
 
   /* Clear out any old possibly longer key matter. */
-  memset(&(priv->ctx), 0, sizeof(priv->ctx) );
+  memset(&(priv->ctx), 0, sizeof(speck_context_t) );
 
   /* TODO: The input key always gets hashed to make a more unpredictable and more complete use of the key space */
   // REVISIT: Hash the key to keymat (formerly used: SHA)
@@ -172,8 +175,8 @@ static int setup_speck_key(transop_speck_t *priv, const uint8_t *key, ssize_t ke
   // ADD: Pearson Hashing
   // FOR NOW: USE KEY ITSELF
   memcpy (key_mat_buf, key, ((key_size>32)?32:key_size) );
-
   speck_expand_key (key_mat_buf, &(priv->ctx));
+
   traceEvent(TRACE_DEBUG, "Speck key setup completed\n");
 
   return(0);
@@ -184,7 +187,6 @@ static int setup_speck_key(transop_speck_t *priv, const uint8_t *key, ssize_t ke
 static void transop_tick_speck(n2n_trans_op_t * arg, time_t now) { ; }
 
 /* ****************************************************** */
-
 /* Speck initialization function */
 int n2n_transop_speck_init(const n2n_edge_conf_t *conf, n2n_trans_op_t *ttt) {
   transop_speck_t *priv;
@@ -198,8 +200,11 @@ int n2n_transop_speck_init(const n2n_edge_conf_t *conf, n2n_trans_op_t *ttt) {
   ttt->deinit = transop_deinit_speck;
   ttt->fwd = transop_encode_speck;
   ttt->rev = transop_decode_speck;
-
+#if defined (SPECK_ALIGNED_CTX)
-  priv = (transop_speck_t*) calloc(1, sizeof(transop_speck_t));
+  priv = (transop_speck_t*) _mm_malloc (sizeof(transop_speck_t), SPECK_ALIGNED_CTX);
+#else
+  priv = (transop_speck_t*) calloc (1, sizeof(transop_speck_t));
+#endif
   if(!priv) {
     traceEvent(TRACE_ERROR, "cannot allocate transop_speck_t memory");
     return(-1);