FD.io VPP  v19.08.3-2-gbabecb413
Vector Packet Processing
aes_gcm.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2019 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <vlib/vlib.h>
19 #include <vnet/plugin/plugin.h>
20 #include <vnet/crypto/crypto.h>
21 #include <x86intrin.h>
23 #include <crypto_ia32/aesni.h>
24 #include <crypto_ia32/ghash.h>
25 
26 #if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
27 #pragma GCC optimize ("O3")
28 #endif
29 
30 typedef struct
31 {
32  /* pre-calculated hash key values */
33  const __m128i Hi[8];
34  /* extracted AES key */
35  const __m128i Ke[15];
37 
38 static const __m128i last_byte_one = { 0, 1ULL << 56 };
39 static const __m128i zero = { 0, 0 };
40 
41 static const u8x16 bswap_mask = {
42  15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
43 };
44 
45 static const u8x16 byte_mask_scale = {
46  0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
47 };
48 
50 aesni_gcm_bswap (__m128i x)
51 {
52  return _mm_shuffle_epi8 (x, (__m128i) bswap_mask);
53 }
54 
56 aesni_gcm_byte_mask (__m128i x, u8 n_bytes)
57 {
58  u8x16 mask = u8x16_is_greater (u8x16_splat (n_bytes), byte_mask_scale);
59 
60  return _mm_blendv_epi8 (zero, x, (__m128i) mask);
61 }
62 
64 aesni_gcm_load_partial (__m128i * p, int n_bytes)
65 {
66 #ifdef __AVX512F__
67  return _mm_mask_loadu_epi8 (zero, (1 << n_bytes) - 1, p);
68 #else
69  return aesni_gcm_byte_mask (_mm_loadu_si128 (p), n_bytes);
70 #endif
71 }
72 
74 aesni_gcm_store_partial (void *p, __m128i r, int n_bytes)
75 {
76 #ifdef __AVX512F__
77  _mm_mask_storeu_epi8 (p, (1 << n_bytes) - 1, r);
78 #else
79  u8x16 mask = u8x16_is_greater (u8x16_splat (n_bytes), byte_mask_scale);
80  _mm_maskmoveu_si128 (r, (__m128i) mask, p);
81 #endif
82 }
83 
85 aesni_gcm_load (__m128i * d, __m128i * inv, int n, int n_bytes)
86 {
87  for (int i = 0; i < n - 1; i++)
88  d[i] = _mm_loadu_si128 (inv + i);
89  d[n - 1] = n_bytes ? aesni_gcm_load_partial (inv + n - 1, n_bytes) :
90  _mm_loadu_si128 (inv + n - 1);
91 }
92 
94 aesni_gcm_store (__m128i * d, __m128i * outv, int n, int n_bytes)
95 {
96  for (int i = 0; i < n - 1; i++)
97  _mm_storeu_si128 (outv + i, d[i]);
98  if (n_bytes & 0xf)
99  aesni_gcm_store_partial (outv + n - 1, d[n - 1], n_bytes);
100  else
101  _mm_storeu_si128 (outv + n - 1, d[n - 1]);
102 }
103 
105 aesni_gcm_enc_first_round (__m128i * r, __m128i * Y, u32 * ctr, __m128i k,
106  int n_blocks)
107 {
108  u32 i;
109 
110  if (PREDICT_TRUE ((u8) ctr[0] < (256 - n_blocks)))
111  {
112  for (i = 0; i < n_blocks; i++)
113  {
114  Y[0] = _mm_add_epi32 (Y[0], last_byte_one);
115  r[i] = k ^ Y[0];
116  }
117  ctr[0] += n_blocks;
118  }
119  else
120  {
121  for (i = 0; i < n_blocks; i++)
122  {
123  Y[0] = _mm_insert_epi32 (Y[0], clib_host_to_net_u32 (++ctr[0]), 3);
124  r[i] = k ^ Y[0];
125  }
126  }
127 }
128 
130 aesni_gcm_enc_round (__m128i * r, __m128i k, int n_blocks)
131 {
132  for (int i = 0; i < n_blocks; i++)
133  r[i] = _mm_aesenc_si128 (r[i], k);
134 }
135 
137 aesni_gcm_enc_last_round (__m128i * r, __m128i * d, const __m128i * k,
138  int rounds, int n_blocks)
139 {
140 
141  /* additional ronuds for AES-192 and AES-256 */
142  for (int i = 10; i < rounds; i++)
143  aesni_gcm_enc_round (r, k[i], n_blocks);
144 
145  for (int i = 0; i < n_blocks; i++)
146  d[i] ^= _mm_aesenclast_si128 (r[i], k[rounds]);
147 }
148 
149 static_always_inline __m128i
151  const __m128i * in, int n_blocks)
152 {
153  ghash_data_t _gd, *gd = &_gd;
154  const __m128i *Hi = kd->Hi + n_blocks - 1;
155  ghash_mul_first (gd, aesni_gcm_bswap (_mm_loadu_si128 (in)) ^ T, Hi[0]);
156  for (int i = 1; i < n_blocks; i++)
157  ghash_mul_next (gd, aesni_gcm_bswap (_mm_loadu_si128 (in + i)), Hi[-i]);
158  ghash_reduce (gd);
159  ghash_reduce2 (gd);
160  return ghash_final (gd);
161 }
162 
163 static_always_inline __m128i
164 aesni_gcm_ghash (__m128i T, aes_gcm_key_data_t * kd, const __m128i * in,
165  u32 n_left)
166 {
167 
168  while (n_left >= 128)
169  {
170  T = aesni_gcm_ghash_blocks (T, kd, in, 8);
171  n_left -= 128;
172  in += 8;
173  }
174 
175  if (n_left >= 64)
176  {
177  T = aesni_gcm_ghash_blocks (T, kd, in, 4);
178  n_left -= 64;
179  in += 4;
180  }
181 
182  if (n_left >= 32)
183  {
184  T = aesni_gcm_ghash_blocks (T, kd, in, 2);
185  n_left -= 32;
186  in += 2;
187  }
188 
189  if (n_left >= 16)
190  {
191  T = aesni_gcm_ghash_blocks (T, kd, in, 1);
192  n_left -= 16;
193  in += 1;
194  }
195 
196  if (n_left)
197  {
198  __m128i r = aesni_gcm_load_partial ((__m128i *) in, n_left);
199  T = ghash_mul (aesni_gcm_bswap (r) ^ T, kd->Hi[0]);
200  }
201  return T;
202 }
203 
204 static_always_inline __m128i
205 aesni_gcm_calc (__m128i T, aes_gcm_key_data_t * kd, __m128i * d,
206  __m128i * Y, u32 * ctr, __m128i * inv, __m128i * outv,
207  int rounds, int n, int last_block_bytes, int with_ghash,
208  int is_encrypt)
209 {
210  __m128i r[n];
211  ghash_data_t _gd = { }, *gd = &_gd;
212  const __m128i *k = kd->Ke;
213  int hidx = is_encrypt ? 4 : n, didx = 0;
214 
215  _mm_prefetch (inv + 4, _MM_HINT_T0);
216 
217  /* AES rounds 0 and 1 */
218  aesni_gcm_enc_first_round (r, Y, ctr, k[0], n);
219  aesni_gcm_enc_round (r, k[1], n);
220 
221  /* load data - decrypt round */
222  if (is_encrypt == 0)
223  aesni_gcm_load (d, inv, n, last_block_bytes);
224 
225  /* GHASH multiply block 1 */
226  if (with_ghash)
227  ghash_mul_first (gd, aesni_gcm_bswap (d[didx++]) ^ T, kd->Hi[--hidx]);
228 
229  /* AES rounds 2 and 3 */
230  aesni_gcm_enc_round (r, k[2], n);
231  aesni_gcm_enc_round (r, k[3], n);
232 
233  /* GHASH multiply block 2 */
234  if (with_ghash && hidx)
235  ghash_mul_next (gd, aesni_gcm_bswap (d[didx++]), kd->Hi[--hidx]);
236 
237  /* AES rounds 4 and 5 */
238  aesni_gcm_enc_round (r, k[4], n);
239  aesni_gcm_enc_round (r, k[5], n);
240 
241  /* GHASH multiply block 3 */
242  if (with_ghash && hidx)
243  ghash_mul_next (gd, aesni_gcm_bswap (d[didx++]), kd->Hi[--hidx]);
244 
245  /* AES rounds 6 and 7 */
246  aesni_gcm_enc_round (r, k[6], n);
247  aesni_gcm_enc_round (r, k[7], n);
248 
249  /* GHASH multiply block 4 */
250  if (with_ghash && hidx)
251  ghash_mul_next (gd, aesni_gcm_bswap (d[didx++]), kd->Hi[--hidx]);
252 
253  /* AES rounds 8 and 9 */
254  aesni_gcm_enc_round (r, k[8], n);
255  aesni_gcm_enc_round (r, k[9], n);
256 
257  /* GHASH reduce 1st step */
258  if (with_ghash)
259  ghash_reduce (gd);
260 
261  /* load data - encrypt round */
262  if (is_encrypt)
263  aesni_gcm_load (d, inv, n, last_block_bytes);
264 
265  /* GHASH reduce 2nd step */
266  if (with_ghash)
267  ghash_reduce2 (gd);
268 
269  /* AES last round(s) */
270  aesni_gcm_enc_last_round (r, d, k, rounds, n);
271 
272  /* store data */
273  aesni_gcm_store (d, outv, n, last_block_bytes);
274 
275  /* GHASH final step */
276  if (with_ghash)
277  T = ghash_final (gd);
278 
279  return T;
280 }
281 
282 static_always_inline __m128i
283 aesni_gcm_calc_double (__m128i T, aes_gcm_key_data_t * kd, __m128i * d,
284  __m128i * Y, u32 * ctr, __m128i * inv, __m128i * outv,
285  int rounds, int is_encrypt)
286 {
287  __m128i r[4];
288  ghash_data_t _gd, *gd = &_gd;
289  const __m128i *k = kd->Ke;
290 
291  /* AES rounds 0 and 1 */
292  aesni_gcm_enc_first_round (r, Y, ctr, k[0], 4);
293  aesni_gcm_enc_round (r, k[1], 4);
294 
295  /* load 4 blocks of data - decrypt round */
296  if (is_encrypt == 0)
297  aesni_gcm_load (d, inv, 4, 0);
298 
299  /* GHASH multiply block 0 */
300  ghash_mul_first (gd, aesni_gcm_bswap (d[0]) ^ T, kd->Hi[7]);
301 
302  /* AES rounds 2 and 3 */
303  aesni_gcm_enc_round (r, k[2], 4);
304  aesni_gcm_enc_round (r, k[3], 4);
305 
306  /* GHASH multiply block 1 */
307  ghash_mul_next (gd, aesni_gcm_bswap (d[1]), kd->Hi[6]);
308 
309  /* AES rounds 4 and 5 */
310  aesni_gcm_enc_round (r, k[4], 4);
311  aesni_gcm_enc_round (r, k[5], 4);
312 
313  /* GHASH multiply block 2 */
314  ghash_mul_next (gd, aesni_gcm_bswap (d[2]), kd->Hi[5]);
315 
316  /* AES rounds 6 and 7 */
317  aesni_gcm_enc_round (r, k[6], 4);
318  aesni_gcm_enc_round (r, k[7], 4);
319 
320  /* GHASH multiply block 3 */
321  ghash_mul_next (gd, aesni_gcm_bswap (d[3]), kd->Hi[4]);
322 
323  /* AES rounds 8 and 9 */
324  aesni_gcm_enc_round (r, k[8], 4);
325  aesni_gcm_enc_round (r, k[9], 4);
326 
327  /* load 4 blocks of data - encrypt round */
328  if (is_encrypt)
329  aesni_gcm_load (d, inv, 4, 0);
330 
331  /* AES last round(s) */
332  aesni_gcm_enc_last_round (r, d, k, rounds, 4);
333 
334  /* store 4 blocks of data */
335  aesni_gcm_store (d, outv, 4, 0);
336 
337  /* load next 4 blocks of data data - decrypt round */
338  if (is_encrypt == 0)
339  aesni_gcm_load (d, inv + 4, 4, 0);
340 
341  /* GHASH multiply block 4 */
342  ghash_mul_next (gd, aesni_gcm_bswap (d[0]), kd->Hi[3]);
343 
344  /* AES rounds 0, 1 and 2 */
345  aesni_gcm_enc_first_round (r, Y, ctr, k[0], 4);
346  aesni_gcm_enc_round (r, k[1], 4);
347  aesni_gcm_enc_round (r, k[2], 4);
348 
349  /* GHASH multiply block 5 */
350  ghash_mul_next (gd, aesni_gcm_bswap (d[1]), kd->Hi[2]);
351 
352  /* AES rounds 3 and 4 */
353  aesni_gcm_enc_round (r, k[3], 4);
354  aesni_gcm_enc_round (r, k[4], 4);
355 
356  /* GHASH multiply block 6 */
357  ghash_mul_next (gd, aesni_gcm_bswap (d[2]), kd->Hi[1]);
358 
359  /* AES rounds 5 and 6 */
360  aesni_gcm_enc_round (r, k[5], 4);
361  aesni_gcm_enc_round (r, k[6], 4);
362 
363  /* GHASH multiply block 7 */
364  ghash_mul_next (gd, aesni_gcm_bswap (d[3]), kd->Hi[0]);
365 
366  /* AES rounds 7 and 8 */
367  aesni_gcm_enc_round (r, k[7], 4);
368  aesni_gcm_enc_round (r, k[8], 4);
369 
370  /* GHASH reduce 1st step */
371  ghash_reduce (gd);
372 
373  /* AES round 9 */
374  aesni_gcm_enc_round (r, k[9], 4);
375 
376  /* load data - encrypt round */
377  if (is_encrypt)
378  aesni_gcm_load (d, inv + 4, 4, 0);
379 
380  /* GHASH reduce 2nd step */
381  ghash_reduce2 (gd);
382 
383  /* AES last round(s) */
384  aesni_gcm_enc_last_round (r, d, k, rounds, 4);
385 
386  /* store data */
387  aesni_gcm_store (d, outv + 4, 4, 0);
388 
389  /* GHASH final step */
390  return ghash_final (gd);
391 }
392 
393 static_always_inline __m128i
394 aesni_gcm_ghash_last (__m128i T, aes_gcm_key_data_t * kd, __m128i * d,
395  int n_blocks, int n_bytes)
396 {
397  ghash_data_t _gd, *gd = &_gd;
398 
399  if (n_bytes)
400  d[n_blocks - 1] = aesni_gcm_byte_mask (d[n_blocks - 1], n_bytes);
401 
402  ghash_mul_first (gd, aesni_gcm_bswap (d[0]) ^ T, kd->Hi[n_blocks - 1]);
403  if (n_blocks > 1)
404  ghash_mul_next (gd, aesni_gcm_bswap (d[1]), kd->Hi[n_blocks - 2]);
405  if (n_blocks > 2)
406  ghash_mul_next (gd, aesni_gcm_bswap (d[2]), kd->Hi[n_blocks - 3]);
407  if (n_blocks > 3)
408  ghash_mul_next (gd, aesni_gcm_bswap (d[3]), kd->Hi[n_blocks - 4]);
409  ghash_reduce (gd);
410  ghash_reduce2 (gd);
411  return ghash_final (gd);
412 }
413 
414 
415 static_always_inline __m128i
416 aesni_gcm_enc (__m128i T, aes_gcm_key_data_t * kd, __m128i Y, const u8 * in,
417  const u8 * out, u32 n_left, int rounds)
418 {
419  __m128i *inv = (__m128i *) in, *outv = (__m128i *) out;
420  __m128i d[4];
421  u32 ctr = 1;
422 
423  if (n_left == 0)
424  return T;
425 
426  if (n_left < 64)
427  {
428  if (n_left > 48)
429  {
430  n_left &= 0x0f;
431  aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 4, n_left,
432  /* with_ghash */ 0, /* is_encrypt */ 1);
433  return aesni_gcm_ghash_last (T, kd, d, 4, n_left);
434  }
435  else if (n_left > 32)
436  {
437  n_left &= 0x0f;
438  aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 3, n_left,
439  /* with_ghash */ 0, /* is_encrypt */ 1);
440  return aesni_gcm_ghash_last (T, kd, d, 3, n_left);
441  }
442  else if (n_left > 16)
443  {
444  n_left &= 0x0f;
445  aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 2, n_left,
446  /* with_ghash */ 0, /* is_encrypt */ 1);
447  return aesni_gcm_ghash_last (T, kd, d, 2, n_left);
448  }
449  else
450  {
451  n_left &= 0x0f;
452  aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 1, n_left,
453  /* with_ghash */ 0, /* is_encrypt */ 1);
454  return aesni_gcm_ghash_last (T, kd, d, 1, n_left);
455  }
456  }
457 
458  aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 4, 0,
459  /* with_ghash */ 0, /* is_encrypt */ 1);
460 
461  /* next */
462  n_left -= 64;
463  outv += 4;
464  inv += 4;
465 
466  while (n_left >= 128)
467  {
468  T = aesni_gcm_calc_double (T, kd, d, &Y, &ctr, inv, outv, rounds,
469  /* is_encrypt */ 1);
470 
471  /* next */
472  n_left -= 128;
473  outv += 8;
474  inv += 8;
475  }
476 
477  if (n_left >= 64)
478  {
479  T = aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 4, 0,
480  /* with_ghash */ 1, /* is_encrypt */ 1);
481 
482  /* next */
483  n_left -= 64;
484  outv += 4;
485  inv += 4;
486  }
487 
488  if (n_left == 0)
489  return aesni_gcm_ghash_last (T, kd, d, 4, 0);
490 
491  if (n_left > 48)
492  {
493  n_left &= 0x0f;
494  T = aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 4, n_left,
495  /* with_ghash */ 1, /* is_encrypt */ 1);
496  return aesni_gcm_ghash_last (T, kd, d, 4, n_left);
497  }
498 
499  if (n_left > 32)
500  {
501  n_left &= 0x0f;
502  T = aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 3, n_left,
503  /* with_ghash */ 1, /* is_encrypt */ 1);
504  return aesni_gcm_ghash_last (T, kd, d, 3, n_left);
505  }
506 
507  if (n_left > 16)
508  {
509  n_left &= 0x0f;
510  T = aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 2, n_left,
511  /* with_ghash */ 1, /* is_encrypt */ 1);
512  return aesni_gcm_ghash_last (T, kd, d, 2, n_left);
513  }
514 
515  n_left &= 0x0f;
516  T = aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 1, n_left,
517  /* with_ghash */ 1, /* is_encrypt */ 1);
518  return aesni_gcm_ghash_last (T, kd, d, 1, n_left);
519 }
520 
521 static_always_inline __m128i
522 aesni_gcm_dec (__m128i T, aes_gcm_key_data_t * kd, __m128i Y, const u8 * in,
523  const u8 * out, u32 n_left, int rounds)
524 {
525  __m128i *inv = (__m128i *) in, *outv = (__m128i *) out;
526  __m128i d[8];
527  u32 ctr = 1;
528 
529  while (n_left >= 128)
530  {
531  T = aesni_gcm_calc_double (T, kd, d, &Y, &ctr, inv, outv, rounds,
532  /* is_encrypt */ 0);
533 
534  /* next */
535  n_left -= 128;
536  outv += 8;
537  inv += 8;
538  }
539 
540  if (n_left >= 64)
541  {
542  T = aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 4, 0, 1, 0);
543 
544  /* next */
545  n_left -= 64;
546  outv += 4;
547  inv += 4;
548  }
549 
550  if (n_left == 0)
551  return T;
552 
553  if (n_left > 48)
554  return aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 4,
555  n_left - 48,
556  /* with_ghash */ 1, /* is_encrypt */ 0);
557 
558  if (n_left > 32)
559  return aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 3,
560  n_left - 32,
561  /* with_ghash */ 1, /* is_encrypt */ 0);
562 
563  if (n_left > 16)
564  return aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 2,
565  n_left - 16,
566  /* with_ghash */ 1, /* is_encrypt */ 0);
567 
568  return aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 1, n_left,
569  /* with_ghash */ 1, /* is_encrypt */ 0);
570 }
571 
573 aes_gcm (const u8 * in, u8 * out, const u8 * addt, const u8 * iv, u8 * tag,
574  u32 data_bytes, u32 aad_bytes, u8 tag_len, aes_gcm_key_data_t * kd,
575  int aes_rounds, int is_encrypt)
576 {
577  int i;
578  __m128i r, Y0, T = { };
579  ghash_data_t _gd, *gd = &_gd;
580 
581  _mm_prefetch (iv, _MM_HINT_T0);
582  _mm_prefetch (in, _MM_HINT_T0);
583  _mm_prefetch (in + CLIB_CACHE_LINE_BYTES, _MM_HINT_T0);
584 
585  /* calculate ghash for AAD - optimized for ipsec common cases */
586  if (aad_bytes == 8)
587  T = aesni_gcm_ghash (T, kd, (__m128i *) addt, 8);
588  else if (aad_bytes == 12)
589  T = aesni_gcm_ghash (T, kd, (__m128i *) addt, 12);
590  else
591  T = aesni_gcm_ghash (T, kd, (__m128i *) addt, aad_bytes);
592 
593  /* initalize counter */
594  Y0 = _mm_loadu_si128 ((__m128i *) iv);
595  Y0 = _mm_insert_epi32 (Y0, clib_host_to_net_u32 (1), 3);
596 
597  /* ghash and encrypt/edcrypt */
598  if (is_encrypt)
599  T = aesni_gcm_enc (T, kd, Y0, in, out, data_bytes, aes_rounds);
600  else
601  T = aesni_gcm_dec (T, kd, Y0, in, out, data_bytes, aes_rounds);
602 
603  _mm_prefetch (tag, _MM_HINT_T0);
604 
605  /* Finalize ghash */
606  r[0] = data_bytes;
607  r[1] = aad_bytes;
608 
609  /* bytes to bits */
610  r <<= 3;
611 
612  /* interleaved computation of final ghash and E(Y0, k) */
613  ghash_mul_first (gd, r ^ T, kd->Hi[0]);
614  r = kd->Ke[0] ^ Y0;
615  for (i = 1; i < 5; i += 1)
616  r = _mm_aesenc_si128 (r, kd->Ke[i]);
617  ghash_reduce (gd);
618  ghash_reduce2 (gd);
619  for (; i < 9; i += 1)
620  r = _mm_aesenc_si128 (r, kd->Ke[i]);
621  T = ghash_final (gd);
622  for (; i < aes_rounds; i += 1)
623  r = _mm_aesenc_si128 (r, kd->Ke[i]);
624  r = _mm_aesenclast_si128 (r, kd->Ke[aes_rounds]);
625  T = aesni_gcm_bswap (T) ^ r;
626 
627  /* tag_len 16 -> 0 */
628  tag_len &= 0xf;
629 
630  if (is_encrypt)
631  {
632  /* store tag */
633  if (tag_len)
634  aesni_gcm_store_partial ((__m128i *) tag, T, (1 << tag_len) - 1);
635  else
636  _mm_storeu_si128 ((__m128i *) tag, T);
637  }
638  else
639  {
640  /* check tag */
641  u16 tag_mask = tag_len ? (1 << tag_len) - 1 : 0xffff;
642  r = _mm_loadu_si128 ((__m128i *) tag);
643  if (_mm_movemask_epi8 (r == T) != tag_mask)
644  return 0;
645  }
646  return 1;
647 }
648 
651  u32 n_ops, aesni_key_size_t ks)
652 {
654  vnet_crypto_op_t *op = ops[0];
655  aes_gcm_key_data_t *kd;
656  u32 n_left = n_ops;
657 
658 
659 next:
660  kd = (aes_gcm_key_data_t *) cm->key_data[op->key_index];
661  aes_gcm (op->src, op->dst, op->aad, op->iv, op->tag, op->len, op->aad_len,
662  op->tag_len, kd, AESNI_KEY_ROUNDS (ks), /* is_encrypt */ 1);
663  op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
664 
665  if (--n_left)
666  {
667  op += 1;
668  goto next;
669  }
670 
671  return n_ops;
672 }
673 
676  u32 n_ops, aesni_key_size_t ks)
677 {
679  vnet_crypto_op_t *op = ops[0];
680  aes_gcm_key_data_t *kd;
681  u32 n_left = n_ops;
682  int rv;
683 
684 next:
685  kd = (aes_gcm_key_data_t *) cm->key_data[op->key_index];
686  rv = aes_gcm (op->src, op->dst, op->aad, op->iv, op->tag, op->len,
687  op->aad_len, op->tag_len, kd, AESNI_KEY_ROUNDS (ks),
688  /* is_encrypt */ 0);
689 
690  if (rv)
691  {
692  op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
693  }
694  else
695  {
696  op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC;
697  n_ops--;
698  }
699 
700  if (--n_left)
701  {
702  op += 1;
703  goto next;
704  }
705 
706  return n_ops;
707 }
708 
711 {
712  aes_gcm_key_data_t *kd;
713  __m128i H;
714  int i;
715 
716  kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES);
717 
718  /* expand AES key */
719  aes_key_expand ((__m128i *) kd->Ke, key->data, ks);
720 
721  /* pre-calculate H */
722  H = kd->Ke[0];
723  for (i = 1; i < AESNI_KEY_ROUNDS (ks); i += 1)
724  H = _mm_aesenc_si128 (H, kd->Ke[i]);
725  H = _mm_aesenclast_si128 (H, kd->Ke[i]);
726  H = aesni_gcm_bswap (H);
727  ghash_precompute (H, (__m128i *) kd->Hi, 8);
728  return kd;
729 }
730 
731 #define foreach_aesni_gcm_handler_type _(128) _(192) _(256)
732 
733 #define _(x) \
734 static u32 aesni_ops_dec_aes_gcm_##x \
735 (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \
736 { return aesni_ops_dec_aes_gcm (vm, ops, n_ops, AESNI_KEY_##x); } \
737 static u32 aesni_ops_enc_aes_gcm_##x \
738 (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \
739 { return aesni_ops_enc_aes_gcm (vm, ops, n_ops, AESNI_KEY_##x); } \
740 static void * aesni_gcm_key_exp_##x (vnet_crypto_key_t *key) \
741 { return aesni_gcm_key_exp (key, AESNI_KEY_##x); }
742 
744 #undef _
745 
746 clib_error_t *
747 #ifdef __AVX512F__
749 #elif __AVX2__
751 #else
753 #endif
754 {
756 
757 #define _(x) \
758  vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \
759  VNET_CRYPTO_OP_AES_##x##_GCM_ENC, \
760  aesni_ops_enc_aes_gcm_##x); \
761  vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \
762  VNET_CRYPTO_OP_AES_##x##_GCM_DEC, \
763  aesni_ops_dec_aes_gcm_##x); \
764  cm->key_fn[VNET_CRYPTO_ALG_AES_##x##_GCM] = aesni_gcm_key_exp_##x;
766 #undef _
767  return 0;
768 }
769 
770 /*
771  * fd.io coding-style-patch-verification: ON
772  *
773  * Local Variables:
774  * eval: (c-set-style "gnu")
775  * End:
776  */
crypto_ia32_main_t crypto_ia32_main
Definition: main.c:23
static_always_inline __m128i aesni_gcm_ghash(__m128i T, aes_gcm_key_data_t *kd, const __m128i *in, u32 n_left)
Definition: aes_gcm.c:164
static_always_inline void aesni_gcm_store_partial(void *p, __m128i r, int n_bytes)
Definition: aes_gcm.c:74
static const __m128i last_byte_one
Definition: aes_gcm.c:38
#define PREDICT_TRUE(x)
Definition: clib.h:113
static_always_inline void aesni_gcm_load(__m128i *d, __m128i *inv, int n, int n_bytes)
Definition: aes_gcm.c:85
static_always_inline void aesni_gcm_store(__m128i *d, __m128i *outv, int n, int n_bytes)
Definition: aes_gcm.c:94
static_always_inline void aesni_gcm_enc_round(__m128i *r, __m128i k, int n_blocks)
Definition: aes_gcm.c:130
static_always_inline __m128i aesni_gcm_calc(__m128i T, aes_gcm_key_data_t *kd, __m128i *d, __m128i *Y, u32 *ctr, __m128i *inv, __m128i *outv, int rounds, int n, int last_block_bytes, int with_ghash, int is_encrypt)
Definition: aes_gcm.c:205
int i
clib_error_t * crypto_ia32_aesni_gcm_init_sse42(vlib_main_t *vm)
Definition: aes_gcm.c:752
#define AESNI_KEY_ROUNDS(x)
Definition: aesni.h:29
static const u8x16 bswap_mask
Definition: aes_gcm.c:41
static_always_inline void aes_key_expand(__m128i *k, u8 *key, aesni_key_size_t ks)
Definition: aesni.h:181
unsigned char u8
Definition: types.h:56
static_always_inline void ghash_precompute(__m128i H, __m128i *Hi, int count)
Definition: ghash.h:226
const __m128i Hi[8]
Definition: aes_gcm.c:33
#define static_always_inline
Definition: clib.h:100
static const u8x16 byte_mask_scale
Definition: aes_gcm.c:45
aesni_key_size_t
Definition: aesni.h:22
static_always_inline u32 aesni_ops_enc_aes_gcm(vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops, aesni_key_size_t ks)
Definition: aes_gcm.c:650
static_always_inline void ghash_reduce(ghash_data_t *gd)
Definition: ghash.h:176
static_always_inline int aes_gcm(const u8 *in, u8 *out, const u8 *addt, const u8 *iv, u8 *tag, u32 data_bytes, u32 aad_bytes, u8 tag_len, aes_gcm_key_data_t *kd, int aes_rounds, int is_encrypt)
Definition: aes_gcm.c:573
unsigned int u32
Definition: types.h:88
static_always_inline void ghash_reduce2(ghash_data_t *gd)
Definition: ghash.h:202
static_always_inline __m128i aesni_gcm_ghash_blocks(__m128i T, aes_gcm_key_data_t *kd, const __m128i *in, int n_blocks)
Definition: aes_gcm.c:150
vnet_crypto_main_t * cm
Definition: quic_crypto.c:41
static u8 iv[]
Definition: aes_cbc.c:24
const __m128i Ke[15]
Definition: aes_gcm.c:35
static const __m128i zero
Definition: aes_gcm.c:39
unsigned short u16
Definition: types.h:57
static_always_inline __m128i aesni_gcm_load_partial(__m128i *p, int n_bytes)
Definition: aes_gcm.c:64
clib_error_t * crypto_ia32_aesni_gcm_init_avx512(vlib_main_t *vm)
static_always_inline __m128i aesni_gcm_bswap(__m128i x)
Definition: aes_gcm.c:50
static_always_inline __m128i ghash_final(ghash_data_t *gd)
Definition: ghash.h:209
static_always_inline __m128i aesni_gcm_calc_double(__m128i T, aes_gcm_key_data_t *kd, __m128i *d, __m128i *Y, u32 *ctr, __m128i *inv, __m128i *outv, int rounds, int is_encrypt)
Definition: aes_gcm.c:283
static_always_inline __m128i aesni_gcm_enc(__m128i T, aes_gcm_key_data_t *kd, __m128i Y, const u8 *in, const u8 *out, u32 n_left, int rounds)
Definition: aes_gcm.c:416
static_always_inline void aesni_gcm_enc_last_round(__m128i *r, __m128i *d, const __m128i *k, int rounds, int n_blocks)
Definition: aes_gcm.c:137
static_always_inline __m128i aesni_gcm_byte_mask(__m128i x, u8 n_bytes)
Definition: aes_gcm.c:56
static_always_inline u8x16 u8x16_is_greater(u8x16 v1, u8x16 v2)
Definition: vector_sse42.h:729
static_always_inline __m128i aesni_gcm_dec(__m128i T, aes_gcm_key_data_t *kd, __m128i Y, const u8 *in, const u8 *out, u32 n_left, int rounds)
Definition: aes_gcm.c:522
typedef key
Definition: ipsec.api:247
clib_error_t * crypto_ia32_aesni_gcm_init_avx2(vlib_main_t *vm)
static_always_inline u32 aesni_ops_dec_aes_gcm(vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops, aesni_key_size_t ks)
Definition: aes_gcm.c:675
#define foreach_aesni_gcm_handler_type
Definition: aes_gcm.c:731
static_always_inline void * aesni_gcm_key_exp(vnet_crypto_key_t *key, aesni_key_size_t ks)
Definition: aes_gcm.c:710
static_always_inline void aesni_gcm_enc_first_round(__m128i *r, __m128i *Y, u32 *ctr, __m128i k, int n_blocks)
Definition: aes_gcm.c:105
vnet_crypto_op_status_t status
Definition: crypto.h:129
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:161
static_always_inline __m128i ghash_mul(__m128i a, __m128i b)
Definition: ghash.h:216
static_always_inline __m128i aesni_gcm_ghash_last(__m128i T, aes_gcm_key_data_t *kd, __m128i *d, int n_blocks, int n_bytes)
Definition: aes_gcm.c:394
static_always_inline void ghash_mul_next(ghash_data_t *gd, __m128i a, __m128i b)
Definition: ghash.h:145
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
static_always_inline void ghash_mul_first(ghash_data_t *gd, __m128i a, __m128i b)
Definition: ghash.h:129