FD.io VPP  v21.06-3-gbb25fbf28
Vector Packet Processing
aes_cbc.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2019 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <vlib/vlib.h>
19 #include <vnet/plugin/plugin.h>
20 #include <vnet/crypto/crypto.h>
21 #include <crypto_native/crypto_native.h>
22 #include <crypto_native/aes.h>
23 
24 #if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
25 #pragma GCC optimize ("O3")
26 #endif
27 
28 typedef struct
29 {
30  u8x16 encrypt_key[15];
31 #if __VAES__
32  u8x64 decrypt_key[15];
33 #else
34  u8x16 decrypt_key[15];
35 #endif
36 } aes_cbc_key_data_t;
37 
38 
39 static_always_inline void __clib_unused
40 aes_cbc_dec (u8x16 * k, u8x16u * src, u8x16u * dst, u8x16u * iv, int count,
41  int rounds)
42 {
43  u8x16 r[4], c[4], f;
44 
45  f = iv[0];
46  while (count >= 64)
47  {
48  clib_prefetch_load (src + 8);
49  clib_prefetch_load (dst + 8);
50 
51  c[0] = r[0] = src[0];
52  c[1] = r[1] = src[1];
53  c[2] = r[2] = src[2];
54  c[3] = r[3] = src[3];
55 
56 #if __x86_64__
57  r[0] ^= k[0];
58  r[1] ^= k[0];
59  r[2] ^= k[0];
60  r[3] ^= k[0];
61 
62  for (int i = 1; i < rounds; i++)
63  {
64  r[0] = aes_dec_round (r[0], k[i]);
65  r[1] = aes_dec_round (r[1], k[i]);
66  r[2] = aes_dec_round (r[2], k[i]);
67  r[3] = aes_dec_round (r[3], k[i]);
68  }
69 
70  r[0] = aes_dec_last_round (r[0], k[rounds]);
71  r[1] = aes_dec_last_round (r[1], k[rounds]);
72  r[2] = aes_dec_last_round (r[2], k[rounds]);
73  r[3] = aes_dec_last_round (r[3], k[rounds]);
74 #else
75  for (int i = 0; i < rounds - 1; i++)
76  {
77  r[0] = vaesimcq_u8 (vaesdq_u8 (r[0], k[i]));
78  r[1] = vaesimcq_u8 (vaesdq_u8 (r[1], k[i]));
79  r[2] = vaesimcq_u8 (vaesdq_u8 (r[2], k[i]));
80  r[3] = vaesimcq_u8 (vaesdq_u8 (r[3], k[i]));
81  }
82  r[0] = vaesdq_u8 (r[0], k[rounds - 1]) ^ k[rounds];
83  r[1] = vaesdq_u8 (r[1], k[rounds - 1]) ^ k[rounds];
84  r[2] = vaesdq_u8 (r[2], k[rounds - 1]) ^ k[rounds];
85  r[3] = vaesdq_u8 (r[3], k[rounds - 1]) ^ k[rounds];
86 #endif
87  dst[0] = r[0] ^ f;
88  dst[1] = r[1] ^ c[0];
89  dst[2] = r[2] ^ c[1];
90  dst[3] = r[3] ^ c[2];
91  f = c[3];
92 
93  count -= 64;
94  src += 4;
95  dst += 4;
96  }
97 
98  while (count > 0)
99  {
100  c[0] = r[0] = src[0];
101 #if __x86_64__
102  r[0] ^= k[0];
103  for (int i = 1; i < rounds; i++)
104  r[0] = aes_dec_round (r[0], k[i]);
105  r[0] = aes_dec_last_round (r[0], k[rounds]);
106 #else
107  c[0] = r[0] = src[0];
108  for (int i = 0; i < rounds - 1; i++)
109  r[0] = vaesimcq_u8 (vaesdq_u8 (r[0], k[i]));
110  r[0] = vaesdq_u8 (r[0], k[rounds - 1]) ^ k[rounds];
111 #endif
112  dst[0] = r[0] ^ f;
113  f = c[0];
114 
115  count -= 16;
116  src += 1;
117  dst += 1;
118  }
119 }
120 
121 #if __x86_64__
122 #ifdef __VAES__
123 
124 static_always_inline u8x64
125 aes_block_load_x4 (u8 * src[], int i)
126 {
127  u8x64 r = { };
128  r = u8x64_insert_u8x16 (r, aes_block_load (src[0] + i), 0);
129  r = u8x64_insert_u8x16 (r, aes_block_load (src[1] + i), 1);
130  r = u8x64_insert_u8x16 (r, aes_block_load (src[2] + i), 2);
131  r = u8x64_insert_u8x16 (r, aes_block_load (src[3] + i), 3);
132  return r;
133 }
134 
135 static_always_inline void
136 aes_block_store_x4 (u8 * dst[], int i, u8x64 r)
137 {
138  aes_block_store (dst[0] + i, u8x64_extract_u8x16 (r, 0));
139  aes_block_store (dst[1] + i, u8x64_extract_u8x16 (r, 1));
140  aes_block_store (dst[2] + i, u8x64_extract_u8x16 (r, 2));
141  aes_block_store (dst[3] + i, u8x64_extract_u8x16 (r, 3));
142 }
143 
144 static_always_inline u8x64
145 aes_cbc_dec_permute (u8x64 a, u8x64 b)
146 {
147  __m512i perm = { 6, 7, 8, 9, 10, 11, 12, 13 };
148  return (u8x64) _mm512_permutex2var_epi64 ((__m512i) a, perm, (__m512i) b);
149 }
150 
151 static_always_inline void
152 vaes_cbc_dec (u8x64 * k, u8x64u * src, u8x64u * dst, u8x16 * iv, int count,
153  aes_key_size_t rounds)
154 {
155  u8x64 f, r[4], c[4] = { };
156  __mmask8 m;
157  int i, n_blocks = count >> 4;
158 
159  f = (u8x64) _mm512_mask_loadu_epi64 (_mm512_setzero_si512 (), 0xc0,
160  (__m512i *) (iv - 3));
161 
162  while (n_blocks >= 16)
163  {
164  c[0] = src[0];
165  c[1] = src[1];
166  c[2] = src[2];
167  c[3] = src[3];
168 
169  r[0] = c[0] ^ k[0];
170  r[1] = c[1] ^ k[0];
171  r[2] = c[2] ^ k[0];
172  r[3] = c[3] ^ k[0];
173 
174  for (i = 1; i < rounds; i++)
175  {
176  r[0] = aes_dec_round_x4 (r[0], k[i]);
177  r[1] = aes_dec_round_x4 (r[1], k[i]);
178  r[2] = aes_dec_round_x4 (r[2], k[i]);
179  r[3] = aes_dec_round_x4 (r[3], k[i]);
180  }
181 
182  r[0] = aes_dec_last_round_x4 (r[0], k[i]);
183  r[1] = aes_dec_last_round_x4 (r[1], k[i]);
184  r[2] = aes_dec_last_round_x4 (r[2], k[i]);
185  r[3] = aes_dec_last_round_x4 (r[3], k[i]);
186 
187  dst[0] = r[0] ^= aes_cbc_dec_permute (f, c[0]);
188  dst[1] = r[1] ^= aes_cbc_dec_permute (c[0], c[1]);
189  dst[2] = r[2] ^= aes_cbc_dec_permute (c[1], c[2]);
190  dst[4] = r[3] ^= aes_cbc_dec_permute (c[2], c[3]);
191  f = c[3];
192 
193  n_blocks -= 16;
194  src += 4;
195  dst += 4;
196  }
197 
198  while (n_blocks > 0)
199  {
200  m = (1 << (n_blocks * 2)) - 1;
201  c[0] = (u8x64) _mm512_mask_loadu_epi64 ((__m512i) c[0], m,
202  (__m512i *) src);
203  f = aes_cbc_dec_permute (f, c[0]);
204  r[0] = c[0] ^ k[0];
205  for (i = 1; i < rounds; i++)
206  r[0] = aes_dec_round_x4 (r[0], k[i]);
207  r[0] = aes_dec_last_round_x4 (r[0], k[i]);
208  _mm512_mask_storeu_epi64 ((__m512i *) dst, m, (__m512i) (r[0] ^ f));
209  f = c[0];
210  n_blocks -= 4;
211  src += 1;
212  dst += 1;
213  }
214 }
215 #endif
216 #endif
217 
218 #ifdef __VAES__
219 #define N 16
220 #define u32xN u32x16
221 #define u32xN_min_scalar u32x16_min_scalar
222 #define u32xN_is_all_zero u32x16_is_all_zero
223 #define u32xN_splat u32x16_splat
224 #else
225 #define N 4
226 #define u32xN u32x4
227 #define u32xN_min_scalar u32x4_min_scalar
228 #define u32xN_is_all_zero u32x4_is_all_zero
229 #define u32xN_splat u32x4_splat
230 #endif
231 
232 static_always_inline u32
233 aes_ops_enc_aes_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[],
234  u32 n_ops, aes_key_size_t ks)
235 {
236  crypto_native_main_t *cm = &crypto_native_main;
237  crypto_native_per_thread_data_t *ptd =
238  vec_elt_at_index (cm->per_thread_data, vm->thread_index);
239  int rounds = AES_KEY_ROUNDS (ks);
240  u8 placeholder[8192];
241  u32 i, j, count, n_left = n_ops;
242  u32xN placeholder_mask = { };
243  u32xN len = { };
244  vnet_crypto_key_index_t key_index[N];
245  u8 *src[N] = { };
246  u8 *dst[N] = { };
247 #if __VAES__
248  u8x64 r[N / 4] = { };
249  u8x64 k[15][N / 4] = { };
250  u8x16 *kq, *rq = (u8x16 *) r;
251 #else
252  u8x16 r[N] = { };
253  u8x16 k[15][N] = { };
254 #endif
255 
256  for (i = 0; i < N; i++)
257  key_index[i] = ~0;
258 
259 more:
260  for (i = 0; i < N; i++)
261  if (len[i] == 0)
262  {
263  if (n_left == 0)
264  {
265  /* no more work to enqueue, so we are enqueueing placeholder buffer */
266  src[i] = dst[i] = placeholder;
267  len[i] = sizeof (placeholder);
268  placeholder_mask[i] = 0;
269  }
270  else
271  {
272  u8x16 t;
273  if (ops[0]->flags & VNET_CRYPTO_OP_FLAG_INIT_IV)
274  {
275  t = ptd->cbc_iv[i];
276  *(u8x16u *) ops[0]->iv = t;
277  ptd->cbc_iv[i] = aes_enc_round (t, t);
278  }
279  else
280  t = aes_block_load (ops[0]->iv);
281 #if __VAES__
282  rq[i] = t;
283 #else
284  r[i] = t;
285 #endif
286 
287  src[i] = ops[0]->src;
288  dst[i] = ops[0]->dst;
289  len[i] = ops[0]->len;
290  placeholder_mask[i] = ~0;
291  if (key_index[i] != ops[0]->key_index)
292  {
293  aes_cbc_key_data_t *kd;
294  key_index[i] = ops[0]->key_index;
295  kd = (aes_cbc_key_data_t *) cm->key_data[key_index[i]];
296  for (j = 0; j < rounds + 1; j++)
297  {
298 #if __VAES__
299  kq = (u8x16 *) k[j];
300  kq[i] = kd->encrypt_key[j];
301 #else
302  k[j][i] = kd->encrypt_key[j];
303 #endif
304  }
305  }
306  ops[0]->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
307  n_left--;
308  ops++;
309  }
310  }
311 
312  count = u32xN_min_scalar (len);
313 
314  ASSERT (count % 16 == 0);
315 
316  for (i = 0; i < count; i += 16)
317  {
318 #ifdef __VAES__
319  r[0] = u8x64_xor3 (r[0], aes_block_load_x4 (src, i), k[0][0]);
320  r[1] = u8x64_xor3 (r[1], aes_block_load_x4 (src, i), k[0][1]);
321  r[2] = u8x64_xor3 (r[2], aes_block_load_x4 (src, i), k[0][2]);
322  r[3] = u8x64_xor3 (r[3], aes_block_load_x4 (src, i), k[0][3]);
323 
324  for (j = 1; j < rounds; j++)
325  {
326  r[0] = aes_enc_round_x4 (r[0], k[j][0]);
327  r[1] = aes_enc_round_x4 (r[1], k[j][1]);
328  r[2] = aes_enc_round_x4 (r[2], k[j][2]);
329  r[3] = aes_enc_round_x4 (r[3], k[j][3]);
330  }
331  r[0] = aes_enc_last_round_x4 (r[0], k[j][0]);
332  r[1] = aes_enc_last_round_x4 (r[1], k[j][1]);
333  r[2] = aes_enc_last_round_x4 (r[2], k[j][2]);
334  r[3] = aes_enc_last_round_x4 (r[3], k[j][3]);
335 
336  aes_block_store_x4 (dst, i, r[0]);
337  aes_block_store_x4 (dst + 4, i, r[1]);
338  aes_block_store_x4 (dst + 8, i, r[2]);
339  aes_block_store_x4 (dst + 12, i, r[3]);
340 #else
341 #if __x86_64__
342  r[0] = u8x16_xor3 (r[0], aes_block_load (src[0] + i), k[0][0]);
343  r[1] = u8x16_xor3 (r[1], aes_block_load (src[1] + i), k[0][1]);
344  r[2] = u8x16_xor3 (r[2], aes_block_load (src[2] + i), k[0][2]);
345  r[3] = u8x16_xor3 (r[3], aes_block_load (src[3] + i), k[0][3]);
346 
347  for (j = 1; j < rounds; j++)
348  {
349  r[0] = aes_enc_round (r[0], k[j][0]);
350  r[1] = aes_enc_round (r[1], k[j][1]);
351  r[2] = aes_enc_round (r[2], k[j][2]);
352  r[3] = aes_enc_round (r[3], k[j][3]);
353  }
354 
355  r[0] = aes_enc_last_round (r[0], k[j][0]);
356  r[1] = aes_enc_last_round (r[1], k[j][1]);
357  r[2] = aes_enc_last_round (r[2], k[j][2]);
358  r[3] = aes_enc_last_round (r[3], k[j][3]);
359 
360  aes_block_store (dst[0] + i, r[0]);
361  aes_block_store (dst[1] + i, r[1]);
362  aes_block_store (dst[2] + i, r[2]);
363  aes_block_store (dst[3] + i, r[3]);
364 #else
365  r[0] ^= aes_block_load (src[0] + i);
366  r[1] ^= aes_block_load (src[1] + i);
367  r[2] ^= aes_block_load (src[2] + i);
368  r[3] ^= aes_block_load (src[3] + i);
369  for (j = 0; j < rounds - 1; j++)
370  {
371  r[0] = vaesmcq_u8 (vaeseq_u8 (r[0], k[j][0]));
372  r[1] = vaesmcq_u8 (vaeseq_u8 (r[1], k[j][1]));
373  r[2] = vaesmcq_u8 (vaeseq_u8 (r[2], k[j][2]));
374  r[3] = vaesmcq_u8 (vaeseq_u8 (r[3], k[j][3]));
375  }
376  r[0] = vaeseq_u8 (r[0], k[j][0]) ^ k[rounds][0];
377  r[1] = vaeseq_u8 (r[1], k[j][1]) ^ k[rounds][1];
378  r[2] = vaeseq_u8 (r[2], k[j][2]) ^ k[rounds][2];
379  r[3] = vaeseq_u8 (r[3], k[j][3]) ^ k[rounds][3];
380  aes_block_store (dst[0] + i, r[0]);
381  aes_block_store (dst[1] + i, r[1]);
382  aes_block_store (dst[2] + i, r[2]);
383  aes_block_store (dst[3] + i, r[3]);
384 #endif
385 #endif
386  }
387 
388  len -= u32xN_splat (count);
389 
390  for (i = 0; i < N; i++)
391  {
392  src[i] += count;
393  dst[i] += count;
394  }
395 
396  if (n_left > 0)
397  goto more;
398 
399  if (!u32xN_is_all_zero (len & placeholder_mask))
400  goto more;
401 
402  return n_ops;
403 }
404 
405 
406 static_always_inline u32
407 aes_ops_dec_aes_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[],
408  u32 n_ops, aes_key_size_t ks)
409 {
410  crypto_native_main_t *cm = &crypto_native_main;
411  int rounds = AES_KEY_ROUNDS (ks);
412  vnet_crypto_op_t *op = ops[0];
413  aes_cbc_key_data_t *kd = (aes_cbc_key_data_t *) cm->key_data[op->key_index];
414  u32 n_left = n_ops;
415 
416  ASSERT (n_ops >= 1);
417 
418 decrypt:
419 #ifdef __VAES__
420  vaes_cbc_dec (kd->decrypt_key, (u8x64u *) op->src, (u8x64u *) op->dst,
421  (u8x16u *) op->iv, op->len, rounds);
422 #else
423  aes_cbc_dec (kd->decrypt_key, (u8x16u *) op->src, (u8x16u *) op->dst,
424  (u8x16u *) op->iv, op->len, rounds);
425 #endif
426  op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
427 
428  if (--n_left)
429  {
430  op += 1;
431  kd = (aes_cbc_key_data_t *) cm->key_data[op->key_index];
432  goto decrypt;
433  }
434 
435  return n_ops;
436 }
437 
438 static_always_inline void *
439 aes_cbc_key_exp (vnet_crypto_key_t * key, aes_key_size_t ks)
440 {
441  u8x16 e[15], d[15];
442  aes_cbc_key_data_t *kd;
443  kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES);
444  aes_key_expand (e, key->data, ks);
445  aes_key_enc_to_dec (e, d, ks);
446  for (int i = 0; i < AES_KEY_ROUNDS (ks) + 1; i++)
447  {
448 #if __VAES__
449  kd->decrypt_key[i] = (u8x64) _mm512_broadcast_i64x2 ((__m128i) d[i]);
450 #else
451  kd->decrypt_key[i] = d[i];
452 #endif
453  kd->encrypt_key[i] = e[i];
454  }
455  return kd;
456 }
457 
458 #define foreach_aes_cbc_handler_type _(128) _(192) _(256)
459 
460 #define _(x) \
461 static u32 aes_ops_dec_aes_cbc_##x \
462 (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \
463 { return aes_ops_dec_aes_cbc (vm, ops, n_ops, AES_KEY_##x); } \
464 static u32 aes_ops_enc_aes_cbc_##x \
465 (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \
466 { return aes_ops_enc_aes_cbc (vm, ops, n_ops, AES_KEY_##x); } \
467 static void * aes_cbc_key_exp_##x (vnet_crypto_key_t *key) \
468 { return aes_cbc_key_exp (key, AES_KEY_##x); }
469 
470 foreach_aes_cbc_handler_type;
471 #undef _
472 
473 #include <fcntl.h>
474 
475 clib_error_t *
476 #ifdef __VAES__
477 crypto_native_aes_cbc_init_icl (vlib_main_t * vm)
478 #elif __AVX512F__
479 crypto_native_aes_cbc_init_skx (vlib_main_t * vm)
480 #elif __aarch64__
481 crypto_native_aes_cbc_init_neon (vlib_main_t * vm)
482 #elif __AVX2__
483 crypto_native_aes_cbc_init_hsw (vlib_main_t * vm)
484 #else
485 crypto_native_aes_cbc_init_slm (vlib_main_t * vm)
486 #endif
487 {
488  crypto_native_main_t *cm = &crypto_native_main;
489  crypto_native_per_thread_data_t *ptd;
490  clib_error_t *err = 0;
491  int fd;
492 
493  if ((fd = open ("/dev/urandom", O_RDONLY)) < 0)
494  return clib_error_return_unix (0, "failed to open '/dev/urandom'");
495 
496  /* *INDENT-OFF* */
497  vec_foreach (ptd, cm->per_thread_data)
498  {
499  for (int i = 0; i < 4; i++)
500  {
501  if (read(fd, ptd->cbc_iv, sizeof (ptd->cbc_iv)) !=
502  sizeof (ptd->cbc_iv))
503  {
504  err = clib_error_return_unix (0, "'/dev/urandom' read failure");
505  goto error;
506  }
507  }
508  }
509  /* *INDENT-ON* */
510 
511 #define _(x) \
512  vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \
513  VNET_CRYPTO_OP_AES_##x##_CBC_ENC, \
514  aes_ops_enc_aes_cbc_##x); \
515  vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \
516  VNET_CRYPTO_OP_AES_##x##_CBC_DEC, \
517  aes_ops_dec_aes_cbc_##x); \
518  cm->key_fn[VNET_CRYPTO_ALG_AES_##x##_CBC] = aes_cbc_key_exp_##x;
519  foreach_aes_cbc_handler_type;
520 #undef _
521 
522 error:
523  close (fd);
524  return err;
525 }
526 
527 /*
528  * fd.io coding-style-patch-verification: ON
529  *
530  * Local Variables:
531  * eval: (c-set-style "gnu")
532  * End:
533  */
vlib.h
aes_dec_round
static_always_inline u8x16 aes_dec_round(u8x16 a, u8x16 k)
Definition: aes.h:90
u32xN
#define u32xN
Definition: aes_cbc.c:226
u8x64_insert_u8x16
#define u8x64_insert_u8x16(a, b, n)
Definition: vector_avx512.h:171
crypto.h
aes_cbc_key_exp
static_always_inline void * aes_cbc_key_exp(vnet_crypto_key_t *key, aes_key_size_t ks)
Definition: aes_cbc.c:439
aes_ops_dec_aes_cbc
static_always_inline u32 aes_ops_dec_aes_cbc(vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops, aes_key_size_t ks)
Definition: aes_cbc.c:407
aes_cbc_key_data_t::decrypt_key
u8x16 decrypt_key[15]
Definition: aes_cbc.c:34
f
vlib_frame_t * f
Definition: interface_output.c:1080
vnet_crypto_op_t::status
vnet_crypto_op_status_t status
Definition: crypto.h:260
u32xN_is_all_zero
#define u32xN_is_all_zero
Definition: aes_cbc.c:228
aes_cbc_key_data_t
Definition: aes_cbc.c:28
aes_dec_last_round
static_always_inline u8x16 aes_dec_last_round(u8x16 a, u8x16 k)
Definition: aes.h:96
aes_enc_last_round
static_always_inline u8x16 aes_enc_last_round(u8x16 a, u8x16 k)
Definition: aes.h:78
vnet_crypto_op_t::src
u8 * src
Definition: crypto.h:277
vnet_crypto_op_t::dst
u8 * dst
Definition: crypto.h:278
vnet_crypto_key_index_t
u32 vnet_crypto_key_index_t
Definition: crypto.h:378
vm
vlib_main_t * vm
X-connect all packets from the HOST to the PHY.
Definition: nat44_ei.c:3047
u8x16_xor3
static_always_inline u8x16 u8x16_xor3(u8x16 a, u8x16 b, u8x16 c)
Definition: vector_neon.h:204
u8x64_extract_u8x16
#define u8x64_extract_u8x16(a, n)
Definition: vector_avx512.h:174
vnet_crypto_op_t::iv
u8 * iv
Definition: crypto.h:293
VNET_CRYPTO_OP_FLAG_INIT_IV
#define VNET_CRYPTO_OP_FLAG_INIT_IV
Definition: crypto.h:262
r
vnet_hw_if_output_node_runtime_t * r
Definition: interface_output.c:1071
vnet_crypto_op_t::len
u32 len
Definition: crypto.h:287
crypto_native_main_t
Definition: crypto_native.h:29
error
Definition: cJSON.c:88
aes_cbc_key_data_t::encrypt_key
u8x16 encrypt_key[15]
Definition: aes_cbc.c:30
key
typedef key
Definition: ipsec_types.api:88
vnet_crypto_op_t
Definition: crypto.h:255
vnet_crypto_op_t::key_index
u32 key_index
Definition: crypto.h:292
u8x16
u8x16
Definition: vector_sse42.h:194
count
u8 count
Definition: dhcp.api:208
aes_key_expand
static_always_inline void aes_key_expand(u8x16 *key_schedule, u8 const *key, aes_key_size_t ks)
Definition: aes.h:410
aes_block_store
static_always_inline void aes_block_store(u8 *p, u8x16 r)
Definition: aes.h:103
crypto_native_aes_cbc_init_slm
clib_error_t * crypto_native_aes_cbc_init_slm(vlib_main_t *vm)
Definition: aes_cbc.c:485
aes_key_size_t
aes_key_size_t
Definition: aes.h:21
len
u8 len
Definition: ip_types.api:103
vec_elt_at_index
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
Definition: vec_bootstrap.h:203
aes.h
c
svmdb_client_t * c
Definition: vpp_get_metrics.c:48
static_always_inline
#define static_always_inline
Definition: clib.h:112
foreach_aes_cbc_handler_type
#define foreach_aes_cbc_handler_type
Definition: aes_cbc.c:458
aes_key_enc_to_dec
static_always_inline void aes_key_enc_to_dec(u8x16 *ke, u8x16 *kd, aes_key_size_t ks)
Definition: aes.h:427
vlib_main_t::thread_index
u32 thread_index
Definition: main.h:213
i
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:261
aes_cbc_dec
static_always_inline void __clib_unused aes_cbc_dec(u8x16 *k, u8x16u *src, u8x16u *dst, u8x16u *iv, int count, int rounds)
Definition: aes_cbc.c:40
cm
vnet_feature_config_main_t * cm
Definition: nat44_ei_hairpinning.c:591
src
vl_api_address_t src
Definition: gre.api:54
crypto_native.h
crypto_native_main
crypto_native_main_t crypto_native_main
Definition: main.c:23
u32xN_min_scalar
#define u32xN_min_scalar
Definition: aes_cbc.c:227
CLIB_CACHE_LINE_BYTES
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
crypto_native_per_thread_data_t::cbc_iv
u8x16 cbc_iv[16]
Definition: crypto_native.h:26
plugin.h
ASSERT
#define ASSERT(truth)
Definition: error_bootstrap.h:69
vnet_crypto_key_t
Definition: crypto.h:198
u32
unsigned int u32
Definition: types.h:88
aes_ops_enc_aes_cbc
static_always_inline u32 aes_ops_enc_aes_cbc(vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops, aes_key_size_t ks)
Definition: aes_cbc.c:233
u8x64_xor3
static_always_inline u8x64 u8x64_xor3(u8x64 a, u8x64 b, u8x64 c)
Definition: vector_avx512.h:181
clib_prefetch_load
static_always_inline void clib_prefetch_load(void *p)
Definition: cache.h:94
dst
vl_api_ip4_address_t dst
Definition: pnat.api:41
vec_foreach
#define vec_foreach(var, vec)
Vector iterator.
Definition: vec_bootstrap.h:213
for
for(i=1;i<=collision_buckets;i++)
Definition: flowhash_template.h:378
n_left
u32 n_left
Definition: interface_output.c:1078
clib_error_return_unix
#define clib_error_return_unix(e, args...)
Definition: error.h:102
iv
static u8 iv[]
Definition: aes_cbc.c:24
aes_enc_round
static_always_inline u8x16 aes_enc_round(u8x16 a, u8x16 k)
Definition: aes.h:42
vlib_main_t
Definition: main.h:102
b
vlib_buffer_t ** b
Definition: nat44_ei_out2in.c:717
u8
unsigned char u8
Definition: types.h:56
clib_error_t
Definition: clib_error.h:21
a
a
Definition: bitmap.h:544
AES_KEY_ROUNDS
#define AES_KEY_ROUNDS(x)
Definition: aes.h:28
clib_mem_alloc_aligned
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:261
u8x64
u8x64
Definition: vector_avx512.h:96
aes_block_load
static_always_inline u8x16 aes_block_load(u8 *p)
Definition: aes.h:36
crypto_native_per_thread_data_t
Definition: crypto_native.h:23
u32xN_splat
#define u32xN_splat
Definition: aes_cbc.c:229
N
#define N
Definition: aes_cbc.c:225
flags
vl_api_wireguard_peer_flags_t flags
Definition: wireguard.api:105