FD.io VPP  v21.10.1-2-g0a485f517
Vector Packet Processing
cryptodev_op_data_path.c
Go to the documentation of this file.
1 
2 /*
3  *------------------------------------------------------------------
4  * Copyright (c) 2019 - 2021 Intel and/or its affiliates.
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at:
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  *------------------------------------------------------------------
17  */
18 
19 #include <vlib/vlib.h>
20 #include <vnet/crypto/crypto.h>
21 
22 #include <dpdk/buffer.h>
23 #include <dpdk/device/dpdk.h>
24 #include <dpdk/device/dpdk_priv.h>
25 #undef always_inline
26 #include <rte_bus_vdev.h>
27 #include <rte_cryptodev.h>
28 #include <rte_crypto_sym.h>
29 #include <rte_crypto.h>
30 #include <rte_cryptodev_pmd.h>
31 #include <rte_ring_peek_zc.h>
32 #include <rte_config.h>
33 
34 #include "cryptodev.h"
35 
36 #if CLIB_DEBUG > 0
37 #define always_inline static inline
38 #else
39 #define always_inline static inline __attribute__ ((__always_inline__))
40 #endif
41 
42 #define CRYPTODEV_AAD_OFFSET (offsetof (cryptodev_op_t, aad))
43 
44 #define foreach_vnet_crypto_status_conversion \
45  _ (SUCCESS, COMPLETED) \
46  _ (NOT_PROCESSED, WORK_IN_PROGRESS) \
47  _ (AUTH_FAILED, FAIL_BAD_HMAC) \
48  _ (INVALID_SESSION, FAIL_ENGINE_ERR) \
49  _ (INVALID_ARGS, FAIL_ENGINE_ERR) \
50  _ (ERROR, FAIL_ENGINE_ERR)
51 
53 #define _(a, b) VNET_CRYPTO_OP_STATUS_##b,
55 #undef _
56 };
57 
58 static_always_inline rte_iova_t
59 cryptodev_get_iova (clib_pmalloc_main_t *pm, enum rte_iova_mode mode,
60  void *data)
61 {
62  u64 index;
63  if (mode == RTE_IOVA_VA)
64  return (rte_iova_t) pointer_to_uword (data);
65 
67  return pointer_to_uword (data) - pm->lookup_table[index];
68 }
69 
73 {
74  struct rte_mbuf *first_mb = mb, *last_mb = mb; /**< last mbuf */
75  /* when input node is not dpdk, mbuf data len is not initialized, for
76  * single buffer it is not a problem since the data length is written
77  * into cryptodev operation. For chained buffer a reference data length
78  * has to be computed through vlib_buffer.
79  *
80  * even when input node is dpdk, it is possible chained vlib_buffers
81  * are updated (either added or removed a buffer) but not not mbuf fields.
82  * we have to re-link every mbuf in the chain.
83  */
85  (b->data + b->current_data - rte_pktmbuf_mtod (mb, u8 *));
86 
87  first_mb->nb_segs = 1;
88  first_mb->pkt_len = first_mb->data_len = data_len;
89 
90  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
91  {
94  if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_EXT_HDR_VALID) == 0))
95  rte_pktmbuf_reset (mb);
96  last_mb->next = mb;
97  last_mb = mb;
98  mb->data_len = b->current_length;
99  mb->pkt_len = b->current_length;
100  mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data;
101  first_mb->nb_segs++;
102  if (PREDICT_FALSE (b->ref_count > 1))
103  mb->pool =
105  }
106 }
107 
109 crypto_op_init (struct rte_mempool *mempool,
110  void *_arg __attribute__ ((unused)), void *_obj,
111  unsigned i __attribute__ ((unused)))
112 {
113  struct rte_crypto_op *op = _obj;
114 
115  op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
116  op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
117  op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
118  op->phys_addr = rte_mempool_virt2iova (_obj);
119  op->mempool = mempool;
120 }
121 
125  cryptodev_op_type_t op_type)
126 {
131  struct rte_cryptodev_sym_session *sess = 0;
132  cryptodev_op_t **cop;
133  u32 *bi;
134  u32 n_enqueue, n_elts;
135  u32 last_key_index = ~0;
136 
137  if (PREDICT_FALSE (frame == 0 || frame->n_elts == 0))
138  return -1;
139  n_elts = frame->n_elts;
140 
141  if (PREDICT_FALSE (CRYPTODEV_NB_CRYPTO_OPS - cet->inflight < n_elts))
142  {
144  VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
145  return -1;
146  }
147 
148  if (PREDICT_FALSE (
149  rte_mempool_get_bulk (cet->cop_pool, (void **) cet->cops, n_elts) < 0))
150  {
152  VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
153  return -1;
154  }
155 
156  cop = cet->cops;
157  fe = frame->elts;
158  bi = frame->buffer_indices;
159  cop[0]->frame = frame;
160  cop[0]->n_elts = n_elts;
161 
162  while (n_elts)
163  {
164  vlib_buffer_t *b = vlib_get_buffer (vm, bi[0]);
165  struct rte_crypto_sym_op *sop = &cop[0]->sop;
166  i16 crypto_offset = fe->crypto_start_offset;
167  i16 integ_offset = fe->integ_start_offset;
168  u32 offset_diff = crypto_offset - integ_offset;
169 
170  if (n_elts > 2)
171  {
172  CLIB_PREFETCH (cop[1], sizeof (*cop[1]), STORE);
173  CLIB_PREFETCH (cop[2], sizeof (*cop[2]), STORE);
174  clib_prefetch_load (&fe[1]);
175  clib_prefetch_load (&fe[2]);
176  }
177  if (last_key_index != fe->key_index)
178  {
180  last_key_index = fe->key_index;
181 
182  if (key->keys[vm->numa_node][op_type] == 0)
183  {
184  if (PREDICT_FALSE (
185  cryptodev_session_create (vm, last_key_index, 0) < 0))
186  {
188  frame, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
189  return -1;
190  }
191  }
192  sess = key->keys[vm->numa_node][op_type];
193  }
194 
195  sop->m_src = rte_mbuf_from_vlib_buffer (b);
196  sop->m_src->data_off = VLIB_BUFFER_PRE_DATA_SIZE;
197  sop->m_dst = 0;
198  /* mbuf prepend happens in the tx, but vlib_buffer happens in the nodes,
199  * so we have to manually adjust mbuf data_off here so cryptodev can
200  * correctly compute the data pointer. The prepend here will be later
201  * rewritten by tx. */
202  if (PREDICT_TRUE (fe->integ_start_offset < 0))
203  {
204  sop->m_src->data_off += fe->integ_start_offset;
205  integ_offset = 0;
206  crypto_offset = offset_diff;
207  }
208  sop->session = sess;
209  sop->cipher.data.offset = crypto_offset;
210  sop->cipher.data.length = fe->crypto_total_length;
211  sop->auth.data.offset = integ_offset;
212  sop->auth.data.length = fe->crypto_total_length + fe->integ_length_adj;
213  sop->auth.digest.data = fe->digest;
214  sop->auth.digest.phys_addr =
215  cryptodev_get_iova (pm, cmt->iova_mode, fe->digest);
217  cryptodev_validate_mbuf_chain (vm, sop->m_src, b);
218  else
219  /* for input nodes that are not dpdk-input, it is possible the mbuf
220  * was updated before as one of the chained mbufs. Setting nb_segs
221  * to 1 here to prevent the cryptodev PMD to access potentially
222  * invalid m_src->next pointers.
223  */
224  sop->m_src->nb_segs = 1;
225  clib_memcpy_fast (cop[0]->iv, fe->iv, 16);
226  cop++;
227  bi++;
228  fe++;
229  n_elts--;
230  }
231 
232  n_enqueue = rte_cryptodev_enqueue_burst (cet->cryptodev_id, cet->cryptodev_q,
233  (struct rte_crypto_op **) cet->cops,
234  frame->n_elts);
235  ASSERT (n_enqueue == frame->n_elts);
236  cet->inflight += n_enqueue;
237 
238  return 0;
239 }
240 
244  cryptodev_op_type_t op_type, u8 aad_len)
245 {
250  struct rte_cryptodev_sym_session *sess = 0;
251  cryptodev_op_t **cop;
252  u32 *bi;
253  u32 n_enqueue = 0, n_elts;
254  u32 last_key_index = ~0;
255 
256  if (PREDICT_FALSE (frame == 0 || frame->n_elts == 0))
257  return -1;
258  n_elts = frame->n_elts;
259 
260  if (PREDICT_FALSE (CRYPTODEV_MAX_INFLIGHT - cet->inflight < n_elts))
261  {
263  VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
264  return -1;
265  }
266 
267  if (PREDICT_FALSE (
268  rte_mempool_get_bulk (cet->cop_pool, (void **) cet->cops, n_elts) < 0))
269  {
271  VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
272  return -1;
273  }
274 
275  cop = cet->cops;
276  fe = frame->elts;
277  bi = frame->buffer_indices;
278  cop[0]->frame = frame;
279  cop[0]->n_elts = n_elts;
280 
281  while (n_elts)
282  {
283  vlib_buffer_t *b = vlib_get_buffer (vm, bi[0]);
284  struct rte_crypto_sym_op *sop = &cop[0]->sop;
285  u16 crypto_offset = fe->crypto_start_offset;
286 
287  if (n_elts > 2)
288  {
289  CLIB_PREFETCH (cop[1], sizeof (*cop[1]), STORE);
290  CLIB_PREFETCH (cop[2], sizeof (*cop[2]), STORE);
291  clib_prefetch_load (&fe[1]);
292  clib_prefetch_load (&fe[2]);
293  }
294  if (last_key_index != fe->key_index)
295  {
297 
298  last_key_index = fe->key_index;
299  if (key->keys[vm->numa_node][op_type] == 0)
300  {
301  if (PREDICT_FALSE (cryptodev_session_create (vm, last_key_index,
302  aad_len) < 0))
303  {
305  frame, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
306  return -1;
307  }
308  }
309  else if (PREDICT_FALSE (
310  key->keys[vm->numa_node][op_type]->opaque_data !=
311  aad_len))
312  {
314  fe->key_index, aad_len);
315  if (PREDICT_FALSE (cryptodev_session_create (vm, last_key_index,
316  aad_len) < 0))
317  {
319  frame, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
320  return -1;
321  }
322  }
323 
324  sess = key->keys[vm->numa_node][op_type];
325  }
326 
327  sop->m_src = rte_mbuf_from_vlib_buffer (b);
328  sop->m_src->data_off = VLIB_BUFFER_PRE_DATA_SIZE;
329  sop->m_dst = 0;
330  /* mbuf prepend happens in the tx, but vlib_buffer happens in the nodes,
331  * so we have to manually adjust mbuf data_off here so cryptodev can
332  * correctly compute the data pointer. The prepend here will be later
333  * rewritten by tx. */
334  if (PREDICT_FALSE (fe->crypto_start_offset < 0))
335  {
336  rte_pktmbuf_prepend (sop->m_src, -fe->crypto_start_offset);
337  crypto_offset = 0;
338  }
339 
340  sop->session = sess;
341  sop->aead.aad.data = cop[0]->aad;
342  sop->aead.aad.phys_addr = cop[0]->op.phys_addr + CRYPTODEV_AAD_OFFSET;
343  sop->aead.data.length = fe->crypto_total_length;
344  sop->aead.data.offset = crypto_offset;
345  sop->aead.digest.data = fe->tag;
346  sop->aead.digest.phys_addr =
347  cryptodev_get_iova (pm, cmt->iova_mode, fe->tag);
349  cryptodev_validate_mbuf_chain (vm, sop->m_src, b);
350  else
351  /* for input nodes that are not dpdk-input, it is possible the mbuf
352  * was updated before as one of the chained mbufs. Setting nb_segs
353  * to 1 here to prevent the cryptodev PMD to access potentially
354  * invalid m_src->next pointers.
355  */
356  sop->m_src->nb_segs = 1;
357  clib_memcpy_fast (cop[0]->iv, fe->iv, 12);
358  clib_memcpy_fast (cop[0]->aad, fe->aad, aad_len);
359  cop++;
360  bi++;
361  fe++;
362  n_elts--;
363  }
364 
365  n_enqueue = rte_cryptodev_enqueue_burst (cet->cryptodev_id, cet->cryptodev_q,
366  (struct rte_crypto_op **) cet->cops,
367  frame->n_elts);
368  ASSERT (n_enqueue == frame->n_elts);
369  cet->inflight += n_enqueue;
370 
371  return 0;
372 }
373 
375 cryptodev_ring_deq (struct rte_ring *r, cryptodev_op_t **cops)
376 {
377  u16 n, n_elts = 0;
378 
379  n = rte_ring_dequeue_bulk_start (r, (void **) cops, 1, 0);
380  rte_ring_dequeue_finish (r, 0);
381  if (!n)
382  return 0;
383 
384  n = cops[0]->n_elts;
385  if (rte_ring_count (r) < n)
386  return 0;
387 
388  n_elts = rte_ring_sc_dequeue_bulk (r, (void **) cops, n, 0);
389  ASSERT (n_elts == n);
390 
391  return n_elts;
392 }
393 
396  u32 *enqueue_thread_idx)
397 {
400  cryptodev_op_t **cop = cet->cops;
403  u32 n_elts, n_completed_ops = rte_ring_count (cet->ring);
404  u32 ss0 = 0, ss1 = 0, ss2 = 0, ss3 = 0; /* sum of status */
405 
406  if (cet->inflight)
407  {
408  n_elts = rte_cryptodev_dequeue_burst (
409  cet->cryptodev_id, cet->cryptodev_q,
410  (struct rte_crypto_op **) cet->cops, VNET_CRYPTO_FRAME_SIZE);
411 
412  if (n_elts)
413  {
414  cet->inflight -= n_elts;
415  n_completed_ops += n_elts;
416 
417  rte_ring_sp_enqueue_burst (cet->ring, (void **) cet->cops, n_elts,
418  NULL);
419  }
420  }
421 
422  if (PREDICT_FALSE (n_completed_ops == 0))
423  return 0;
424 
425  n_elts = cryptodev_ring_deq (cet->ring, cop);
426  if (!n_elts)
427  return 0;
428 
429  frame = cop[0]->frame;
430  fe = frame->elts;
431 
432  while (n_elts > 4)
433  {
434  ss0 |= fe[0].status = cryptodev_status_conversion[cop[0]->op.status];
435  ss1 |= fe[1].status = cryptodev_status_conversion[cop[1]->op.status];
436  ss2 |= fe[2].status = cryptodev_status_conversion[cop[2]->op.status];
437  ss3 |= fe[3].status = cryptodev_status_conversion[cop[3]->op.status];
438 
439  cop += 4;
440  fe += 4;
441  n_elts -= 4;
442  }
443 
444  while (n_elts)
445  {
446  ss0 |= fe[0].status = cryptodev_status_conversion[cop[0]->op.status];
447  fe++;
448  cop++;
449  n_elts--;
450  }
451 
452  frame->state = (ss0 | ss1 | ss2 | ss3) == VNET_CRYPTO_OP_STATUS_COMPLETED ?
455 
456  rte_mempool_put_bulk (cet->cop_pool, (void **) cet->cops, frame->n_elts);
457  *nb_elts_processed = frame->n_elts;
458  *enqueue_thread_idx = frame->enqueue_thread_index;
459  return frame;
460 }
461 
465 {
467  8);
468 }
472 {
474  12);
475 }
476 
480 {
482  8);
483 }
487 {
489  12);
490 }
491 
495 {
498 }
499 
503 {
506 }
507 
508 clib_error_t *
510 {
513  struct rte_cryptodev_sym_capability_idx cap_auth_idx;
514  struct rte_cryptodev_sym_capability_idx cap_cipher_idx;
515  struct rte_cryptodev_sym_capability_idx cap_aead_idx;
516  u8 *name;
517  clib_error_t *error = 0;
518 
519  vec_foreach (cet, cmt->per_thread_data)
520  {
521  u32 thread_index = cet - cmt->per_thread_data;
523  name = format (0, "vpp_cop_pool_%u_%u", numa, thread_index);
524  cet->cop_pool = rte_mempool_create (
525  (char *) name, CRYPTODEV_NB_CRYPTO_OPS, sizeof (cryptodev_op_t), 0,
526  sizeof (struct rte_crypto_op_pool_private), NULL, NULL, crypto_op_init,
527  NULL, vm->numa_node, 0);
528  if (!cet->cop_pool)
529  {
531  0, "Failed to create cryptodev op pool %s", name);
532 
533  goto error_exit;
534  }
535  vec_free (name);
536 
537  name = format (0, "frames_ring_%u_%u", numa, thread_index);
538  cet->ring =
539  rte_ring_create ((char *) name, CRYPTODEV_NB_CRYPTO_OPS, vm->numa_node,
540  RING_F_SP_ENQ | RING_F_SC_DEQ);
541  if (!cet->ring)
542  {
544  0, "Failed to create cryptodev op pool %s", name);
545 
546  goto error_exit;
547  }
548  vec_free (name);
549 
551  }
552 
553  /** INDENT-OFF **/
554 #define _(a, b, c, d, e, f, g) \
555  cap_aead_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD; \
556  cap_aead_idx.algo.aead = RTE_CRYPTO_##b##_##c; \
557  if (cryptodev_check_cap_support (&cap_aead_idx, g, e, f)) \
558  { \
559  vnet_crypto_register_async_handler ( \
560  vm, eidx, VNET_CRYPTO_OP_##a##_TAG##e##_AAD##f##_ENC, \
561  cryptodev_enqueue_aead_aad_##f##_enc, cryptodev_frame_dequeue); \
562  vnet_crypto_register_async_handler ( \
563  vm, eidx, VNET_CRYPTO_OP_##a##_TAG##e##_AAD##f##_DEC, \
564  cryptodev_enqueue_aead_aad_##f##_dec, cryptodev_frame_dequeue); \
565  }
567 #undef _
568 
569 #define _(a, b, c, d, e) \
570  cap_auth_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH; \
571  cap_auth_idx.algo.auth = RTE_CRYPTO_AUTH_##d##_HMAC; \
572  cap_cipher_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER; \
573  cap_cipher_idx.algo.cipher = RTE_CRYPTO_CIPHER_##b; \
574  if (cryptodev_check_cap_support (&cap_cipher_idx, c, -1, -1) && \
575  cryptodev_check_cap_support (&cap_auth_idx, -1, e, -1)) \
576  { \
577  vnet_crypto_register_async_handler ( \
578  vm, eidx, VNET_CRYPTO_OP_##a##_##d##_TAG##e##_ENC, \
579  cryptodev_enqueue_linked_alg_enc, cryptodev_frame_dequeue); \
580  vnet_crypto_register_async_handler ( \
581  vm, eidx, VNET_CRYPTO_OP_##a##_##d##_TAG##e##_DEC, \
582  cryptodev_enqueue_linked_alg_dec, cryptodev_frame_dequeue); \
583  }
585 #undef _
586  /** INDENT-ON **/
587 
588  return 0;
589 
590 error_exit:
591  vec_foreach (cet, cmt->per_thread_data)
592  {
593  if (cet->ring)
594  rte_ring_free (cet->ring);
595 
596  if (cet->cop_pool)
597  rte_mempool_free (cet->cop_pool);
598  }
599 
600  return error;
601 }
vlib.h
VNET_CRYPTO_KEY_OP_DEL
@ VNET_CRYPTO_KEY_OP_DEL
Definition: crypto.h:132
vlib_buffer_t::next_buffer
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:149
cryptodev_engine_thread_t::inflight
u16 inflight
Definition: cryptodev.h:160
dpdk.h
vlib_buffer_t::buffer_pool_index
u8 buffer_pool_index
index of buffer pool this buffer belongs.
Definition: buffer.h:142
cryptodev_frame_dequeue
static_always_inline vnet_crypto_async_frame_t * cryptodev_frame_dequeue(vlib_main_t *vm, u32 *nb_elts_processed, u32 *enqueue_thread_idx)
Definition: cryptodev_op_data_path.c:395
thread_index
u32 thread_index
Definition: nat44_ei_hairpinning.c:495
cryptodev_engine_thread_t::cryptodev_q
u16 cryptodev_q
Definition: cryptodev.h:159
cryptodev_main_t::iova_mode
enum rte_iova_mode iova_mode
Definition: cryptodev.h:168
cryptodev_engine_thread_t::cop_pool
struct rte_mempool * cop_pool
Definition: cryptodev.h:145
frame
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
Definition: nat44_ei.c:3048
vnet_crypto_async_frame_elt_t::integ_length_adj
u16 integ_length_adj
Definition: crypto.h:340
vnet_crypto_async_frame_elt_t::crypto_total_length
u32 crypto_total_length
Definition: crypto.h:336
crypto.h
cryptodev_frame_linked_algs_enqueue
static_always_inline int cryptodev_frame_linked_algs_enqueue(vlib_main_t *vm, vnet_crypto_async_frame_t *frame, cryptodev_op_type_t op_type)
Definition: cryptodev_op_data_path.c:123
vlib_get_buffer
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:111
pointer_to_uword
static uword pointer_to_uword(const void *p)
Definition: types.h:131
name
string name[64]
Definition: fib.api:25
cryptodev_ring_deq
static_always_inline u16 cryptodev_ring_deq(struct rte_ring *r, cryptodev_op_t **cops)
Definition: cryptodev_op_data_path.c:375
CRYPTODEV_OP_TYPE_DECRYPT
@ CRYPTODEV_OP_TYPE_DECRYPT
Definition: cryptodev.h:74
cryptodev_main_t
Definition: cryptodev.h:163
clib_error_return
#define clib_error_return(e, args...)
Definition: error.h:99
VNET_CRYPTO_FRAME_SIZE
#define VNET_CRYPTO_FRAME_SIZE
Definition: crypto.h:21
u16
unsigned short u16
Definition: types.h:57
vnet_crypto_async_frame_t
Definition: crypto.h:358
mode
vl_api_tunnel_mode_t mode
Definition: gre.api:48
vm
vlib_main_t * vm
X-connect all packets from the HOST to the PHY.
Definition: nat44_ei.c:3047
CRYPTODEV_OP_TYPE_ENCRYPT
@ CRYPTODEV_OP_TYPE_ENCRYPT
Definition: cryptodev.h:73
vlib_physmem_main_t::pmalloc_main
clib_pmalloc_main_t * pmalloc_main
Definition: physmem.h:64
vnet_crypto_async_frame_elt_t::crypto_start_offset
i16 crypto_start_offset
Definition: crypto.h:337
clib_pmalloc_get_page_index
static u32 clib_pmalloc_get_page_index(clib_pmalloc_main_t *pm, void *va)
Definition: pmalloc.h:128
r
vnet_hw_if_output_node_runtime_t * r
Definition: interface_output.c:1089
clib_memcpy_fast
static_always_inline void * clib_memcpy_fast(void *restrict dst, const void *restrict src, size_t n)
Definition: string.h:92
cryptodev_frame_aead_enqueue
static_always_inline int cryptodev_frame_aead_enqueue(vlib_main_t *vm, vnet_crypto_async_frame_t *frame, cryptodev_op_type_t op_type, u8 aad_len)
Definition: cryptodev_op_data_path.c:242
error
Definition: cJSON.c:88
VNET_CRYPTO_FRAME_STATE_ELT_ERROR
@ VNET_CRYPTO_FRAME_STATE_ELT_ERROR
Definition: crypto.h:355
cryptodev_engine_thread_t
Definition: cryptodev.h:136
key
typedef key
Definition: ipsec_types.api:91
cryptodev_enqueue_aead_aad_12_dec
static_always_inline int cryptodev_enqueue_aead_aad_12_dec(vlib_main_t *vm, vnet_crypto_async_frame_t *frame)
Definition: cryptodev_op_data_path.c:485
cryptodev_main_t::keys
cryptodev_key_t * keys
Definition: cryptodev.h:166
foreach_vnet_crypto_status_conversion
#define foreach_vnet_crypto_status_conversion
Definition: cryptodev_op_data_path.c:44
cryptodev_register_cop_hdl
clib_error_t * cryptodev_register_cop_hdl(vlib_main_t *vm, u32 eidx)
Definition: cryptodev_op_data_path.c:509
cryptodev_engine_thread_t::cryptodev_id
u16 cryptodev_id
Definition: cryptodev.h:158
CLIB_PREFETCH
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:76
i16
signed short i16
Definition: types.h:46
vlib_buffer_t::current_data
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:119
cryptodev_enqueue_linked_alg_enc
static_always_inline int cryptodev_enqueue_linked_alg_enc(vlib_main_t *vm, vnet_crypto_async_frame_t *frame)
Definition: cryptodev_op_data_path.c:493
cryptodev_main
cryptodev_main_t cryptodev_main
Definition: cryptodev.c:43
vnet_crypto_async_frame_elt_t::status
vnet_crypto_op_status_t status
Definition: crypto.h:341
vlib_main_t::physmem_main
vlib_physmem_main_t physmem_main
Definition: main.h:170
vnet_crypto_async_frame_elt_t::key_index
u32 key_index
Definition: crypto.h:335
vec_elt_at_index
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
Definition: vec_bootstrap.h:203
error_exit
static void error_exit(int code)
Definition: error.c:88
PREDICT_FALSE
#define PREDICT_FALSE(x)
Definition: clib.h:124
cryptodev_key_t
Definition: cryptodev.h:79
cryptodev_get_iova
static_always_inline rte_iova_t cryptodev_get_iova(clib_pmalloc_main_t *pm, enum rte_iova_mode mode, void *data)
Definition: cryptodev_op_data_path.c:59
cryptodev_op_t
Definition: cryptodev.h:125
VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS
#define VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS
Definition: crypto.h:264
cryptodev_op_t::op
struct rte_crypto_op op
Definition: cryptodev.h:128
static_always_inline
#define static_always_inline
Definition: clib.h:112
cryptodev_main_t::per_thread_data
cryptodev_engine_thread_t * per_thread_data
Definition: cryptodev.h:167
buffer.h
clib_pmalloc_main_t::lookup_table
uword * lookup_table
Definition: pmalloc.h:93
vlib_main_t::thread_index
u32 thread_index
Definition: main.h:215
cryptodev_sess_handler
void cryptodev_sess_handler(vlib_main_t *vm, vnet_crypto_key_op_t kop, vnet_crypto_key_index_t idx, u32 aad_len)
Definition: cryptodev.c:272
vec_validate
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment)
Definition: vec.h:523
vlib_buffer_t::ref_count
volatile u8 ref_count
Reference count for this buffer.
Definition: buffer.h:139
vnet_crypto_async_frame_elt_t::flags
u8 flags
share same VNET_CRYPTO_OP_FLAG_* values
Definition: crypto.h:342
cryptodev_engine_thread_t::cops
cryptodev_op_t ** cops
Definition: cryptodev.h:144
cryptodev_mark_frame_err_status
static_always_inline void cryptodev_mark_frame_err_status(vnet_crypto_async_frame_t *f, vnet_crypto_op_status_t s)
Definition: cryptodev.h:179
CRYPTODEV_AAD_OFFSET
#define CRYPTODEV_AAD_OFFSET
Definition: cryptodev_op_data_path.c:42
clib_pmalloc_main_t
Definition: pmalloc.h:60
cryptodev_engine_thread_t::ring
struct rte_ring * ring
Definition: cryptodev.h:146
vlib_buffer_t::current_length
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:122
CRYPTODEV_NB_CRYPTO_OPS
#define CRYPTODEV_NB_CRYPTO_OPS
Definition: cryptodev.h:24
cryptodev_enqueue_aead_aad_12_enc
static_always_inline int cryptodev_enqueue_aead_aad_12_enc(vlib_main_t *vm, vnet_crypto_async_frame_t *frame)
Definition: cryptodev_op_data_path.c:470
dpdk_no_cache_mempool_by_buffer_pool_index
struct rte_mempool ** dpdk_no_cache_mempool_by_buffer_pool_index
Definition: buffer.c:34
rte_mbuf_from_vlib_buffer
#define rte_mbuf_from_vlib_buffer(x)
Definition: buffer.h:19
data
u8 data[128]
Definition: ipsec_types.api:95
vlib_main_t::numa_node
u32 numa_node
Definition: main.h:217
cryptodev_status_conversion
static const vnet_crypto_op_status_t cryptodev_status_conversion[]
Definition: cryptodev_op_data_path.c:52
cryptodev_op_type_t
cryptodev_op_type_t
Definition: cryptodev.h:71
vec_free
#define vec_free(V)
Free vector's memory (no header).
Definition: vec.h:395
index
u32 index
Definition: flow_types.api:221
vnet_crypto_async_frame_elt_t::aad
u8 * aad
Definition: crypto.h:334
u64
unsigned long u64
Definition: types.h:89
format
description fragment has unexpected format
Definition: map.api:433
ASSERT
#define ASSERT(truth)
Definition: error_bootstrap.h:69
data_len
u8 data_len
Definition: ikev2_types.api:24
cryptodev_enqueue_aead_aad_8_enc
static_always_inline int cryptodev_enqueue_aead_aad_8_enc(vlib_main_t *vm, vnet_crypto_async_frame_t *frame)
Definition: cryptodev_op_data_path.c:463
VLIB_BUFFER_PRE_DATA_SIZE
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:51
u32
unsigned int u32
Definition: types.h:88
clib_prefetch_load
static_always_inline void clib_prefetch_load(void *p)
Definition: cache.h:92
VNET_CRYPTO_FRAME_STATE_SUCCESS
@ VNET_CRYPTO_FRAME_STATE_SUCCESS
Definition: crypto.h:354
vec_foreach
#define vec_foreach(var, vec)
Vector iterator.
Definition: vec_bootstrap.h:213
vlib_get_main_by_index
static vlib_main_t * vlib_get_main_by_index(u32 thread_index)
Definition: global_funcs.h:29
iv
static u8 iv[]
Definition: aes_cbc.c:24
foreach_vnet_aead_crypto_conversion
#define foreach_vnet_aead_crypto_conversion
Definition: cryptodev.h:40
vnet_crypto_async_frame_elt_t::integ_start_offset
i16 integ_start_offset
Definition: crypto.h:338
cryptodev_op_t::aad
u8 aad[CRYPTODEV_MAX_AAD_SIZE]
Definition: cryptodev.h:131
cryptodev_enqueue_aead_aad_8_dec
static_always_inline int cryptodev_enqueue_aead_aad_8_dec(vlib_main_t *vm, vnet_crypto_async_frame_t *frame)
Definition: cryptodev_op_data_path.c:478
cryptodev_enqueue_linked_alg_dec
static_always_inline int cryptodev_enqueue_linked_alg_dec(vlib_main_t *vm, vnet_crypto_async_frame_t *frame)
Definition: cryptodev_op_data_path.c:501
vlib_main_t
Definition: main.h:102
b
vlib_buffer_t ** b
Definition: nat44_ei_out2in.c:717
u8
unsigned char u8
Definition: types.h:56
clib_error_t
Definition: clib_error.h:21
cryptodev.h
vnet_crypto_async_frame_elt_t::tag
u8 * tag
Definition: crypto.h:332
vlib_buffer_t::data
u8 data[]
Packet data.
Definition: buffer.h:204
i
int i
Definition: flowhash_template.h:376
vnet_crypto_async_frame_elt_t::iv
u8 * iv
Definition: crypto.h:328
cryptodev_op_t::frame
vnet_crypto_async_frame_t * frame
Definition: cryptodev.h:132
CRYPTODEV_MAX_INFLIGHT
#define CRYPTODEV_MAX_INFLIGHT
Definition: cryptodev.h:27
cryptodev_op_t::sop
struct rte_crypto_sym_op sop
Definition: cryptodev.h:129
vnet_crypto_async_frame_elt_t
Definition: crypto.h:326
PREDICT_TRUE
#define PREDICT_TRUE(x)
Definition: clib.h:125
crypto_op_init
static_always_inline void crypto_op_init(struct rte_mempool *mempool, void *_arg, void *_obj, unsigned i)
Definition: cryptodev_op_data_path.c:109
foreach_cryptodev_link_async_alg
#define foreach_cryptodev_link_async_alg
crypto (alg, cryptodev_alg, key_size), hash (alg, digest-size)
Definition: cryptodev.h:51
cryptodev_session_create
int cryptodev_session_create(vlib_main_t *vm, vnet_crypto_key_index_t idx, u32 aad_len)
Definition: cryptodev.c:326
dpdk_priv.h
cryptodev_validate_mbuf_chain
static_always_inline void cryptodev_validate_mbuf_chain(vlib_main_t *vm, struct rte_mbuf *mb, vlib_buffer_t *b)
Definition: cryptodev_op_data_path.c:71
vnet_crypto_op_status_t
vnet_crypto_op_status_t
Definition: crypto.h:136
cryptodev_op_t::n_elts
u32 n_elts
Definition: cryptodev.h:133
vlib_buffer_t::flags
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index,...
Definition: buffer.h:133
vlib_buffer_t
VLIB buffer representation.
Definition: buffer.h:111
vnet_crypto_async_frame_elt_t::digest
u8 * digest
Definition: crypto.h:331