FD.io VPP  v18.01.2-1-g9b554f3
Vector Packet Processing
ipsec.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 Intel and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include <vnet/vnet.h>
16 #include <vnet/ip/ip.h>
17 #include <vnet/api_errno.h>
18 #include <vnet/ipsec/ipsec.h>
19 #include <vlib/node_funcs.h>
20 
21 #include <dpdk/device/dpdk.h>
22 #include <dpdk/ipsec/ipsec.h>
23 
25 
26 #define EMPTY_STRUCT {0}
27 
28 static void
29 algos_init (u32 n_mains)
30 {
32  crypto_alg_t *a;
33 
35 
36  {
37 #define _(v,f,str) \
38  dcm->cipher_algs[IPSEC_CRYPTO_ALG_##f].name = str; \
39  dcm->cipher_algs[IPSEC_CRYPTO_ALG_##f].disabled = n_mains;
41 #undef _
42  }
43 
44  /* Minimum boundary for ciphers is 4B, required by ESP */
45  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_NONE];
46  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
47  a->alg = RTE_CRYPTO_CIPHER_NULL;
48  a->boundary = 4; /* 1 */
49  a->key_len = 0;
50  a->iv_len = 0;
51 
52  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_128];
53  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
54  a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
55  a->boundary = 16;
56  a->key_len = 16;
57  a->iv_len = 16;
58 
59  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_192];
60  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
61  a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
62  a->boundary = 16;
63  a->key_len = 24;
64  a->iv_len = 16;
65 
66  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_256];
67  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
68  a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
69  a->boundary = 16;
70  a->key_len = 32;
71  a->iv_len = 16;
72 
73  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_128];
74  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
75  a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
76  a->boundary = 4; /* 1 */
77  a->key_len = 16;
78  a->iv_len = 8;
79 
80  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_192];
81  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
82  a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
83  a->boundary = 4; /* 1 */
84  a->key_len = 24;
85  a->iv_len = 8;
86 
87  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_256];
88  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
89  a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
90  a->boundary = 4; /* 1 */
91  a->key_len = 32;
92  a->iv_len = 8;
93 
94 #define AES_GCM_TYPE RTE_CRYPTO_SYM_XFORM_AEAD
95 #define AES_GCM_ALG RTE_CRYPTO_AEAD_AES_GCM
96 
97  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_128];
98  a->type = AES_GCM_TYPE;
99  a->alg = AES_GCM_ALG;
100  a->boundary = 4; /* 1 */
101  a->key_len = 16;
102  a->iv_len = 8;
103  a->trunc_size = 16;
104 
105  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_192];
106  a->type = AES_GCM_TYPE;
107  a->alg = AES_GCM_ALG;
108  a->boundary = 4; /* 1 */
109  a->key_len = 24;
110  a->iv_len = 8;
111  a->trunc_size = 16;
112 
113  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_256];
114  a->type = AES_GCM_TYPE;
115  a->alg = AES_GCM_ALG;
116  a->boundary = 4; /* 1 */
117  a->key_len = 32;
118  a->iv_len = 8;
119  a->trunc_size = 16;
120 
122 
123  {
124 #define _(v,f,str) \
125  dcm->auth_algs[IPSEC_INTEG_ALG_##f].name = str; \
126  dcm->auth_algs[IPSEC_INTEG_ALG_##f].disabled = n_mains;
128 #undef _
129  }
130 
131  a = &dcm->auth_algs[IPSEC_INTEG_ALG_NONE];
132  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
133  a->alg = RTE_CRYPTO_AUTH_NULL;
134  a->key_len = 0;
135  a->trunc_size = 0;
136 
137  a = &dcm->auth_algs[IPSEC_INTEG_ALG_MD5_96];
138  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
139  a->alg = RTE_CRYPTO_AUTH_MD5_HMAC;
140  a->key_len = 16;
141  a->trunc_size = 12;
142 
143  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA1_96];
144  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
145  a->alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
146  a->key_len = 20;
147  a->trunc_size = 12;
148 
149  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_256_96];
150  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
151  a->alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
152  a->key_len = 32;
153  a->trunc_size = 12;
154 
155  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_256_128];
156  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
157  a->alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
158  a->key_len = 32;
159  a->trunc_size = 16;
160 
161  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_384_192];
162  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
163  a->alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
164  a->key_len = 48;
165  a->trunc_size = 24;
166 
167  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_512_256];
168  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
169  a->alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
170  a->key_len = 64;
171  a->trunc_size = 32;
172 }
173 
174 static u8
176 {
178 
179  return (alg - dcm->cipher_algs);
180 }
181 
182 static u8
184 {
186 
187  return (alg - dcm->auth_algs);
188 }
189 
190 static crypto_alg_t *
191 cipher_cap_to_alg (const struct rte_cryptodev_capabilities *cap, u8 key_len)
192 {
194  crypto_alg_t *alg;
195 
196  if (cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
197  return NULL;
198 
199  /* *INDENT-OFF* */
200  vec_foreach (alg, dcm->cipher_algs)
201  {
202  if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
203  (alg->type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
204  (cap->sym.cipher.algo == alg->alg) &&
205  (alg->key_len == key_len))
206  return alg;
207  if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) &&
208  (alg->type == RTE_CRYPTO_SYM_XFORM_AEAD) &&
209  (cap->sym.aead.algo == alg->alg) &&
210  (alg->key_len == key_len))
211  return alg;
212  }
213  /* *INDENT-ON* */
214 
215  return NULL;
216 }
217 
218 static crypto_alg_t *
219 auth_cap_to_alg (const struct rte_cryptodev_capabilities *cap, u8 trunc_size)
220 {
222  crypto_alg_t *alg;
223 
224  if ((cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) ||
225  (cap->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH))
226  return NULL;
227 
228  /* *INDENT-OFF* */
229  vec_foreach (alg, dcm->auth_algs)
230  {
231  if ((cap->sym.auth.algo == alg->alg) &&
232  (alg->trunc_size == trunc_size))
233  return alg;
234  }
235  /* *INDENT-ON* */
236 
237  return NULL;
238 }
239 
240 static void
241 crypto_set_aead_xform (struct rte_crypto_sym_xform *xform,
242  ipsec_sa_t * sa, u8 is_outbound)
243 {
245  crypto_alg_t *c;
246 
247  c = vec_elt_at_index (dcm->cipher_algs, sa->crypto_alg);
248 
249  ASSERT (c->type == RTE_CRYPTO_SYM_XFORM_AEAD);
250 
251  xform->type = RTE_CRYPTO_SYM_XFORM_AEAD;
252  xform->aead.algo = c->alg;
253  xform->aead.key.data = sa->crypto_key;
254  xform->aead.key.length = c->key_len;
255  xform->aead.iv.offset =
256  crypto_op_get_priv_offset () + offsetof (dpdk_op_priv_t, cb);
257  xform->aead.iv.length = 12;
258  xform->aead.digest_length = c->trunc_size;
259  xform->aead.aad_length = sa->use_esn ? 12 : 8;
260  xform->next = NULL;
261 
262  if (is_outbound)
263  xform->aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT;
264  else
265  xform->aead.op = RTE_CRYPTO_AEAD_OP_DECRYPT;
266 }
267 
268 static void
269 crypto_set_cipher_xform (struct rte_crypto_sym_xform *xform,
270  ipsec_sa_t * sa, u8 is_outbound)
271 {
273  crypto_alg_t *c;
274 
275  c = vec_elt_at_index (dcm->cipher_algs, sa->crypto_alg);
276 
277  ASSERT (c->type == RTE_CRYPTO_SYM_XFORM_CIPHER);
278 
279  xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
280  xform->cipher.algo = c->alg;
281  xform->cipher.key.data = sa->crypto_key;
282  xform->cipher.key.length = c->key_len;
283  xform->cipher.iv.offset =
284  crypto_op_get_priv_offset () + offsetof (dpdk_op_priv_t, cb);
285  xform->cipher.iv.length = c->iv_len;
286  xform->next = NULL;
287 
288  if (is_outbound)
289  xform->cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
290  else
291  xform->cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
292 }
293 
294 static void
295 crypto_set_auth_xform (struct rte_crypto_sym_xform *xform,
296  ipsec_sa_t * sa, u8 is_outbound)
297 {
299  crypto_alg_t *a;
300 
301  a = vec_elt_at_index (dcm->auth_algs, sa->integ_alg);
302 
303  ASSERT (a->type == RTE_CRYPTO_SYM_XFORM_AUTH);
304 
305  xform->type = RTE_CRYPTO_SYM_XFORM_AUTH;
306  xform->auth.algo = a->alg;
307  xform->auth.key.data = sa->integ_key;
308  xform->auth.key.length = a->key_len;
309  xform->auth.digest_length = a->trunc_size;
310  xform->next = NULL;
311 
312  if (is_outbound)
313  xform->auth.op = RTE_CRYPTO_AUTH_OP_GENERATE;
314  else
315  xform->auth.op = RTE_CRYPTO_AUTH_OP_VERIFY;
316 }
317 
318 clib_error_t *
319 create_sym_session (struct rte_cryptodev_sym_session **session,
320  u32 sa_idx,
321  crypto_resource_t * res,
322  crypto_worker_main_t * cwm, u8 is_outbound)
323 {
325  ipsec_main_t *im = &ipsec_main;
326  crypto_data_t *data;
327  ipsec_sa_t *sa;
328  struct rte_crypto_sym_xform cipher_xform = { 0 };
329  struct rte_crypto_sym_xform auth_xform = { 0 };
330  struct rte_crypto_sym_xform *xfs;
331  struct rte_cryptodev_sym_session **s;
332  crypto_session_key_t key = { 0 };
333 
334  key.drv_id = res->drv_id;
335  key.sa_idx = sa_idx;
336 
337  data = vec_elt_at_index (dcm->data, res->numa);
338 
339  sa = pool_elt_at_index (im->sad, sa_idx);
340 
341  if ((sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) |
342  (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_192) |
343  (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_256))
344  {
345  crypto_set_aead_xform (&cipher_xform, sa, is_outbound);
346  xfs = &cipher_xform;
347  }
348  else
349  {
350  crypto_set_cipher_xform (&cipher_xform, sa, is_outbound);
351  crypto_set_auth_xform (&auth_xform, sa, is_outbound);
352 
353  if (is_outbound)
354  {
355  cipher_xform.next = &auth_xform;
356  xfs = &cipher_xform;
357  }
358  else
359  {
360  auth_xform.next = &cipher_xform;
361  xfs = &auth_xform;
362  }
363  }
364 
365  /*
366  * DPDK_VER >= 1708:
367  * Multiple worker/threads share the session for an SA
368  * Single session per SA, initialized for each device driver
369  */
370  s = (void *) hash_get (data->session_by_sa_index, sa_idx);
371 
372  if (!s)
373  {
374  session[0] = rte_cryptodev_sym_session_create (data->session_h);
375  if (!session[0])
376  {
377  data->session_h_failed += 1;
378  return clib_error_return (0, "failed to create session header");
379  }
380  hash_set (data->session_by_sa_index, sa_idx, session[0]);
381  }
382  else
383  session[0] = s[0];
384 
385  struct rte_mempool **mp;
386  mp = vec_elt_at_index (data->session_drv, res->drv_id);
387  ASSERT (mp[0] != NULL);
388 
389  i32 ret =
390  rte_cryptodev_sym_session_init (res->dev_id, session[0], xfs, mp[0]);
391  if (ret)
392  {
393  data->session_drv_failed[res->drv_id] += 1;
394  return clib_error_return (0, "failed to init session for drv %u",
395  res->drv_id);
396  }
397 
398  hash_set (data->session_by_drv_id_and_sa_index, key.val, session[0]);
399 
400  return 0;
401 }
402 
403 static void __attribute__ ((unused)) clear_and_free_obj (void *obj)
404 {
405  struct rte_mempool *mp = rte_mempool_from_obj (obj);
406 
407  memset (obj, 0, mp->elt_size);
408 
409  rte_mempool_put (mp, obj);
410 }
411 
412 /* This is from rte_cryptodev_pmd.h */
413 static inline void *
414 get_session_private_data (const struct rte_cryptodev_sym_session *sess,
415  uint8_t driver_id)
416 {
417  return sess->sess_private_data[driver_id];
418 }
419 
420 /* This is from rte_cryptodev_pmd.h */
421 static inline void
422 set_session_private_data (struct rte_cryptodev_sym_session *sess,
423  uint8_t driver_id, void *private_data)
424 {
425  sess->sess_private_data[driver_id] = private_data;
426 }
427 
428 static clib_error_t *
430 {
433  void *drv_session;
434  u32 drv_id;
435  i32 ret;
436 
437  /* *INDENT-OFF* */
438  vec_foreach (s, v)
439  {
440  /* ordered vector by timestamp */
441  if (!(s->ts + dcm->session_timeout < ts))
442  break;
443 
444  vec_foreach_index (drv_id, dcm->drv)
445  {
446  drv_session = get_session_private_data (s->session, drv_id);
447  if (!drv_session)
448  continue;
449 
450  /*
451  * Custom clear to avoid finding a dev_id for drv_id:
452  * ret = rte_cryptodev_sym_session_clear (dev_id, drv_session);
453  * ASSERT (!ret);
454  */
455  clear_and_free_obj (drv_session);
456 
457  set_session_private_data (s->session, drv_id, NULL);
458  }
459 
460  ret = rte_cryptodev_sym_session_free (s->session);
461  ASSERT (!ret);
462  }
463  /* *INDENT-ON* */
464 
465  if (s < vec_end (v))
466  vec_delete (v, s - v, 0);
467 
468  return 0;
469 }
470 
471 static clib_error_t *
472 add_del_sa_session (u32 sa_index, u8 is_add)
473 {
474  ipsec_main_t *im = &ipsec_main;
476  crypto_data_t *data;
477  struct rte_cryptodev_sym_session *s;
478  crypto_session_key_t key = { 0 };
479  uword *val;
480  u32 drv_id;
481 
482  if (is_add)
483  {
484 #if 1
485  ipsec_sa_t *sa = pool_elt_at_index (im->sad, sa_index);
486  u32 seed;
487  switch (sa->crypto_alg)
488  {
489  case IPSEC_CRYPTO_ALG_AES_GCM_128:
490  case IPSEC_CRYPTO_ALG_AES_GCM_192:
491  case IPSEC_CRYPTO_ALG_AES_GCM_256:
492  clib_memcpy (&sa->salt, &sa->crypto_key[sa->crypto_key_len - 4], 4);
493  break;
494  default:
495  seed = (u32) clib_cpu_time_now ();
496  sa->salt = random_u32 (&seed);
497  }
498 #endif
499  return 0;
500  }
501 
502  key.sa_idx = sa_index;
503 
504  /* *INDENT-OFF* */
505  vec_foreach (data, dcm->data)
506  {
507  val = hash_get (data->session_by_sa_index, sa_index);
508  s = (struct rte_cryptodev_sym_session *) val;
509 
510  if (!s)
511  continue;
512 
513  vec_foreach_index (drv_id, dcm->drv)
514  {
515  key.drv_id = drv_id;
516  val = hash_get (data->session_by_drv_id_and_sa_index, key.val);
517  s = (struct rte_cryptodev_sym_session *) val;
518 
519  if (!s)
520  continue;
521 
523  }
524 
525  hash_unset (data->session_by_sa_index, sa_index);
526 
527  u64 ts = unix_time_now_nsec ();
529 
531  sd.ts = ts;
532  sd.session = s;
533 
534  vec_add1 (data->session_disposal, sd);
535  }
536  /* *INDENT-ON* */
537 
538  return 0;
539 }
540 
541 static clib_error_t *
543 {
545 
546  if (sa->integ_alg == IPSEC_INTEG_ALG_NONE)
547  switch (sa->crypto_alg)
548  {
549  case IPSEC_CRYPTO_ALG_AES_GCM_128:
550  case IPSEC_CRYPTO_ALG_AES_GCM_192:
551  case IPSEC_CRYPTO_ALG_AES_GCM_256:
552  break;
553  default:
554  return clib_error_return (0, "unsupported integ-alg %U crypto-alg %U",
557  }
558 
559  /* XXX do we need the NONE check? */
560  if (sa->crypto_alg != IPSEC_CRYPTO_ALG_NONE &&
561  dcm->cipher_algs[sa->crypto_alg].disabled)
562  return clib_error_return (0, "disabled crypto-alg %U",
564 
565  /* XXX do we need the NONE check? */
566  if (sa->integ_alg != IPSEC_INTEG_ALG_NONE &&
567  dcm->auth_algs[sa->integ_alg].disabled)
568  return clib_error_return (0, "disabled integ-alg %U",
570  return NULL;
571 }
572 
573 static void
575  const struct rte_cryptodev_capabilities *cap,
576  u32 n_mains)
577 {
579  crypto_alg_t *alg;
580  u8 len, inc;
581 
582  for (; cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; cap++)
583  {
584  /* A single capability maps to multiple cipher/auth algorithms */
585  switch (cap->sym.xform_type)
586  {
587  case RTE_CRYPTO_SYM_XFORM_AEAD:
588  case RTE_CRYPTO_SYM_XFORM_CIPHER:
589  inc = cap->sym.cipher.key_size.increment;
590  inc = inc ? inc : 1;
591  for (len = cap->sym.cipher.key_size.min;
592  len <= cap->sym.cipher.key_size.max; len += inc)
593  {
594  alg = cipher_cap_to_alg (cap, len);
595  if (!alg)
596  continue;
597  dev->cipher_support[cipher_alg_index (alg)] = 1;
598  alg->resources += vec_len (dev->free_resources);
599  /* At least enough resources to support one algo */
600  dcm->enabled |= (alg->resources >= n_mains);
601  }
602  break;
603  case RTE_CRYPTO_SYM_XFORM_AUTH:
604  inc = cap->sym.auth.digest_size.increment;
605  inc = inc ? inc : 1;
606  for (len = cap->sym.auth.digest_size.min;
607  len <= cap->sym.auth.digest_size.max; len += inc)
608  {
609  alg = auth_cap_to_alg (cap, len);
610  if (!alg)
611  continue;
612  dev->auth_support[auth_alg_index (alg)] = 1;
613  alg->resources += vec_len (dev->free_resources);
614  /* At least enough resources to support one algo */
615  dcm->enabled |= (alg->resources >= n_mains);
616  }
617  break;
618  default:
619  ;
620  }
621  }
622 }
623 
624 #define DPDK_CRYPTO_N_QUEUE_DESC 2048
625 #define DPDK_CRYPTO_NB_SESS_OBJS 20000
626 
627 static clib_error_t *
628 crypto_dev_conf (u8 dev, u16 n_qp, u8 numa)
629 {
630  struct rte_cryptodev_config dev_conf;
631  struct rte_cryptodev_qp_conf qp_conf;
632  i32 ret;
633  u16 qp;
634  i8 *error_str;
635 
636  dev_conf.socket_id = numa;
637  dev_conf.nb_queue_pairs = n_qp;
638 
639  error_str = "failed to configure crypto device %u";
640  ret = rte_cryptodev_configure (dev, &dev_conf);
641  if (ret < 0)
642  return clib_error_return (0, error_str, dev);
643 
644  error_str = "failed to setup crypto device %u queue pair %u";
645  qp_conf.nb_descriptors = DPDK_CRYPTO_N_QUEUE_DESC;
646  for (qp = 0; qp < n_qp; qp++)
647  {
648  ret = rte_cryptodev_queue_pair_setup (dev, qp, &qp_conf, numa, NULL);
649  if (ret < 0)
650  return clib_error_return (0, error_str, dev, qp);
651  }
652 
653  return 0;
654 }
655 
656 static void
658 {
660  struct rte_cryptodev *cryptodev;
661  struct rte_cryptodev_info info;
662  crypto_dev_t *dev;
663  crypto_resource_t *res;
664  clib_error_t *error;
665  u32 i;
666  u16 max_res_idx, res_idx, j;
667  u8 drv_id;
668 
669  vec_validate_init_empty (dcm->dev, rte_cryptodev_count () - 1,
671 
672  for (i = 0; i < rte_cryptodev_count (); i++)
673  {
674  dev = vec_elt_at_index (dcm->dev, i);
675 
676  cryptodev = &rte_cryptodevs[i];
677  rte_cryptodev_info_get (i, &info);
678 
679  dev->id = i;
680  dev->name = cryptodev->data->name;
681  dev->numa = rte_cryptodev_socket_id (i);
682  dev->features = info.feature_flags;
683  dev->max_qp = info.max_nb_queue_pairs;
684  drv_id = info.driver_id;
685  if (drv_id >= vec_len (dcm->drv))
686  vec_validate_init_empty (dcm->drv, drv_id,
687  (crypto_drv_t) EMPTY_STRUCT);
688  vec_elt_at_index (dcm->drv, drv_id)->name = info.driver_name;
689  dev->drv_id = drv_id;
690  vec_add1 (vec_elt_at_index (dcm->drv, drv_id)->devs, i);
691 
692  if (!(info.feature_flags & RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING))
693  continue;
694 
695  if ((error = crypto_dev_conf (i, dev->max_qp, dev->numa)))
696  {
697  clib_error_report (error);
698  continue;
699  }
700 
701  max_res_idx = (dev->max_qp / 2) - 1;
702 
703  vec_validate (dev->free_resources, max_res_idx);
704 
705  res_idx = vec_len (dcm->resource);
706  vec_validate_init_empty_aligned (dcm->resource, res_idx + max_res_idx,
707  (crypto_resource_t) EMPTY_STRUCT,
709 
710  for (j = 0; j <= max_res_idx; j++, res_idx++)
711  {
712  vec_elt (dev->free_resources, max_res_idx - j) = res_idx;
713  res = &dcm->resource[res_idx];
714  res->dev_id = i;
715  res->drv_id = drv_id;
716  res->qp_id = j * 2;
717  res->numa = dev->numa;
718  res->thread_idx = (u16) ~ 0;
719  }
720 
721  crypto_parse_capabilities (dev, info.capabilities, n_mains);
722  }
723 }
724 
725 void
727 {
729  crypto_resource_t *res;
731  crypto_dev_t *dev;
732  u32 thread_idx, skip_master;
733  u16 res_idx, *idx;
734  u8 used;
735  u16 i;
736 
737  skip_master = vlib_num_workers () > 0;
738 
739  /* *INDENT-OFF* */
740  vec_foreach (dev, dcm->dev)
741  {
742  vec_foreach_index (thread_idx, dcm->workers_main)
743  {
744  if (vec_len (dev->free_resources) == 0)
745  break;
746 
747  if (thread_idx < skip_master)
748  continue;
749 
750  /* Check thread is not already using the device */
751  vec_foreach (idx, dev->used_resources)
752  if (dcm->resource[idx[0]].thread_idx == thread_idx)
753  continue;
754 
755  cwm = vec_elt_at_index (dcm->workers_main, thread_idx);
756 
757  used = 0;
758  res_idx = vec_pop (dev->free_resources);
759 
760  /* Set device only for supported algos */
761  for (i = 0; i < IPSEC_CRYPTO_N_ALG; i++)
762  if (dev->cipher_support[i] &&
763  cwm->cipher_resource_idx[i] == (u16) ~0)
764  {
765  dcm->cipher_algs[i].disabled--;
766  cwm->cipher_resource_idx[i] = res_idx;
767  used = 1;
768  }
769 
770  for (i = 0; i < IPSEC_INTEG_N_ALG; i++)
771  if (dev->auth_support[i] &&
772  cwm->auth_resource_idx[i] == (u16) ~0)
773  {
774  dcm->auth_algs[i].disabled--;
775  cwm->auth_resource_idx[i] = res_idx;
776  used = 1;
777  }
778 
779  if (!used)
780  {
781  vec_add1 (dev->free_resources, res_idx);
782  continue;
783  }
784 
785  vec_add1 (dev->used_resources, res_idx);
786 
787  res = vec_elt_at_index (dcm->resource, res_idx);
788 
789  ASSERT (res->thread_idx == (u16) ~0);
790  res->thread_idx = thread_idx;
791 
792  /* Add device to vector of polling resources */
793  vec_add1 (cwm->resource_idx, res_idx);
794  }
795  }
796  /* *INDENT-ON* */
797 }
798 
799 static void
800 crypto_op_init (struct rte_mempool *mempool,
801  void *_arg __attribute__ ((unused)),
802  void *_obj, unsigned i __attribute__ ((unused)))
803 {
804  struct rte_crypto_op *op = _obj;
805 
806  op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
807  op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
808  op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
809 #if RTE_VERSION < RTE_VERSION_NUM(17, 11, 0, 0)
810  op->phys_addr = rte_mempool_virt2phy (NULL, _obj);
811 #else
812  op->phys_addr = rte_mempool_virt2iova (_obj);
813 #endif
814  op->mempool = mempool;
815 }
816 
817 static clib_error_t *
819 {
822  crypto_data_t *data;
823  u8 *pool_name;
824  u32 pool_priv_size = sizeof (struct rte_crypto_op_pool_private);
825  struct rte_crypto_op_pool_private *priv;
826  struct rte_mempool *mp;
827  clib_error_t *error = NULL;
829 
830  data = vec_elt_at_index (dcm->data, numa);
831 
832  /* Already allocated */
833  if (data->crypto_op)
834  return NULL;
835 
836  pool_name = format (0, "crypto_pool_numa%u%c", numa, 0);
837 
838  error =
839  dpdk_pool_create (vm, pool_name, crypto_op_len (), conf->num_mbufs,
840  pool_priv_size, 512, numa, &mp, &pri);
841 
842  vec_free (pool_name);
843 
844  if (error)
845  return error;
846 
847  /* Initialize mempool private data */
848  priv = rte_mempool_get_priv (mp);
849  priv->priv_size = pool_priv_size;
850  priv->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
851 
852  /* call the object initializers */
853  rte_mempool_obj_iter (mp, crypto_op_init, 0);
854 
855  data->crypto_op = mp;
856 
857  return NULL;
858 }
859 
860 static clib_error_t *
862 {
864  crypto_data_t *data;
865  u8 *pool_name;
866  struct rte_mempool *mp;
867  clib_error_t *error = NULL;
869  u32 elt_size;
870 
871  data = vec_elt_at_index (dcm->data, numa);
872 
873  if (data->session_h)
874  return NULL;
875 
876  pool_name = format (0, "session_h_pool_numa%u%c", numa, 0);
877 
878  elt_size = rte_cryptodev_get_header_session_size ();
879 
880  error =
881  dpdk_pool_create (vm, pool_name, elt_size, DPDK_CRYPTO_NB_SESS_OBJS,
882  0, 512, numa, &mp, &pri);
883 
884  vec_free (pool_name);
885 
886  if (error)
887  return error;
888 
889  data->session_h = mp;
890 
891  return NULL;
892 }
893 
894 static clib_error_t *
896 {
898  crypto_data_t *data;
899  u8 *pool_name;
900  struct rte_mempool *mp;
901  clib_error_t *error = NULL;
903  u32 elt_size;
904  u8 numa = dev->numa;
905 
906  data = vec_elt_at_index (dcm->data, numa);
907 
908  vec_validate (data->session_drv, dev->drv_id);
909  vec_validate (data->session_drv_failed, dev->drv_id);
910 
911  if (data->session_drv[dev->drv_id])
912  return NULL;
913 
914  pool_name = format (0, "session_drv%u_pool_numa%u%c", dev->drv_id, numa, 0);
915  elt_size = rte_cryptodev_get_private_session_size (dev->id);
916 
917  error =
918  dpdk_pool_create (vm, pool_name, elt_size, DPDK_CRYPTO_NB_SESS_OBJS,
919  0, 512, numa, &mp, &pri);
920 
921  vec_free (pool_name);
922 
923  if (error)
924  return error;
925 
926  data->session_drv[dev->drv_id] = mp;
927 
928  return NULL;
929 }
930 
931 static clib_error_t *
933 {
935  clib_error_t *error = NULL;
936  crypto_dev_t *dev;
937 
938  /* *INDENT-OFF* */
939  vec_foreach (dev, dcm->dev)
940  {
942 
943  error = crypto_create_crypto_op_pool (vm, dev->numa);
944  if (error)
945  return error;
946 
947  error = crypto_create_session_h_pool (vm, dev->numa);
948  if (error)
949  return error;
950 
951  error = crypto_create_session_drv_pool (vm, dev);
952  if (error)
953  return error;
954  }
955  /* *INDENT-ON* */
956 
957  return NULL;
958 }
959 
960 static void
962 {
964  crypto_data_t *data;
965  u8 i;
966 
967  dcm->enabled = 0;
968 
969  /* *INDENT-OFF* */
970  vec_foreach (data, dcm->data)
971  {
972  rte_mempool_free (data->crypto_op);
973  rte_mempool_free (data->session_h);
974 
975  vec_foreach_index (i, data->session_drv)
976  rte_mempool_free (data->session_drv[i]);
977 
978  vec_free (data->session_drv);
979  }
980  /* *INDENT-ON* */
981 
982  vec_free (dcm->data);
983  vec_free (dcm->workers_main);
984  vec_free (dcm->sa_session);
985  vec_free (dcm->dev);
986  vec_free (dcm->resource);
987  vec_free (dcm->cipher_algs);
988  vec_free (dcm->auth_algs);
989 }
990 
991 static uword
993  vlib_frame_t * f)
994 {
995  ipsec_main_t *im = &ipsec_main;
999  clib_error_t *error = NULL;
1000  u32 i, skip_master, n_mains;
1001 
1002  n_mains = tm->n_vlib_mains;
1003  skip_master = vlib_num_workers () > 0;
1004 
1005  algos_init (n_mains - skip_master);
1006 
1007  crypto_scan_devs (n_mains - skip_master);
1008 
1009  if (!(dcm->enabled))
1010  {
1011  clib_warning ("not enough DPDK crypto resources, default to OpenSSL");
1012  crypto_disable ();
1013  return 0;
1014  }
1015 
1016  dcm->session_timeout = 10e9;
1017 
1018  vec_validate_init_empty_aligned (dcm->workers_main, n_mains - 1,
1021 
1022  /* *INDENT-OFF* */
1023  vec_foreach (cwm, dcm->workers_main)
1024  {
1027  memset (cwm->cipher_resource_idx, ~0,
1028  IPSEC_CRYPTO_N_ALG * sizeof(*cwm->cipher_resource_idx));
1029  memset (cwm->auth_resource_idx, ~0,
1030  IPSEC_INTEG_N_ALG * sizeof(*cwm->auth_resource_idx));
1031  }
1032  /* *INDENT-ON* */
1033 
1035 
1036  error = crypto_create_pools (vm);
1037  if (error)
1038  {
1039  clib_error_report (error);
1040  crypto_disable ();
1041  return 0;
1042  }
1043 
1044  /* Add new next node and set it as default */
1045  vlib_node_t *node, *next_node;
1046 
1047  next_node = vlib_get_node_by_name (vm, (u8 *) "dpdk-esp-encrypt");
1048  ASSERT (next_node);
1049  node = vlib_get_node_by_name (vm, (u8 *) "ipsec-output-ip4");
1050  ASSERT (node);
1051  im->esp_encrypt_node_index = next_node->index;
1053  vlib_node_add_next (vm, node->index, next_node->index);
1054 
1055  next_node = vlib_get_node_by_name (vm, (u8 *) "dpdk-esp-decrypt");
1056  ASSERT (next_node);
1057  node = vlib_get_node_by_name (vm, (u8 *) "ipsec-input-ip4");
1058  ASSERT (node);
1059  im->esp_decrypt_node_index = next_node->index;
1061  vlib_node_add_next (vm, node->index, next_node->index);
1062 
1065 
1066  node = vlib_get_node_by_name (vm, (u8 *) "dpdk-crypto-input");
1067  ASSERT (node);
1068  for (i = skip_master; i < n_mains; i++)
1069  vlib_node_set_state (vlib_mains[i], node->index, VLIB_NODE_STATE_POLLING);
1070  return 0;
1071 }
1072 
1073 /* *INDENT-OFF* */
1074 VLIB_REGISTER_NODE (dpdk_ipsec_process_node,static) = {
1075  .function = dpdk_ipsec_process,
1076  .type = VLIB_NODE_TYPE_PROCESS,
1077  .name = "dpdk-ipsec-process",
1078  .process_log2_n_stack_bytes = 17,
1079 };
1080 /* *INDENT-ON* */
1081 
1082 /*
1083  * fd.io coding-style-patch-verification: ON
1084  *
1085  * Local Variables:
1086  * eval: (c-set-style "gnu")
1087  * End:
1088  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:432
u32 alg
Definition: ipsec.h:77
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define hash_set(h, key, value)
Definition: hash.h:254
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:337
#define hash_unset(h, key)
Definition: hash.h:260
a
Definition: bitmap.h:516
u8 numa
Definition: ipsec.h:93
static u8 cipher_alg_index(const crypto_alg_t *alg)
Definition: ipsec.c:175
#define NULL
Definition: clib.h:55
u32 index
Definition: node.h:237
u64 session_h_failed
Definition: ipsec.h:136
#define DPDK_CRYPTO_NB_SESS_OBJS
Definition: ipsec.c:625
ipsec_integ_alg_t integ_alg
Definition: ipsec.h:121
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:518
static clib_error_t * crypto_create_session_drv_pool(vlib_main_t *vm, crypto_dev_t *dev)
Definition: ipsec.c:895
static u64 clib_cpu_time_now(void)
Definition: time.h:73
#define EMPTY_STRUCT
Definition: ipsec.c:26
static_always_inline u32 crypto_op_get_priv_offset(void)
Definition: ipsec.h:176
struct rte_cryptodev_sym_session * session
Definition: ipsec.h:124
u8 disabled
Definition: ipsec.h:82
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:443
static void crypto_set_cipher_xform(struct rte_crypto_sym_xform *xform, ipsec_sa_t *sa, u8 is_outbound)
Definition: ipsec.c:269
vlib_main_t ** vlib_mains
Definition: buffer.c:292
static clib_error_t * crypto_create_session_h_pool(vlib_main_t *vm, u8 numa)
Definition: ipsec.c:861
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1108
#define vec_pop(V)
Returns last element of a vector and decrements its length.
Definition: vec.h:612
u8 crypto_key[128]
Definition: ipsec.h:119
static crypto_alg_t * cipher_cap_to_alg(const struct rte_cryptodev_capabilities *cap, u8 key_len)
Definition: ipsec.c:191
static clib_error_t * add_del_sa_session(u32 sa_index, u8 is_add)
Definition: ipsec.c:472
u16 * resource_idx
Definition: ipsec.h:67
#define AES_GCM_TYPE
u8 integ_key[128]
Definition: ipsec.h:123
u16 cipher_resource_idx[IPSEC_CRYPTO_N_ALG]
Definition: ipsec.h:69
u32 esp_encrypt_next_index
Definition: ipsec.h:288
static void algos_init(u32 n_mains)
Definition: ipsec.c:29
dpdk_crypto_main_t dpdk_crypto_main
Definition: ipsec.c:24
dpdk_config_main_t dpdk_config_main
Definition: init.c:40
ipsec_main_t ipsec_main
Definition: ipsec.c:28
u8 resources
Definition: ipsec.h:83
u8 use_esn
Definition: ipsec.h:125
int i32
Definition: types.h:81
crypto_drv_t * drv
Definition: ipsec.h:149
char i8
Definition: types.h:45
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
u8 drv_id
Definition: ipsec.h:92
#define clib_error_return(e, args...)
Definition: error.h:99
static void crypto_parse_capabilities(crypto_dev_t *dev, const struct rte_cryptodev_capabilities *cap, u32 n_mains)
Definition: ipsec.c:574
ipsec_main_callbacks_t cb
Definition: ipsec.h:294
unsigned long u64
Definition: types.h:89
u16 * free_resources
Definition: ipsec.h:88
#define vec_end(v)
End (last data address) of vector.
crypto_alg_t * auth_algs
Definition: ipsec.h:147
u8 * format_ipsec_crypto_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:58
static clib_error_t * crypto_dev_conf(u8 dev, u16 n_qp, u8 numa)
Definition: ipsec.c:628
u8 trunc_size
Definition: ipsec.h:80
static void crypto_disable(void)
Definition: ipsec.c:961
static void * get_session_private_data(const struct rte_cryptodev_sym_session *sess, uint8_t driver_id)
Definition: ipsec.c:414
u16 * used_resources
Definition: ipsec.h:89
#define hash_get(h, key)
Definition: hash.h:248
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:459
crypto_alg_t * cipher_algs
Definition: ipsec.h:146
clib_error_t *(* check_support_cb)(ipsec_sa_t *sa)
Definition: ipsec.h:251
u32 salt
Definition: ipsec.h:133
#define v
Definition: acl.c:341
#define VLIB_FRAME_SIZE
Definition: node.h:328
u64 session_timeout
Definition: ipsec.h:150
static_always_inline u32 crypto_op_len(void)
Definition: ipsec.h:166
#define AES_GCM_ALG
#define foreach_ipsec_integ_alg
Definition: ipsec.h:88
static clib_error_t * crypto_create_crypto_op_pool(vlib_main_t *vm, u8 numa)
Definition: ipsec.c:818
static u8 auth_alg_index(const crypto_alg_t *alg)
Definition: ipsec.c:183
struct rte_mempool ** session_drv
Definition: ipsec.h:131
clib_error_t * dpdk_pool_create(vlib_main_t *vm, u8 *pool_name, u32 elt_size, u32 num_elts, u32 pool_priv_size, u16 cache_size, u8 numa, struct rte_mempool **_mp, vlib_physmem_region_index_t *pri)
Definition: buffer.c:461
vlib node functions
#define foreach_ipsec_crypto_alg
Definition: ipsec.h:66
svmdb_client_t * c
vlib_main_t * vm
Definition: buffer.c:283
u32 esp_encrypt_node_index
Definition: ipsec.h:283
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:336
u32 esp_decrypt_next_index
Definition: ipsec.h:289
#define clib_warning(format, args...)
Definition: error.h:59
#define clib_memcpy(a, b, c)
Definition: string.h:75
static u64 unix_time_now_nsec(void)
Definition: time.h:238
crypto_session_disposal_t * session_disposal
Definition: ipsec.h:132
static void crypto_set_aead_xform(struct rte_crypto_sym_xform *xform, ipsec_sa_t *sa, u8 is_outbound)
Definition: ipsec.c:241
vlib_node_t * vlib_get_node_by_name(vlib_main_t *vm, u8 *name)
Definition: node.c:45
u16 id
Definition: ipsec.h:94
static void set_session_private_data(struct rte_cryptodev_sym_session *sess, uint8_t driver_id, void *private_data)
Definition: ipsec.c:422
u8 iv_len
Definition: ipsec.h:79
#define ASSERT(truth)
unsigned int u32
Definition: types.h:88
#define vec_delete(V, N, M)
Delete N elements starting at element M.
Definition: vec.h:781
struct rte_mempool * crypto_op
Definition: ipsec.h:129
ipsec_sa_t * sad
Definition: ipsec.h:258
static clib_error_t * dpdk_crypto_session_disposal(crypto_session_disposal_t *v, u64 ts)
Definition: ipsec.c:429
crypto_worker_main_t * workers_main
Definition: ipsec.h:142
#define clib_error_report(e)
Definition: error.h:113
static void vlib_node_set_state(vlib_main_t *vm, u32 node_index, vlib_node_state_t new_state)
Set node dispatch state.
Definition: node_funcs.h:147
crypto_resource_t * resource
Definition: ipsec.h:145
u64 uword
Definition: types.h:112
#define vec_elt(v, i)
Get vector value at index i.
u8 * format_ipsec_integ_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:90
u8 crypto_key_len
Definition: ipsec.h:118
static clib_error_t * crypto_create_pools(vlib_main_t *vm)
Definition: ipsec.c:932
struct rte_mempool * session_h
Definition: ipsec.h:130
unsigned short u16
Definition: types.h:57
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
void crypto_auto_placement(void)
Definition: ipsec.c:726
unsigned char u8
Definition: types.h:56
u64 * session_drv_failed
Definition: ipsec.h:137
uword * session_by_sa_index
Definition: ipsec.h:133
static void crypto_scan_devs(u32 n_mains)
Definition: ipsec.c:657
static void crypto_op_init(struct rte_mempool *mempool, void *_arg, void *_obj, unsigned i)
Definition: ipsec.c:800
struct rte_crypto_op ** ops
Definition: ipsec.h:68
clib_error_t * create_sym_session(struct rte_cryptodev_sym_session **session, u32 sa_idx, crypto_resource_t *res, crypto_worker_main_t *cwm, u8 is_outbound)
Definition: ipsec.c:319
crypto_dev_t * dev
Definition: ipsec.h:144
enum rte_crypto_sym_xform_type type
Definition: ipsec.h:76
#define DPDK_CRYPTO_N_QUEUE_DESC
Definition: ipsec.c:624
crypto_data_t * data
Definition: ipsec.h:148
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
ipsec_crypto_alg_t crypto_alg
Definition: ipsec.h:117
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:143
uword * session_by_drv_id_and_sa_index
Definition: ipsec.h:134
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
static u32 vlib_num_workers()
Definition: threads.h:368
static uword dpdk_ipsec_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: ipsec.c:992
u16 auth_resource_idx[IPSEC_INTEG_N_ALG]
Definition: ipsec.h:70
u8 auth_support[IPSEC_INTEG_N_ALG]
Definition: ipsec.h:91
#define vec_validate_init_empty_aligned(V, I, INIT, A)
Make sure vector is long enough for given index and initialize empty space (no header, alignment alignment)
Definition: vec.h:494
#define vec_foreach(var, vec)
Vector iterator.
clib_error_t *(* add_del_sa_sess_cb)(u32 sa_index, u8 is_add)
Definition: ipsec.h:250
u8 key_len
Definition: ipsec.h:78
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
Definition: vec.h:481
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
u8 vlib_physmem_region_index_t
Definition: physmem.h:43
struct rte_cryptodev_sym_session ** sa_session
Definition: ipsec.h:143
static void crypto_set_auth_xform(struct rte_crypto_sym_xform *xform, ipsec_sa_t *sa, u8 is_outbound)
Definition: ipsec.c:295
static clib_error_t * dpdk_ipsec_check_support(ipsec_sa_t *sa)
Definition: ipsec.c:542
u32 esp_decrypt_node_index
Definition: ipsec.h:284
u8 cipher_support[IPSEC_CRYPTO_N_ALG]
Definition: ipsec.h:90
static void clear_and_free_obj(void *obj)
Definition: ipsec.c:403
static crypto_alg_t * auth_cap_to_alg(const struct rte_cryptodev_capabilities *cap, u8 trunc_size)
Definition: ipsec.c:219