FD.io VPP  v21.01.1
Vector Packet Processing
esp_encrypt.c
Go to the documentation of this file.
1 /*
2  * esp_encrypt.c : IPSec ESP encrypt node using DPDK Cryptodev
3  *
4  * Copyright (c) 2017 Intel and/or its affiliates.
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at:
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  */
17 
18 #include <vnet/vnet.h>
19 #include <vnet/api_errno.h>
20 #include <vnet/ip/ip.h>
21 
22 #include <vnet/ipsec/ipsec.h>
23 #include <vnet/ipsec/esp.h>
24 #include <vnet/udp/udp_local.h>
25 #include <dpdk/buffer.h>
26 #include <dpdk/ipsec/ipsec.h>
27 #include <vnet/ipsec/ipsec_tun.h>
28 #include <dpdk/device/dpdk.h>
29 #include <dpdk/device/dpdk_priv.h>
30 
31 #define foreach_esp_encrypt_next \
32 _(DROP, "error-drop") \
33 _(IP4_LOOKUP, "ip4-lookup") \
34 _(IP6_LOOKUP, "ip6-lookup") \
35 _(INTERFACE_OUTPUT, "interface-output")
36 
37 #define _(v, s) ESP_ENCRYPT_NEXT_##v,
38 typedef enum
39 {
41 #undef _
44 
45 #define foreach_esp_encrypt_error \
46  _(RX_PKTS, "ESP pkts received") \
47  _(SEQ_CYCLED, "Sequence number cycled") \
48  _(ENQ_FAIL, "Enqueue encrypt failed (queue full)") \
49  _(DISCARD, "Not enough crypto operations") \
50  _(SESSION, "Failed to get crypto session") \
51  _(NOSUP, "Cipher/Auth not supported")
52 
53 
54 typedef enum
55 {
56 #define _(sym,str) ESP_ENCRYPT_ERROR_##sym,
58 #undef _
61 
62 static char *esp_encrypt_error_strings[] = {
63 #define _(sym,string) string,
65 #undef _
66 };
67 
72 
73 typedef struct
74 {
75  ipsec_crypto_alg_t crypto_alg;
76  ipsec_integ_alg_t integ_alg;
77  u8 packet_data[64];
79 
80 /* packet trace format function */
81 static u8 *
82 format_esp_encrypt_trace (u8 * s, va_list * args)
83 {
84  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
85  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
86  esp_encrypt_trace_t *t = va_arg (*args, esp_encrypt_trace_t *);
88  u32 indent = format_get_indent (s), offset;
89 
90  s = format (s, "cipher %U auth %U\n",
93 
94  if ((ih4->ip_version_and_header_length & 0xF0) == 0x60)
95  {
96  s = format (s, "%U%U", format_white_space, indent,
97  format_ip6_header, ih4);
98  offset = sizeof (ip6_header_t);
99  }
100  else
101  {
102  s = format (s, "%U%U", format_white_space, indent,
103  format_ip4_header, ih4);
104  offset = ip4_header_bytes (ih4);
105  }
106 
107  s = format (s, "\n%U%U", format_white_space, indent,
109 
110  return s;
111 }
112 
116  vlib_frame_t * from_frame, int is_ip6, int is_tun)
117 {
118  u32 n_left_from, *from, *to_next, next_index, thread_index;
119  ipsec_main_t *im = &ipsec_main;
120  vnet_main_t *vnm = im->vnet_main;
124  crypto_resource_t *res = 0;
125  ipsec_sa_t *sa0 = 0;
126  crypto_alg_t *cipher_alg = 0, *auth_alg = 0;
127  struct rte_cryptodev_sym_session *session = 0;
128  u32 ret, last_sa_index = ~0;
129  u8 numa = rte_socket_id ();
130  u8 is_aead = 0;
131  crypto_worker_main_t *cwm =
132  vec_elt_at_index (dcm->workers_main, thread_idx);
133  struct rte_crypto_op **ops = cwm->ops;
134 
135  from = vlib_frame_vector_args (from_frame);
136  n_left_from = from_frame->n_vectors;
137  thread_index = vm->thread_index;
138 
139  ret = crypto_alloc_ops (numa, ops, n_left_from);
140  if (ret)
141  {
142  if (is_ip6)
144  ESP_ENCRYPT_ERROR_DISCARD, n_left_from);
145  else
147  ESP_ENCRYPT_ERROR_DISCARD, n_left_from);
148  /* Discard whole frame */
149  vlib_buffer_free (vm, from, n_left_from);
150  return n_left_from;
151  }
152 
153  next_index = ESP_ENCRYPT_NEXT_DROP;
154 
155  while (n_left_from > 0)
156  {
157  u32 n_left_to_next;
158 
159  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
160 
161  while (n_left_from > 0 && n_left_to_next > 0)
162  {
164  u32 bi0, bi1;
165  vlib_buffer_t *b0, *b1;
166  u32 sa_index0;
167  ip4_and_esp_header_t *ih0, *oh0 = 0;
168  ip6_and_esp_header_t *ih6_0, *oh6_0 = 0;
169  ip4_and_udp_and_esp_header_t *ouh0 = 0;
170  esp_header_t *esp0;
171  esp_footer_t *f0;
172  u8 next_hdr_type;
173  u32 iv_size;
174  u16 orig_sz;
175  u8 trunc_size;
176  u16 rewrite_len;
177  u16 udp_encap_adv = 0;
178  struct rte_mbuf *mb0;
179  struct rte_crypto_op *op;
180  u16 res_idx;
181 
182  bi0 = from[0];
183  from += 1;
184  n_left_from -= 1;
185 
186  b0 = vlib_get_buffer (vm, bi0);
187  ih0 = vlib_buffer_get_current (b0);
188  mb0 = rte_mbuf_from_vlib_buffer (b0);
189 
190  /* ih0/ih6_0 */
191  CLIB_PREFETCH (ih0, sizeof (ih6_0[0]), LOAD);
192  /* f0 */
193  CLIB_PREFETCH (vlib_buffer_get_tail (b0), 20, STORE);
194  /* mb0 */
195  CLIB_PREFETCH (mb0, CLIB_CACHE_LINE_BYTES, STORE);
196 
197  if (n_left_from > 1)
198  {
199  bi1 = from[1];
200  b1 = vlib_get_buffer (vm, bi1);
201 
204  CLIB_CACHE_LINE_BYTES, STORE);
205  }
206 
207  op = ops[0];
208  ops += 1;
209  ASSERT (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED);
210 
211  dpdk_op_priv_t *priv = crypto_op_get_priv (op);
212  /* store bi in op private */
213  priv->bi = bi0;
214  priv->encrypt = 1;
215 
216  u16 op_len =
217  sizeof (op[0]) + sizeof (op[0].sym[0]) + sizeof (priv[0]);
218  CLIB_PREFETCH (op, op_len, STORE);
219 
220  if (is_tun)
221  {
222  /* we are on a ipsec tunnel's feature arc */
223  vnet_buffer (b0)->ipsec.sad_index =
224  sa_index0 = ipsec_tun_protect_get_sa_out
225  (vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
226  }
227  else
228  sa_index0 = vnet_buffer (b0)->ipsec.sad_index;
229 
230  if (sa_index0 != last_sa_index)
231  {
232  sa0 = pool_elt_at_index (im->sad, sa_index0);
233 
234  cipher_alg =
236  auth_alg = vec_elt_at_index (dcm->auth_algs, sa0->integ_alg);
237 
238  is_aead = (cipher_alg->type == RTE_CRYPTO_SYM_XFORM_AEAD);
239 
240  if (is_aead)
241  auth_alg = cipher_alg;
242 
243  res_idx = get_resource (cwm, sa0);
244 
245  if (PREDICT_FALSE (res_idx == (u16) ~ 0))
246  {
247  if (is_ip6)
250  ESP_ENCRYPT_ERROR_NOSUP, 1);
251  else
254  ESP_ENCRYPT_ERROR_NOSUP, 1);
255  to_next[0] = bi0;
256  to_next += 1;
257  n_left_to_next -= 1;
258  goto trace;
259  }
260  res = vec_elt_at_index (dcm->resource, res_idx);
261 
262  error = crypto_get_session (&session, sa_index0, res, cwm, 1);
263  if (PREDICT_FALSE (error || !session))
264  {
265  if (is_ip6)
268  ESP_ENCRYPT_ERROR_SESSION,
269  1);
270  else
273  ESP_ENCRYPT_ERROR_SESSION,
274  1);
275  to_next[0] = bi0;
276  to_next += 1;
277  n_left_to_next -= 1;
278  goto trace;
279  }
280 
281  last_sa_index = sa_index0;
282  }
283 
284  if (PREDICT_FALSE (esp_seq_advance (sa0)))
285  {
286  if (is_ip6)
289  ESP_ENCRYPT_ERROR_SEQ_CYCLED, 1);
290  else
293  ESP_ENCRYPT_ERROR_SEQ_CYCLED, 1);
294  //TODO: rekey SA
295  to_next[0] = bi0;
296  to_next += 1;
297  n_left_to_next -= 1;
298  goto trace;
299  }
300 
301  orig_sz = b0->current_length;
302 
303  /* TODO multi-seg support - total_length_not_including_first_buffer */
305  (&ipsec_sa_counters, thread_index, sa_index0,
306  1, b0->current_length);
307 
308  /* Update tunnel interface tx counters */
309  if (is_tun)
312  thread_index, vnet_buffer (b0)->sw_if_index[VLIB_TX],
313  1, b0->current_length);
314 
315  res->ops[res->n_ops] = op;
316  res->bi[res->n_ops] = bi0;
317  res->n_ops += 1;
318 
319  dpdk_gcm_cnt_blk *icb = &priv->cb;
320 
321  crypto_set_icb (icb, sa0->salt, sa0->seq, sa0->seq_hi);
322 
323  iv_size = cipher_alg->iv_len;
324  trunc_size = auth_alg->trunc_size;
325 
326  /* if UDP encapsulation is used adjust the address of the IP header */
327  if (ipsec_sa_is_set_UDP_ENCAP (sa0) && !is_ip6)
328  udp_encap_adv = sizeof (udp_header_t);
329 
330  if (ipsec_sa_is_set_IS_TUNNEL (sa0))
331  {
332  rewrite_len = 0;
333  if (!ipsec_sa_is_set_IS_TUNNEL_V6 (sa0)) /* ip4 */
334  {
335  /* in tunnel mode send it back to FIB */
336  priv->next = DPDK_CRYPTO_INPUT_NEXT_IP4_LOOKUP;
337  u8 adv = sizeof (ip4_header_t) + udp_encap_adv +
338  sizeof (esp_header_t) + iv_size;
339  vlib_buffer_advance (b0, -adv);
340  oh0 = vlib_buffer_get_current (b0);
341  ouh0 = vlib_buffer_get_current (b0);
342  next_hdr_type = (is_ip6 ?
343  IP_PROTOCOL_IPV6 : IP_PROTOCOL_IP_IN_IP);
344  /*
345  * oh0->ip4.ip_version_and_header_length = 0x45;
346  * oh0->ip4.tos = ih0->ip4.tos;
347  * oh0->ip4.fragment_id = 0;
348  * oh0->ip4.flags_and_fragment_offset = 0;
349  */
350  oh0->ip4.checksum_data_64[0] =
351  clib_host_to_net_u64 (0x45ULL << 56);
352  /*
353  * oh0->ip4.ttl = 254;
354  * oh0->ip4.protocol = IP_PROTOCOL_IPSEC_ESP;
355  */
356  oh0->ip4.checksum_data_32[2] =
357  clib_host_to_net_u32 (0xfe320000);
358 
359  oh0->ip4.src_address.as_u32 =
360  sa0->tunnel_src_addr.ip4.as_u32;
361  oh0->ip4.dst_address.as_u32 =
362  sa0->tunnel_dst_addr.ip4.as_u32;
363 
364  if (ipsec_sa_is_set_UDP_ENCAP (sa0))
365  {
366  oh0->ip4.protocol = IP_PROTOCOL_UDP;
367  esp0 = &ouh0->esp;
368  }
369  else
370  esp0 = &oh0->esp;
371  esp0->spi = clib_host_to_net_u32 (sa0->spi);
372  esp0->seq = clib_host_to_net_u32 (sa0->seq);
373  }
374  else
375  {
376  /* ip6 */
377  /* in tunnel mode send it back to FIB */
378  priv->next = DPDK_CRYPTO_INPUT_NEXT_IP6_LOOKUP;
379 
380  u8 adv =
381  sizeof (ip6_header_t) + sizeof (esp_header_t) + iv_size;
382  vlib_buffer_advance (b0, -adv);
383  ih6_0 = (ip6_and_esp_header_t *) ih0;
384  oh6_0 = vlib_buffer_get_current (b0);
385 
386  next_hdr_type = (is_ip6 ?
387  IP_PROTOCOL_IPV6 : IP_PROTOCOL_IP_IN_IP);
388 
389  oh6_0->ip6.ip_version_traffic_class_and_flow_label =
390  ih6_0->ip6.ip_version_traffic_class_and_flow_label;
391 
392  oh6_0->ip6.protocol = IP_PROTOCOL_IPSEC_ESP;
393  oh6_0->ip6.hop_limit = 254;
394  oh6_0->ip6.src_address.as_u64[0] =
395  sa0->tunnel_src_addr.ip6.as_u64[0];
396  oh6_0->ip6.src_address.as_u64[1] =
397  sa0->tunnel_src_addr.ip6.as_u64[1];
398  oh6_0->ip6.dst_address.as_u64[0] =
399  sa0->tunnel_dst_addr.ip6.as_u64[0];
400  oh6_0->ip6.dst_address.as_u64[1] =
401  sa0->tunnel_dst_addr.ip6.as_u64[1];
402  esp0 = &oh6_0->esp;
403  oh6_0->esp.spi = clib_host_to_net_u32 (sa0->spi);
404  oh6_0->esp.seq = clib_host_to_net_u32 (sa0->seq);
405  }
406 
407  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
408  }
409  else /* transport mode */
410  {
411  if (is_tun)
412  {
413  rewrite_len = 0;
414  priv->next = DPDK_CRYPTO_INPUT_NEXT_MIDCHAIN;
415  }
416  else
417  {
418  priv->next = DPDK_CRYPTO_INPUT_NEXT_INTERFACE_OUTPUT;
419  rewrite_len = vnet_buffer (b0)->ip.save_rewrite_length;
420  }
421  u16 adv = sizeof (esp_header_t) + iv_size + udp_encap_adv;
422  vlib_buffer_advance (b0, -adv - rewrite_len);
423  u8 *src = ((u8 *) ih0) - rewrite_len;
425  oh0 = vlib_buffer_get_current (b0) + rewrite_len;
426  ouh0 = vlib_buffer_get_current (b0) + rewrite_len;
427 
428  if (is_ip6)
429  {
430  orig_sz -= sizeof (ip6_header_t);
431  ih6_0 = (ip6_and_esp_header_t *) ih0;
432  next_hdr_type = ih6_0->ip6.protocol;
433  memmove (dst, src, rewrite_len + sizeof (ip6_header_t));
434  oh6_0 = (ip6_and_esp_header_t *) oh0;
435  oh6_0->ip6.protocol = IP_PROTOCOL_IPSEC_ESP;
436  esp0 = &oh6_0->esp;
437  }
438  else /* ipv4 */
439  {
440  u16 ip_size = ip4_header_bytes (&ih0->ip4);
441  orig_sz -= ip_size;
442  next_hdr_type = ih0->ip4.protocol;
443  memmove (dst, src, rewrite_len + ip_size);
444  oh0->ip4.protocol = IP_PROTOCOL_IPSEC_ESP;
445  esp0 = (esp_header_t *) (((u8 *) oh0) + ip_size);
446  if (ipsec_sa_is_set_UDP_ENCAP (sa0))
447  {
448  oh0->ip4.protocol = IP_PROTOCOL_UDP;
449  esp0 = (esp_header_t *)
450  (((u8 *) oh0) + ip_size + udp_encap_adv);
451  }
452  else
453  {
454  oh0->ip4.protocol = IP_PROTOCOL_IPSEC_ESP;
455  esp0 = (esp_header_t *) (((u8 *) oh0) + ip_size);
456  }
457  }
458  esp0->spi = clib_host_to_net_u32 (sa0->spi);
459  esp0->seq = clib_host_to_net_u32 (sa0->seq);
460  }
461 
462  if (ipsec_sa_is_set_UDP_ENCAP (sa0) && ouh0)
463  {
464  ouh0->udp.src_port = clib_host_to_net_u16 (UDP_DST_PORT_ipsec);
465  ouh0->udp.dst_port = clib_host_to_net_u16 (UDP_DST_PORT_ipsec);
466  ouh0->udp.checksum = 0;
467  }
468  ASSERT (is_pow2 (cipher_alg->boundary));
469  u16 mask = cipher_alg->boundary - 1;
470  u16 pad_payload_len = ((orig_sz + 2) + mask) & ~mask;
471  u8 pad_bytes = pad_payload_len - 2 - orig_sz;
472 
473  u8 *padding =
474  vlib_buffer_put_uninit (b0, pad_bytes + 2 + trunc_size);
475 
476  /* The extra pad bytes would be overwritten by the digest */
477  if (pad_bytes)
478  clib_memcpy_fast (padding, pad_data, 16);
479 
480  f0 = (esp_footer_t *) (padding + pad_bytes);
481  f0->pad_length = pad_bytes;
482  f0->next_header = next_hdr_type;
483 
484  if (oh6_0)
485  {
486  u16 len = b0->current_length - sizeof (ip6_header_t);
487  oh6_0->ip6.payload_length =
488  clib_host_to_net_u16 (len - rewrite_len);
489  }
490  else if (oh0)
491  {
492  oh0->ip4.length =
493  clib_host_to_net_u16 (b0->current_length - rewrite_len);
494  oh0->ip4.checksum = ip4_header_checksum (&oh0->ip4);
495  if (ipsec_sa_is_set_UDP_ENCAP (sa0) && ouh0)
496  {
497  ouh0->udp.length =
498  clib_host_to_net_u16 (clib_net_to_host_u16
499  (ouh0->ip4.length) -
500  ip4_header_bytes (&ouh0->ip4));
501  }
502  }
503  else /* should never happen */
504  clib_warning ("No outer header found for ESP packet");
505 
506  b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
507 
508  /* mbuf packet starts at ESP header */
509  mb0->data_len = vlib_buffer_get_tail (b0) - ((u8 *) esp0);
510  mb0->pkt_len = vlib_buffer_get_tail (b0) - ((u8 *) esp0);
511  mb0->data_off = ((void *) esp0) - mb0->buf_addr;
512 
513  u32 cipher_off, cipher_len, auth_len = 0;
514  u32 *aad = NULL;
515 
516  u8 *digest = vlib_buffer_get_tail (b0) - trunc_size;
517  u64 digest_paddr = mb0->buf_iova + digest - ((u8 *) mb0->buf_addr);
518 
519  if (!is_aead && (cipher_alg->alg == RTE_CRYPTO_CIPHER_AES_CBC ||
520  cipher_alg->alg == RTE_CRYPTO_CIPHER_NULL))
521  {
522  cipher_off = sizeof (esp_header_t);
523  cipher_len = iv_size + pad_payload_len;
524  }
525  else /* CTR/GCM */
526  {
527  u32 *esp_iv = (u32 *) (esp0 + 1);
528  esp_iv[0] = sa0->seq;
529  esp_iv[1] = sa0->seq_hi;
530 
531  cipher_off = sizeof (esp_header_t) + iv_size;
532  cipher_len = pad_payload_len;
533  }
534 
535  if (is_aead)
536  {
537  aad = (u32 *) priv->aad;
538  aad[0] = esp0->spi;
539 
540  /* aad[3] should always be 0 */
541  if (PREDICT_FALSE (ipsec_sa_is_set_USE_ESN (sa0)))
542  {
543  aad[1] = clib_host_to_net_u32 (sa0->seq_hi);
544  aad[2] = esp0->seq;
545  }
546  else
547  {
548  aad[1] = esp0->seq;
549  aad[2] = 0;
550  }
551  }
552  else
553  {
554  auth_len =
555  vlib_buffer_get_tail (b0) - ((u8 *) esp0) - trunc_size;
556  if (ipsec_sa_is_set_USE_ESN (sa0))
557  {
558  u32 *_digest = (u32 *) digest;
559  _digest[0] = clib_host_to_net_u32 (sa0->seq_hi);
560  auth_len += 4;
561  }
562  }
563 
564  crypto_op_setup (is_aead, mb0, op, session, cipher_off, cipher_len,
565  0, auth_len, (u8 *) aad, digest, digest_paddr);
566 
567  trace:
568  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
569  {
570  esp_encrypt_trace_t *tr =
571  vlib_add_trace (vm, node, b0, sizeof (*tr));
572  tr->crypto_alg = sa0->crypto_alg;
573  tr->integ_alg = sa0->integ_alg;
574  u8 *p = vlib_buffer_get_current (b0);
575  if (!ipsec_sa_is_set_IS_TUNNEL (sa0) && !is_tun)
576  p += vnet_buffer (b0)->ip.save_rewrite_length;
577  clib_memcpy_fast (tr->packet_data, p, sizeof (tr->packet_data));
578  }
579  }
580  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
581  }
582  if (is_ip6)
583  {
585  (is_tun ?
587  dpdk_esp6_encrypt_node.index),
588  ESP_ENCRYPT_ERROR_RX_PKTS,
589  from_frame->n_vectors);
590 
592  ESP_ENCRYPT_ERROR_ENQ_FAIL, numa, 1 /* encrypt */ );
593  }
594  else
595  {
597  (is_tun ?
599  dpdk_esp4_encrypt_node.index),
600  ESP_ENCRYPT_ERROR_RX_PKTS,
601  from_frame->n_vectors);
602 
604  ESP_ENCRYPT_ERROR_ENQ_FAIL, numa, 1 /* encrypt */ );
605  }
606 
607  crypto_free_ops (numa, ops, cwm->ops + from_frame->n_vectors - ops);
608 
609  return from_frame->n_vectors;
610 }
611 
614  vlib_frame_t * from_frame)
615 {
616  return dpdk_esp_encrypt_inline (vm, node, from_frame, 0 /*is_ip6 */ , 0);
617 }
618 
619 /* *INDENT-OFF* */
621  .name = "dpdk-esp4-encrypt",
622  .flags = VLIB_NODE_FLAG_IS_OUTPUT,
623  .vector_size = sizeof (u32),
624  .format_trace = format_esp_encrypt_trace,
625  .n_errors = ARRAY_LEN (esp_encrypt_error_strings),
626  .error_strings = esp_encrypt_error_strings,
627  .n_next_nodes = 1,
628  .next_nodes =
629  {
630  [ESP_ENCRYPT_NEXT_DROP] = "error-drop",
631  }
632 };
633 /* *INDENT-ON* */
634 
637  vlib_frame_t * from_frame)
638 {
639  return dpdk_esp_encrypt_inline (vm, node, from_frame, 1 /*is_ip6 */ , 0);
640 }
641 
642 /* *INDENT-OFF* */
644  .name = "dpdk-esp6-encrypt",
645  .flags = VLIB_NODE_FLAG_IS_OUTPUT,
646  .vector_size = sizeof (u32),
647  .format_trace = format_esp_encrypt_trace,
648  .n_errors = ARRAY_LEN (esp_encrypt_error_strings),
649  .error_strings = esp_encrypt_error_strings,
650  .n_next_nodes = 1,
651  .next_nodes =
652  {
653  [ESP_ENCRYPT_NEXT_DROP] = "error-drop",
654  }
655 };
656 /* *INDENT-ON* */
657 
660  vlib_frame_t * from_frame)
661 {
662  return dpdk_esp_encrypt_inline (vm, node, from_frame, 0 /*is_ip6 */ , 1);
663 }
664 
665 /* *INDENT-OFF* */
667  .name = "dpdk-esp4-encrypt-tun",
668  .flags = VLIB_NODE_FLAG_IS_OUTPUT,
669  .vector_size = sizeof (u32),
670  .format_trace = format_esp_encrypt_trace,
671  .n_errors = ARRAY_LEN (esp_encrypt_error_strings),
672  .error_strings = esp_encrypt_error_strings,
673  .n_next_nodes = 1,
674  .next_nodes =
675  {
676  [ESP_ENCRYPT_NEXT_DROP] = "error-drop",
677  }
678 };
679 /* *INDENT-ON* */
680 
683  vlib_frame_t * from_frame)
684 {
685  return dpdk_esp_encrypt_inline (vm, node, from_frame, 1 /*is_ip6 */ , 1);
686 }
687 
688 /* *INDENT-OFF* */
690  .name = "dpdk-esp6-encrypt-tun",
691  .flags = VLIB_NODE_FLAG_IS_OUTPUT,
692  .vector_size = sizeof (u32),
693  .format_trace = format_esp_encrypt_trace,
694  .n_errors = ARRAY_LEN (esp_encrypt_error_strings),
695  .error_strings = esp_encrypt_error_strings,
696  .n_next_nodes = 1,
697  .next_nodes =
698  {
699  [ESP_ENCRYPT_NEXT_DROP] = "error-drop",
700  }
701 };
702 /* *INDENT-ON* */
703 
704 /*
705  * fd.io coding-style-patch-verification: ON
706  *
707  * Local Variables:
708  * eval: (c-set-style "gnu")
709  * End:
710  */
u32 alg
Definition: ipsec.h:87
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
u8 * format_ipsec_integ_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:111
static_always_inline void crypto_op_setup(u8 is_aead, struct rte_mbuf *mb0, struct rte_crypto_op *op, void *session, u32 cipher_off, u32 cipher_len, u32 auth_off, u32 auth_len, u8 *aad, u8 *digest, u64 digest_paddr)
Definition: ipsec.h:357
static vlib_cli_command_t trace
(constructor) VLIB_CLI_COMMAND (trace)
Definition: vlib_api_cli.c:899
static u8 * vlib_buffer_get_tail(vlib_buffer_t *b)
Get pointer to the end of buffer&#39;s data.
Definition: buffer.h:314
#define CLIB_UNUSED(x)
Definition: clib.h:87
ip46_address_t tunnel_src_addr
Definition: ipsec_sa.h:197
vlib_node_registration_t dpdk_esp4_encrypt_node
(constructor) VLIB_REGISTER_NODE (dpdk_esp4_encrypt_node)
Definition: esp_encrypt.c:620
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
Definition: counter.h:239
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:937
vnet_interface_main_t interface_main
Definition: vnet.h:65
format_function_t format_ip4_header
Definition: format.h:81
esp_encrypt_next_t
Definition: esp_encrypt.c:38
unsigned long u64
Definition: types.h:89
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
static_always_inline i32 crypto_alloc_ops(u8 numa, struct rte_crypto_op **ops, u32 n)
Definition: ipsec.h:284
ipsec_integ_alg_t integ_alg
Definition: ipsec_sa.h:214
#define thread_idx
Definition: tls_async.c:40
u32 thread_index
Definition: main.h:250
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
static u8 * format_esp_encrypt_trace(u8 *s, va_list *args)
Definition: esp_encrypt.c:82
static_always_inline void crypto_set_icb(dpdk_gcm_cnt_blk *icb, u32 salt, u32 seq, u32 seq_hi)
Definition: ipsec.h:349
vl_api_address_t src
Definition: gre.api:54
ipsec_integ_alg_t integ_alg
Definition: esp_encrypt.c:76
static u32 format_get_indent(u8 *s)
Definition: format.h:72
vlib_main_t * vm
Definition: in2out_ed.c:1580
ipsec_crypto_alg_t crypto_alg
Definition: esp_encrypt.c:75
#define VLIB_NODE_FN(node)
Definition: node.h:203
u16 mask
Definition: flow_types.api:52
unsigned char u8
Definition: types.h:56
u32 bi[VLIB_FRAME_SIZE]
Definition: ipsec.h:128
u32 seq_hi
Definition: ipsec_sa.h:123
static int esp_seq_advance(ipsec_sa_t *sa)
Definition: esp.h:84
static_always_inline clib_error_t * crypto_get_session(struct rte_cryptodev_sym_session **session, u32 sa_idx, crypto_resource_t *res, crypto_worker_main_t *cwm, u8 is_outbound)
Definition: ipsec.h:238
static_always_inline void crypto_free_ops(u8 numa, struct rte_crypto_op **ops, u32 n)
Definition: ipsec.h:300
dpdk_crypto_main_t dpdk_crypto_main
Definition: ipsec.c:26
ipsec_main_t ipsec_main
Definition: ipsec.c:28
vlib_combined_counter_main_t * combined_sw_if_counters
Definition: interface.h:882
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:129
static_always_inline void crypto_enqueue_ops(vlib_main_t *vm, crypto_worker_main_t *cwm, u32 node_index, u32 error, u8 numa, u8 encrypt)
Definition: ipsec.h:312
description fragment has unexpected format
Definition: map.api:433
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
u8 * format_ipsec_crypto_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:79
unsigned int u32
Definition: types.h:88
u32 padding
Definition: vhost_user.h:105
crypto_alg_t * auth_algs
Definition: ipsec.h:167
static const u8 pad_data[]
Definition: ipsec.h:176
bool is_ip6
Definition: ip.api:43
u32 next
Definition: ipsec.h:64
Definition: cJSON.c:84
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:546
vlib_node_registration_t dpdk_esp6_encrypt_tun_node
(constructor) VLIB_REGISTER_NODE (dpdk_esp6_encrypt_tun_node)
Definition: esp_encrypt.c:689
static index_t ipsec_tun_protect_get_sa_out(adj_index_t ai)
Definition: ipsec_tun.h:192
crypto_alg_t * cipher_algs
Definition: ipsec.h:166
u32 salt
Definition: ipsec_sa.h:164
vnet_main_t * vnet_main
Definition: ipsec.h:122
unsigned short u16
Definition: types.h:57
#define rte_mbuf_from_vlib_buffer(x)
Definition: buffer.h:19
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:233
#define PREDICT_FALSE(x)
Definition: clib.h:121
#define always_inline
Definition: ipsec.h:28
vl_api_address_t dst
Definition: gre.api:55
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:391
u8 encrypt
Definition: ipsec.h:66
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
Definition: node_funcs.h:1231
u8 len
Definition: ip_types.api:103
ip46_address_t tunnel_dst_addr
Definition: ipsec_sa.h:198
u8 aad[16]
Definition: ipsec.h:69
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:170
vlib_node_registration_t dpdk_esp4_encrypt_tun_node
(constructor) VLIB_REGISTER_NODE (dpdk_esp4_encrypt_tun_node)
Definition: esp_encrypt.c:666
u16 n_vectors
Definition: node.h:397
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:219
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:80
dpdk_gcm_cnt_blk cb
Definition: ipsec.h:68
#define foreach_esp_encrypt_error
Definition: esp_encrypt.c:45
vlib_node_registration_t dpdk_esp6_encrypt_node
(constructor) VLIB_REGISTER_NODE (dpdk_esp6_encrypt_node)
Definition: esp_encrypt.c:643
#define clib_warning(format, args...)
Definition: error.h:59
u8 data[]
Packet data.
Definition: buffer.h:181
static char * esp_encrypt_error_strings[]
Definition: esp_encrypt.c:62
#define ARRAY_LEN(x)
Definition: clib.h:67
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:483
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1580
u8 * format_esp_header(u8 *s, va_list *args)
Definition: esp_format.c:23
u8 iv_len
Definition: ipsec.h:89
vlib_combined_counter_main_t ipsec_sa_counters
SA packet & bytes counters.
Definition: ipsec_sa.c:27
#define ASSERT(truth)
ipsec_sa_t * sad
Definition: ipsec.h:111
esp_encrypt_error_t
Definition: esp_encrypt.c:54
crypto_worker_main_t * workers_main
Definition: ipsec.h:163
u32 seq
Definition: esp.h:29
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:252
format_function_t format_ip6_header
Definition: format.h:95
u32 spi
Definition: esp.h:26
u8 boundary
Definition: ipsec.h:91
crypto_resource_t * resource
Definition: ipsec.h:165
static uword is_pow2(uword x)
Definition: clib.h:253
struct _vlib_node_registration vlib_node_registration_t
template key/value backing page structure
Definition: bihash_doc.h:44
Definition: defs.h:47
static uword dpdk_esp_encrypt_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame, int is_ip6, int is_tun)
Definition: esp_encrypt.c:114
vl_api_address_t ip
Definition: l2.api:501
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:297
struct rte_crypto_op ** ops
Definition: ipsec.h:76
struct clib_bihash_value offset
template key/value backing page structure
enum rte_crypto_sym_xform_type type
Definition: ipsec.h:86
#define vnet_buffer(b)
Definition: buffer.h:417
static_always_inline u16 get_resource(crypto_worker_main_t *cwm, ipsec_sa_t *sa)
Definition: ipsec.h:259
ipsec_crypto_alg_t crypto_alg
Definition: ipsec_sa.h:213
#define VLIB_NODE_FLAG_IS_OUTPUT
Definition: node.h:296
#define foreach_esp_encrypt_next
Definition: esp_encrypt.c:31
void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace.c:634
static void * vlib_buffer_put_uninit(vlib_buffer_t *b, u16 size)
Append uninitialized data to buffer.
Definition: buffer.h:325
static int ip4_header_bytes(const ip4_header_t *i)
Definition: ip4_packet.h:190
u8 ip_version_and_header_length
Definition: ip4_packet.h:93
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
struct rte_crypto_op * ops[VLIB_FRAME_SIZE]
Definition: ipsec.h:127
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
static u16 ip4_header_checksum(ip4_header_t *i)
Definition: ip4_packet.h:314
vl_api_interface_index_t sw_if_index
Definition: wireguard.api:34
static_always_inline dpdk_op_priv_t * crypto_op_get_priv(struct rte_crypto_op *op)
Definition: ipsec.h:208