FD.io VPP  v20.09-64-g4f7b92f0a
Vector Packet Processing
esp_encrypt.c
Go to the documentation of this file.
1 /*
2  * esp_encrypt.c : IPSec ESP encrypt node using DPDK Cryptodev
3  *
4  * Copyright (c) 2017 Intel and/or its affiliates.
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at:
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  */
17 
18 #include <vnet/vnet.h>
19 #include <vnet/api_errno.h>
20 #include <vnet/ip/ip.h>
21 
22 #include <vnet/ipsec/ipsec.h>
23 #include <vnet/ipsec/esp.h>
24 #include <vnet/udp/udp.h>
25 #include <dpdk/buffer.h>
26 #include <dpdk/ipsec/ipsec.h>
27 #include <vnet/ipsec/ipsec_tun.h>
28 #include <dpdk/device/dpdk.h>
29 #include <dpdk/device/dpdk_priv.h>
30 
31 #define foreach_esp_encrypt_next \
32 _(DROP, "error-drop") \
33 _(IP4_LOOKUP, "ip4-lookup") \
34 _(IP6_LOOKUP, "ip6-lookup") \
35 _(INTERFACE_OUTPUT, "interface-output")
36 
37 #define _(v, s) ESP_ENCRYPT_NEXT_##v,
38 typedef enum
39 {
41 #undef _
44 
45 #define foreach_esp_encrypt_error \
46  _(RX_PKTS, "ESP pkts received") \
47  _(SEQ_CYCLED, "Sequence number cycled") \
48  _(ENQ_FAIL, "Enqueue encrypt failed (queue full)") \
49  _(DISCARD, "Not enough crypto operations") \
50  _(SESSION, "Failed to get crypto session") \
51  _(NOSUP, "Cipher/Auth not supported")
52 
53 
54 typedef enum
55 {
56 #define _(sym,str) ESP_ENCRYPT_ERROR_##sym,
58 #undef _
61 
62 static char *esp_encrypt_error_strings[] = {
63 #define _(sym,string) string,
65 #undef _
66 };
67 
72 
73 typedef struct
74 {
77  u8 packet_data[64];
79 
80 /* packet trace format function */
81 static u8 *
82 format_esp_encrypt_trace (u8 * s, va_list * args)
83 {
84  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
85  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
86  esp_encrypt_trace_t *t = va_arg (*args, esp_encrypt_trace_t *);
88  u32 indent = format_get_indent (s), offset;
89 
90  s = format (s, "cipher %U auth %U\n",
93 
94  if ((ih4->ip_version_and_header_length & 0xF0) == 0x60)
95  {
96  s = format (s, "%U%U", format_white_space, indent,
97  format_ip6_header, ih4);
98  offset = sizeof (ip6_header_t);
99  }
100  else
101  {
102  s = format (s, "%U%U", format_white_space, indent,
103  format_ip4_header, ih4);
104  offset = ip4_header_bytes (ih4);
105  }
106 
107  s = format (s, "\n%U%U", format_white_space, indent,
109 
110  return s;
111 }
112 
116  vlib_frame_t * from_frame, int is_ip6, int is_tun)
117 {
118  u32 n_left_from, *from, *to_next, next_index, thread_index;
119  ipsec_main_t *im = &ipsec_main;
120  vnet_main_t *vnm = im->vnet_main;
124  crypto_resource_t *res = 0;
125  ipsec_sa_t *sa0 = 0;
126  crypto_alg_t *cipher_alg = 0, *auth_alg = 0;
127  struct rte_cryptodev_sym_session *session = 0;
128  u32 ret, last_sa_index = ~0;
129  u8 numa = rte_socket_id ();
130  u8 is_aead = 0;
131  crypto_worker_main_t *cwm =
132  vec_elt_at_index (dcm->workers_main, thread_idx);
133  struct rte_crypto_op **ops = cwm->ops;
134 
135  from = vlib_frame_vector_args (from_frame);
136  n_left_from = from_frame->n_vectors;
137  thread_index = vm->thread_index;
138 
139  ret = crypto_alloc_ops (numa, ops, n_left_from);
140  if (ret)
141  {
142  if (is_ip6)
144  ESP_ENCRYPT_ERROR_DISCARD, n_left_from);
145  else
147  ESP_ENCRYPT_ERROR_DISCARD, n_left_from);
148  /* Discard whole frame */
149  vlib_buffer_free (vm, from, n_left_from);
150  return n_left_from;
151  }
152 
153  next_index = ESP_ENCRYPT_NEXT_DROP;
154 
155  while (n_left_from > 0)
156  {
157  u32 n_left_to_next;
158 
159  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
160 
161  while (n_left_from > 0 && n_left_to_next > 0)
162  {
163  clib_error_t *error;
164  u32 bi0, bi1;
165  vlib_buffer_t *b0, *b1;
166  u32 sa_index0;
167  ip4_and_esp_header_t *ih0, *oh0 = 0;
168  ip6_and_esp_header_t *ih6_0, *oh6_0 = 0;
169  ip4_and_udp_and_esp_header_t *ouh0 = 0;
170  esp_header_t *esp0;
171  esp_footer_t *f0;
172  u8 next_hdr_type;
173  u32 iv_size;
174  u16 orig_sz;
175  u8 trunc_size;
176  u16 rewrite_len;
177  u16 udp_encap_adv = 0;
178  struct rte_mbuf *mb0;
179  struct rte_crypto_op *op;
180  u16 res_idx;
181 
182  bi0 = from[0];
183  from += 1;
184  n_left_from -= 1;
185 
186  b0 = vlib_get_buffer (vm, bi0);
187  ih0 = vlib_buffer_get_current (b0);
188  mb0 = rte_mbuf_from_vlib_buffer (b0);
189 
190  /* ih0/ih6_0 */
191  CLIB_PREFETCH (ih0, sizeof (ih6_0[0]), LOAD);
192  /* f0 */
193  CLIB_PREFETCH (vlib_buffer_get_tail (b0), 20, STORE);
194  /* mb0 */
195  CLIB_PREFETCH (mb0, CLIB_CACHE_LINE_BYTES, STORE);
196 
197  if (n_left_from > 1)
198  {
199  bi1 = from[1];
200  b1 = vlib_get_buffer (vm, bi1);
201 
204  CLIB_CACHE_LINE_BYTES, STORE);
205  }
206 
207  op = ops[0];
208  ops += 1;
209  ASSERT (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED);
210 
211  dpdk_op_priv_t *priv = crypto_op_get_priv (op);
212  /* store bi in op private */
213  priv->bi = bi0;
214  priv->encrypt = 1;
215 
216  u16 op_len =
217  sizeof (op[0]) + sizeof (op[0].sym[0]) + sizeof (priv[0]);
218  CLIB_PREFETCH (op, op_len, STORE);
219 
220  if (is_tun)
221  {
222  /* we are on a ipsec tunnel's feature arc */
223  vnet_buffer (b0)->ipsec.sad_index =
224  sa_index0 = ipsec_tun_protect_get_sa_out
225  (vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
226  }
227  else
228  sa_index0 = vnet_buffer (b0)->ipsec.sad_index;
229 
230  if (sa_index0 != last_sa_index)
231  {
232  sa0 = pool_elt_at_index (im->sad, sa_index0);
233 
234  cipher_alg =
236  auth_alg = vec_elt_at_index (dcm->auth_algs, sa0->integ_alg);
237 
238  is_aead = (cipher_alg->type == RTE_CRYPTO_SYM_XFORM_AEAD);
239 
240  if (is_aead)
241  auth_alg = cipher_alg;
242 
243  res_idx = get_resource (cwm, sa0);
244 
245  if (PREDICT_FALSE (res_idx == (u16) ~ 0))
246  {
247  if (is_ip6)
250  ESP_ENCRYPT_ERROR_NOSUP, 1);
251  else
254  ESP_ENCRYPT_ERROR_NOSUP, 1);
255  to_next[0] = bi0;
256  to_next += 1;
257  n_left_to_next -= 1;
258  goto trace;
259  }
260  res = vec_elt_at_index (dcm->resource, res_idx);
261 
262  error = crypto_get_session (&session, sa_index0, res, cwm, 1);
263  if (PREDICT_FALSE (error || !session))
264  {
265  if (is_ip6)
268  ESP_ENCRYPT_ERROR_SESSION,
269  1);
270  else
273  ESP_ENCRYPT_ERROR_SESSION,
274  1);
275  to_next[0] = bi0;
276  to_next += 1;
277  n_left_to_next -= 1;
278  goto trace;
279  }
280 
281  last_sa_index = sa_index0;
282  }
283 
284  if (PREDICT_FALSE (esp_seq_advance (sa0)))
285  {
286  if (is_ip6)
289  ESP_ENCRYPT_ERROR_SEQ_CYCLED, 1);
290  else
293  ESP_ENCRYPT_ERROR_SEQ_CYCLED, 1);
294  //TODO: rekey SA
295  to_next[0] = bi0;
296  to_next += 1;
297  n_left_to_next -= 1;
298  goto trace;
299  }
300 
301  orig_sz = b0->current_length;
302 
303  /* TODO multi-seg support - total_length_not_including_first_buffer */
305  (&ipsec_sa_counters, thread_index, sa_index0,
306  1, b0->current_length);
307 
308  /* Update tunnel interface tx counters */
309  if (is_tun)
312  thread_index, vnet_buffer (b0)->sw_if_index[VLIB_TX],
313  1, b0->current_length);
314 
315  res->ops[res->n_ops] = op;
316  res->bi[res->n_ops] = bi0;
317  res->n_ops += 1;
318 
319  dpdk_gcm_cnt_blk *icb = &priv->cb;
320 
321  crypto_set_icb (icb, sa0->salt, sa0->seq, sa0->seq_hi);
322 
323  iv_size = cipher_alg->iv_len;
324  trunc_size = auth_alg->trunc_size;
325 
326  /* if UDP encapsulation is used adjust the address of the IP header */
327  if (ipsec_sa_is_set_UDP_ENCAP (sa0) && !is_ip6)
328  udp_encap_adv = sizeof (udp_header_t);
329 
330  if (ipsec_sa_is_set_IS_TUNNEL (sa0))
331  {
332  rewrite_len = 0;
333  if (!ipsec_sa_is_set_IS_TUNNEL_V6 (sa0)) /* ip4 */
334  {
335  /* in tunnel mode send it back to FIB */
336  priv->next = DPDK_CRYPTO_INPUT_NEXT_IP4_LOOKUP;
337  u8 adv = sizeof (ip4_header_t) + udp_encap_adv +
338  sizeof (esp_header_t) + iv_size;
339  vlib_buffer_advance (b0, -adv);
340  oh0 = vlib_buffer_get_current (b0);
341  ouh0 = vlib_buffer_get_current (b0);
342  next_hdr_type = (is_ip6 ?
343  IP_PROTOCOL_IPV6 : IP_PROTOCOL_IP_IN_IP);
344  /*
345  * oh0->ip4.ip_version_and_header_length = 0x45;
346  * oh0->ip4.tos = ih0->ip4.tos;
347  * oh0->ip4.fragment_id = 0;
348  * oh0->ip4.flags_and_fragment_offset = 0;
349  */
350  oh0->ip4.checksum_data_64[0] =
351  clib_host_to_net_u64 (0x45ULL << 56);
352  /*
353  * oh0->ip4.ttl = 254;
354  * oh0->ip4.protocol = IP_PROTOCOL_IPSEC_ESP;
355  */
356  oh0->ip4.checksum_data_32[2] =
357  clib_host_to_net_u32 (0xfe320000);
358 
359  oh0->ip4.src_address.as_u32 =
360  sa0->tunnel_src_addr.ip4.as_u32;
361  oh0->ip4.dst_address.as_u32 =
362  sa0->tunnel_dst_addr.ip4.as_u32;
363 
364  if (ipsec_sa_is_set_UDP_ENCAP (sa0))
365  {
366  oh0->ip4.protocol = IP_PROTOCOL_UDP;
367  esp0 = &ouh0->esp;
368  }
369  else
370  esp0 = &oh0->esp;
371  esp0->spi = clib_host_to_net_u32 (sa0->spi);
372  esp0->seq = clib_host_to_net_u32 (sa0->seq);
373  }
374  else
375  {
376  /* ip6 */
377  /* in tunnel mode send it back to FIB */
378  priv->next = DPDK_CRYPTO_INPUT_NEXT_IP6_LOOKUP;
379 
380  u8 adv =
381  sizeof (ip6_header_t) + sizeof (esp_header_t) + iv_size;
382  vlib_buffer_advance (b0, -adv);
383  ih6_0 = (ip6_and_esp_header_t *) ih0;
384  oh6_0 = vlib_buffer_get_current (b0);
385 
386  next_hdr_type = (is_ip6 ?
387  IP_PROTOCOL_IPV6 : IP_PROTOCOL_IP_IN_IP);
388 
389  oh6_0->ip6.ip_version_traffic_class_and_flow_label =
390  ih6_0->ip6.ip_version_traffic_class_and_flow_label;
391 
392  oh6_0->ip6.protocol = IP_PROTOCOL_IPSEC_ESP;
393  oh6_0->ip6.hop_limit = 254;
394  oh6_0->ip6.src_address.as_u64[0] =
395  sa0->tunnel_src_addr.ip6.as_u64[0];
396  oh6_0->ip6.src_address.as_u64[1] =
397  sa0->tunnel_src_addr.ip6.as_u64[1];
398  oh6_0->ip6.dst_address.as_u64[0] =
399  sa0->tunnel_dst_addr.ip6.as_u64[0];
400  oh6_0->ip6.dst_address.as_u64[1] =
401  sa0->tunnel_dst_addr.ip6.as_u64[1];
402  esp0 = &oh6_0->esp;
403  oh6_0->esp.spi = clib_host_to_net_u32 (sa0->spi);
404  oh6_0->esp.seq = clib_host_to_net_u32 (sa0->seq);
405  }
406 
407  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
408  }
409  else /* transport mode */
410  {
411  if (is_tun)
412  {
413  rewrite_len = 0;
414  priv->next = DPDK_CRYPTO_INPUT_NEXT_MIDCHAIN;
415  }
416  else
417  {
418  priv->next = DPDK_CRYPTO_INPUT_NEXT_INTERFACE_OUTPUT;
419  rewrite_len = vnet_buffer (b0)->ip.save_rewrite_length;
420  }
421  u16 adv = sizeof (esp_header_t) + iv_size + udp_encap_adv;
422  vlib_buffer_advance (b0, -adv - rewrite_len);
423  u8 *src = ((u8 *) ih0) - rewrite_len;
425  oh0 = vlib_buffer_get_current (b0) + rewrite_len;
426  ouh0 = vlib_buffer_get_current (b0) + rewrite_len;
427 
428  if (is_ip6)
429  {
430  orig_sz -= sizeof (ip6_header_t);
431  ih6_0 = (ip6_and_esp_header_t *) ih0;
432  next_hdr_type = ih6_0->ip6.protocol;
433  memmove (dst, src, rewrite_len + sizeof (ip6_header_t));
434  oh6_0 = (ip6_and_esp_header_t *) oh0;
435  oh6_0->ip6.protocol = IP_PROTOCOL_IPSEC_ESP;
436  esp0 = &oh6_0->esp;
437  }
438  else /* ipv4 */
439  {
440  u16 ip_size = ip4_header_bytes (&ih0->ip4);
441  orig_sz -= ip_size;
442  next_hdr_type = ih0->ip4.protocol;
443  memmove (dst, src, rewrite_len + ip_size);
444  oh0->ip4.protocol = IP_PROTOCOL_IPSEC_ESP;
445  esp0 = (esp_header_t *) (((u8 *) oh0) + ip_size);
446  if (ipsec_sa_is_set_UDP_ENCAP (sa0))
447  {
448  oh0->ip4.protocol = IP_PROTOCOL_UDP;
449  esp0 = (esp_header_t *)
450  (((u8 *) oh0) + ip_size + udp_encap_adv);
451  }
452  else
453  {
454  oh0->ip4.protocol = IP_PROTOCOL_IPSEC_ESP;
455  esp0 = (esp_header_t *) (((u8 *) oh0) + ip_size);
456  }
457  }
458  esp0->spi = clib_host_to_net_u32 (sa0->spi);
459  esp0->seq = clib_host_to_net_u32 (sa0->seq);
460  }
461 
462  if (ipsec_sa_is_set_UDP_ENCAP (sa0) && ouh0)
463  {
464  ouh0->udp.src_port = clib_host_to_net_u16 (UDP_DST_PORT_ipsec);
465  ouh0->udp.dst_port = clib_host_to_net_u16 (UDP_DST_PORT_ipsec);
466  ouh0->udp.checksum = 0;
467  }
468  ASSERT (is_pow2 (cipher_alg->boundary));
469  u16 mask = cipher_alg->boundary - 1;
470  u16 pad_payload_len = ((orig_sz + 2) + mask) & ~mask;
471  u8 pad_bytes = pad_payload_len - 2 - orig_sz;
472 
473  u8 *padding =
474  vlib_buffer_put_uninit (b0, pad_bytes + 2 + trunc_size);
475 
476  /* The extra pad bytes would be overwritten by the digest */
477  if (pad_bytes)
478  clib_memcpy_fast (padding, pad_data, 16);
479 
480  f0 = (esp_footer_t *) (padding + pad_bytes);
481  f0->pad_length = pad_bytes;
482  f0->next_header = next_hdr_type;
483 
484  if (oh6_0)
485  {
486  u16 len = b0->current_length - sizeof (ip6_header_t);
487  oh6_0->ip6.payload_length =
488  clib_host_to_net_u16 (len - rewrite_len);
489  }
490  else if (oh0)
491  {
492  oh0->ip4.length =
493  clib_host_to_net_u16 (b0->current_length - rewrite_len);
494  oh0->ip4.checksum = ip4_header_checksum (&oh0->ip4);
495  if (ipsec_sa_is_set_UDP_ENCAP (sa0) && ouh0)
496  {
497  ouh0->udp.length =
498  clib_host_to_net_u16 (clib_net_to_host_u16
499  (ouh0->ip4.length) -
500  ip4_header_bytes (&ouh0->ip4));
501  }
502  }
503  else /* should never happen */
504  clib_warning ("No outer header found for ESP packet");
505 
506  b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
507 
508  /* mbuf packet starts at ESP header */
509  mb0->data_len = vlib_buffer_get_tail (b0) - ((u8 *) esp0);
510  mb0->pkt_len = vlib_buffer_get_tail (b0) - ((u8 *) esp0);
511  mb0->data_off = ((void *) esp0) - mb0->buf_addr;
512 
513  u32 cipher_off, cipher_len, auth_len = 0;
514  u32 *aad = NULL;
515 
516  u8 *digest = vlib_buffer_get_tail (b0) - trunc_size;
517  u64 digest_paddr =
518  mb0->buf_physaddr + digest - ((u8 *) mb0->buf_addr);
519 
520  if (!is_aead && (cipher_alg->alg == RTE_CRYPTO_CIPHER_AES_CBC ||
521  cipher_alg->alg == RTE_CRYPTO_CIPHER_NULL))
522  {
523  cipher_off = sizeof (esp_header_t);
524  cipher_len = iv_size + pad_payload_len;
525  }
526  else /* CTR/GCM */
527  {
528  u32 *esp_iv = (u32 *) (esp0 + 1);
529  esp_iv[0] = sa0->seq;
530  esp_iv[1] = sa0->seq_hi;
531 
532  cipher_off = sizeof (esp_header_t) + iv_size;
533  cipher_len = pad_payload_len;
534  }
535 
536  if (is_aead)
537  {
538  aad = (u32 *) priv->aad;
539  aad[0] = esp0->spi;
540 
541  /* aad[3] should always be 0 */
542  if (PREDICT_FALSE (ipsec_sa_is_set_USE_ESN (sa0)))
543  {
544  aad[1] = clib_host_to_net_u32 (sa0->seq_hi);
545  aad[2] = esp0->seq;
546  }
547  else
548  {
549  aad[1] = esp0->seq;
550  aad[2] = 0;
551  }
552  }
553  else
554  {
555  auth_len =
556  vlib_buffer_get_tail (b0) - ((u8 *) esp0) - trunc_size;
557  if (ipsec_sa_is_set_USE_ESN (sa0))
558  {
559  u32 *_digest = (u32 *) digest;
560  _digest[0] = clib_host_to_net_u32 (sa0->seq_hi);
561  auth_len += 4;
562  }
563  }
564 
565  crypto_op_setup (is_aead, mb0, op, session, cipher_off, cipher_len,
566  0, auth_len, (u8 *) aad, digest, digest_paddr);
567 
568  trace:
569  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
570  {
571  esp_encrypt_trace_t *tr =
572  vlib_add_trace (vm, node, b0, sizeof (*tr));
573  tr->crypto_alg = sa0->crypto_alg;
574  tr->integ_alg = sa0->integ_alg;
575  u8 *p = vlib_buffer_get_current (b0);
576  if (!ipsec_sa_is_set_IS_TUNNEL (sa0) && !is_tun)
577  p += vnet_buffer (b0)->ip.save_rewrite_length;
578  clib_memcpy_fast (tr->packet_data, p, sizeof (tr->packet_data));
579  }
580  }
581  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
582  }
583  if (is_ip6)
584  {
586  (is_tun ?
588  dpdk_esp6_encrypt_node.index),
589  ESP_ENCRYPT_ERROR_RX_PKTS,
590  from_frame->n_vectors);
591 
593  ESP_ENCRYPT_ERROR_ENQ_FAIL, numa, 1 /* encrypt */ );
594  }
595  else
596  {
598  (is_tun ?
600  dpdk_esp4_encrypt_node.index),
601  ESP_ENCRYPT_ERROR_RX_PKTS,
602  from_frame->n_vectors);
603 
605  ESP_ENCRYPT_ERROR_ENQ_FAIL, numa, 1 /* encrypt */ );
606  }
607 
608  crypto_free_ops (numa, ops, cwm->ops + from_frame->n_vectors - ops);
609 
610  return from_frame->n_vectors;
611 }
612 
615  vlib_frame_t * from_frame)
616 {
617  return dpdk_esp_encrypt_inline (vm, node, from_frame, 0 /*is_ip6 */ , 0);
618 }
619 
620 /* *INDENT-OFF* */
622  .name = "dpdk-esp4-encrypt",
623  .flags = VLIB_NODE_FLAG_IS_OUTPUT,
624  .vector_size = sizeof (u32),
625  .format_trace = format_esp_encrypt_trace,
626  .n_errors = ARRAY_LEN (esp_encrypt_error_strings),
627  .error_strings = esp_encrypt_error_strings,
628  .n_next_nodes = 1,
629  .next_nodes =
630  {
631  [ESP_ENCRYPT_NEXT_DROP] = "error-drop",
632  }
633 };
634 /* *INDENT-ON* */
635 
638  vlib_frame_t * from_frame)
639 {
640  return dpdk_esp_encrypt_inline (vm, node, from_frame, 1 /*is_ip6 */ , 0);
641 }
642 
643 /* *INDENT-OFF* */
645  .name = "dpdk-esp6-encrypt",
646  .flags = VLIB_NODE_FLAG_IS_OUTPUT,
647  .vector_size = sizeof (u32),
648  .format_trace = format_esp_encrypt_trace,
649  .n_errors = ARRAY_LEN (esp_encrypt_error_strings),
650  .error_strings = esp_encrypt_error_strings,
651  .n_next_nodes = 1,
652  .next_nodes =
653  {
654  [ESP_ENCRYPT_NEXT_DROP] = "error-drop",
655  }
656 };
657 /* *INDENT-ON* */
658 
661  vlib_frame_t * from_frame)
662 {
663  return dpdk_esp_encrypt_inline (vm, node, from_frame, 0 /*is_ip6 */ , 1);
664 }
665 
666 /* *INDENT-OFF* */
668  .name = "dpdk-esp4-encrypt-tun",
669  .flags = VLIB_NODE_FLAG_IS_OUTPUT,
670  .vector_size = sizeof (u32),
671  .format_trace = format_esp_encrypt_trace,
672  .n_errors = ARRAY_LEN (esp_encrypt_error_strings),
673  .error_strings = esp_encrypt_error_strings,
674  .n_next_nodes = 1,
675  .next_nodes =
676  {
677  [ESP_ENCRYPT_NEXT_DROP] = "error-drop",
678  }
679 };
680 /* *INDENT-ON* */
681 
684  vlib_frame_t * from_frame)
685 {
686  return dpdk_esp_encrypt_inline (vm, node, from_frame, 1 /*is_ip6 */ , 1);
687 }
688 
689 /* *INDENT-OFF* */
691  .name = "dpdk-esp6-encrypt-tun",
692  .flags = VLIB_NODE_FLAG_IS_OUTPUT,
693  .vector_size = sizeof (u32),
694  .format_trace = format_esp_encrypt_trace,
695  .n_errors = ARRAY_LEN (esp_encrypt_error_strings),
696  .error_strings = esp_encrypt_error_strings,
697  .n_next_nodes = 1,
698  .next_nodes =
699  {
700  [ESP_ENCRYPT_NEXT_DROP] = "error-drop",
701  }
702 };
703 /* *INDENT-ON* */
704 
705 /*
706  * fd.io coding-style-patch-verification: ON
707  *
708  * Local Variables:
709  * eval: (c-set-style "gnu")
710  * End:
711  */
u32 alg
Definition: ipsec.h:87
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
u8 * format_ipsec_integ_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:110
static_always_inline void crypto_op_setup(u8 is_aead, struct rte_mbuf *mb0, struct rte_crypto_op *op, void *session, u32 cipher_off, u32 cipher_len, u32 auth_off, u32 auth_len, u8 *aad, u8 *digest, u64 digest_paddr)
Definition: ipsec.h:357
static vlib_cli_command_t trace
(constructor) VLIB_CLI_COMMAND (trace)
Definition: vlib_api_cli.c:899
static u8 * vlib_buffer_get_tail(vlib_buffer_t *b)
Get pointer to the end of buffer&#39;s data.
Definition: buffer.h:310
#define CLIB_UNUSED(x)
Definition: clib.h:87
ip46_address_t tunnel_src_addr
Definition: ipsec_sa.h:175
vlib_node_registration_t dpdk_esp4_encrypt_node
(constructor) VLIB_REGISTER_NODE (dpdk_esp4_encrypt_node)
Definition: esp_encrypt.c:621
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
Definition: counter.h:220
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:937
vnet_interface_main_t interface_main
Definition: vnet.h:59
format_function_t format_ip4_header
Definition: format.h:81
ipsec_integ_alg_t
Definition: ipsec_sa.h:59
esp_encrypt_next_t
Definition: esp_encrypt.c:38
unsigned long u64
Definition: types.h:89
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
static_always_inline i32 crypto_alloc_ops(u8 numa, struct rte_crypto_op **ops, u32 n)
Definition: ipsec.h:284
ipsec_integ_alg_t integ_alg
Definition: ipsec_sa.h:171
#define thread_idx
Definition: tls_async.c:40
u32 thread_index
Definition: main.h:249
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
static u8 * format_esp_encrypt_trace(u8 *s, va_list *args)
Definition: esp_encrypt.c:82
static_always_inline void crypto_set_icb(dpdk_gcm_cnt_blk *icb, u32 salt, u32 seq, u32 seq_hi)
Definition: ipsec.h:349
vl_api_address_t src
Definition: gre.api:54
ipsec_integ_alg_t integ_alg
Definition: esp_encrypt.c:76
static u32 format_get_indent(u8 *s)
Definition: format.h:72
vlib_main_t * vm
Definition: in2out_ed.c:1582
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
ipsec_crypto_alg_t crypto_alg
Definition: esp_encrypt.c:75
#define VLIB_NODE_FN(node)
Definition: node.h:202
u16 mask
Definition: flow_types.api:52
unsigned char u8
Definition: types.h:56
u32 bi[VLIB_FRAME_SIZE]
Definition: ipsec.h:128
u32 seq_hi
Definition: ipsec_sa.h:122
static int esp_seq_advance(ipsec_sa_t *sa)
Definition: esp.h:84
static_always_inline clib_error_t * crypto_get_session(struct rte_cryptodev_sym_session **session, u32 sa_idx, crypto_resource_t *res, crypto_worker_main_t *cwm, u8 is_outbound)
Definition: ipsec.h:238
static_always_inline void crypto_free_ops(u8 numa, struct rte_crypto_op **ops, u32 n)
Definition: ipsec.h:300
dpdk_crypto_main_t dpdk_crypto_main
Definition: ipsec.c:25
ipsec_main_t ipsec_main
Definition: ipsec.c:28
vlib_combined_counter_main_t * combined_sw_if_counters
Definition: interface.h:881
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:129
static_always_inline void crypto_enqueue_ops(vlib_main_t *vm, crypto_worker_main_t *cwm, u32 node_index, u32 error, u8 numa, u8 encrypt)
Definition: ipsec.h:312
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
u8 * format_ipsec_crypto_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:78
unsigned int u32
Definition: types.h:88
u32 padding
Definition: vhost_user.h:105
crypto_alg_t * auth_algs
Definition: ipsec.h:167
static const u8 pad_data[]
Definition: ipsec.h:176
bool is_ip6
Definition: ip.api:43
u32 next
Definition: ipsec.h:64
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:534
vlib_node_registration_t dpdk_esp6_encrypt_tun_node
(constructor) VLIB_REGISTER_NODE (dpdk_esp6_encrypt_tun_node)
Definition: esp_encrypt.c:690
static index_t ipsec_tun_protect_get_sa_out(adj_index_t ai)
Definition: ipsec_tun.h:169
crypto_alg_t * cipher_algs
Definition: ipsec.h:166
u32 salt
Definition: ipsec_sa.h:184
vnet_main_t * vnet_main
Definition: ipsec.h:118
unsigned short u16
Definition: types.h:57
#define rte_mbuf_from_vlib_buffer(x)
Definition: buffer.h:19
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
#define PREDICT_FALSE(x)
Definition: clib.h:120
#define always_inline
Definition: ipsec.h:28
vl_api_address_t dst
Definition: gre.api:55
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:391
u8 encrypt
Definition: ipsec.h:66
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
Definition: node_funcs.h:1231
u8 len
Definition: ip_types.api:92
ip46_address_t tunnel_dst_addr
Definition: ipsec_sa.h:176
u8 aad[16]
Definition: ipsec.h:69
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
vlib_node_registration_t dpdk_esp4_encrypt_tun_node
(constructor) VLIB_REGISTER_NODE (dpdk_esp4_encrypt_tun_node)
Definition: esp_encrypt.c:667
u16 n_vectors
Definition: node.h:396
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:219
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:80
dpdk_gcm_cnt_blk cb
Definition: ipsec.h:68
#define foreach_esp_encrypt_error
Definition: esp_encrypt.c:45
vlib_node_registration_t dpdk_esp6_encrypt_node
(constructor) VLIB_REGISTER_NODE (dpdk_esp6_encrypt_node)
Definition: esp_encrypt.c:644
#define clib_warning(format, args...)
Definition: error.h:59
u8 data[]
Packet data.
Definition: buffer.h:181
static char * esp_encrypt_error_strings[]
Definition: esp_encrypt.c:62
#define ARRAY_LEN(x)
Definition: clib.h:67
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:483
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1582
u8 * format_esp_header(u8 *s, va_list *args)
Definition: esp_format.c:23
u8 iv_len
Definition: ipsec.h:89
vlib_combined_counter_main_t ipsec_sa_counters
SA packet & bytes counters.
Definition: ipsec_sa.c:27
#define ASSERT(truth)
ipsec_sa_t * sad
Definition: ipsec.h:107
esp_encrypt_error_t
Definition: esp_encrypt.c:54
crypto_worker_main_t * workers_main
Definition: ipsec.h:163
u32 seq
Definition: esp.h:29
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:248
format_function_t format_ip6_header
Definition: format.h:95
u32 spi
Definition: esp.h:26
u8 boundary
Definition: ipsec.h:91
crypto_resource_t * resource
Definition: ipsec.h:165
static uword is_pow2(uword x)
Definition: clib.h:252
struct _vlib_node_registration vlib_node_registration_t
template key/value backing page structure
Definition: bihash_doc.h:44
Definition: defs.h:47
static uword dpdk_esp_encrypt_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame, int is_ip6, int is_tun)
Definition: esp_encrypt.c:114
vl_api_address_t ip
Definition: l2.api:501
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:297
ipsec_crypto_alg_t
Definition: ipsec_sa.h:37
struct rte_crypto_op ** ops
Definition: ipsec.h:76
struct clib_bihash_value offset
template key/value backing page structure
enum rte_crypto_sym_xform_type type
Definition: ipsec.h:86
#define vnet_buffer(b)
Definition: buffer.h:417
static_always_inline u16 get_resource(crypto_worker_main_t *cwm, ipsec_sa_t *sa)
Definition: ipsec.h:259
ipsec_crypto_alg_t crypto_alg
Definition: ipsec_sa.h:167
#define VLIB_NODE_FLAG_IS_OUTPUT
Definition: node.h:295
#define foreach_esp_encrypt_next
Definition: esp_encrypt.c:31
void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace.c:577
static void * vlib_buffer_put_uninit(vlib_buffer_t *b, u16 size)
Append uninitialized data to buffer.
Definition: buffer.h:321
static int ip4_header_bytes(const ip4_header_t *i)
Definition: ip4_packet.h:190
u8 ip_version_and_header_length
Definition: ip4_packet.h:93
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
struct rte_crypto_op * ops[VLIB_FRAME_SIZE]
Definition: ipsec.h:127
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
static u16 ip4_header_checksum(ip4_header_t *i)
Definition: ip4_packet.h:314
vl_api_interface_index_t sw_if_index
Definition: wireguard.api:33
static_always_inline dpdk_op_priv_t * crypto_op_get_priv(struct rte_crypto_op *op)
Definition: ipsec.h:208