FD.io VPP  v20.05.1-6-gf53edbc3b
Vector Packet Processing
esp_encrypt.c
Go to the documentation of this file.
1 /*
2  * esp_encrypt.c : IPSec ESP encrypt node using DPDK Cryptodev
3  *
4  * Copyright (c) 2017 Intel and/or its affiliates.
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at:
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  */
17 
18 #include <vnet/vnet.h>
19 #include <vnet/api_errno.h>
20 #include <vnet/ip/ip.h>
21 
22 #include <vnet/ipsec/ipsec.h>
23 #include <vnet/ipsec/esp.h>
24 #include <vnet/udp/udp.h>
25 #include <dpdk/buffer.h>
26 #include <dpdk/ipsec/ipsec.h>
27 #include <vnet/ipsec/ipsec_tun.h>
28 #include <dpdk/device/dpdk.h>
29 #include <dpdk/device/dpdk_priv.h>
30 
31 #define foreach_esp_encrypt_next \
32 _(DROP, "error-drop") \
33 _(IP4_LOOKUP, "ip4-lookup") \
34 _(IP6_LOOKUP, "ip6-lookup") \
35 _(INTERFACE_OUTPUT, "interface-output")
36 
37 #define _(v, s) ESP_ENCRYPT_NEXT_##v,
38 typedef enum
39 {
41 #undef _
44 
45 #define foreach_esp_encrypt_error \
46  _(RX_PKTS, "ESP pkts received") \
47  _(SEQ_CYCLED, "Sequence number cycled") \
48  _(ENQ_FAIL, "Enqueue failed to crypto device") \
49  _(DISCARD, "Not enough crypto operations, discarding frame") \
50  _(SESSION, "Failed to get crypto session") \
51  _(NOSUP, "Cipher/Auth not supported")
52 
53 
54 typedef enum
55 {
56 #define _(sym,str) ESP_ENCRYPT_ERROR_##sym,
58 #undef _
61 
62 static char *esp_encrypt_error_strings[] = {
63 #define _(sym,string) string,
65 #undef _
66 };
67 
72 
73 typedef struct
74 {
77  u8 packet_data[64];
79 
80 /* packet trace format function */
81 static u8 *
82 format_esp_encrypt_trace (u8 * s, va_list * args)
83 {
84  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
85  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
86  esp_encrypt_trace_t *t = va_arg (*args, esp_encrypt_trace_t *);
88  u32 indent = format_get_indent (s), offset;
89 
90  s = format (s, "cipher %U auth %U\n",
93 
94  if ((ih4->ip_version_and_header_length & 0xF0) == 0x60)
95  {
96  s = format (s, "%U%U", format_white_space, indent,
97  format_ip6_header, ih4);
98  offset = sizeof (ip6_header_t);
99  }
100  else
101  {
102  s = format (s, "%U%U", format_white_space, indent,
103  format_ip4_header, ih4);
104  offset = ip4_header_bytes (ih4);
105  }
106 
107  s = format (s, "\n%U%U", format_white_space, indent,
109 
110  return s;
111 }
112 
116  vlib_frame_t * from_frame, int is_ip6, int is_tun)
117 {
118  u32 n_left_from, *from, *to_next, next_index, thread_index;
119  ipsec_main_t *im = &ipsec_main;
120  vnet_main_t *vnm = im->vnet_main;
124  crypto_resource_t *res = 0;
125  ipsec_sa_t *sa0 = 0;
126  crypto_alg_t *cipher_alg = 0, *auth_alg = 0;
127  struct rte_cryptodev_sym_session *session = 0;
128  u32 ret, last_sa_index = ~0;
129  u8 numa = rte_socket_id ();
130  u8 is_aead = 0;
131  crypto_worker_main_t *cwm =
132  vec_elt_at_index (dcm->workers_main, thread_idx);
133  struct rte_crypto_op **ops = cwm->ops;
134 
135  from = vlib_frame_vector_args (from_frame);
136  n_left_from = from_frame->n_vectors;
137  thread_index = vm->thread_index;
138 
139  ret = crypto_alloc_ops (numa, ops, n_left_from);
140  if (ret)
141  {
142  if (is_ip6)
144  ESP_ENCRYPT_ERROR_DISCARD, 1);
145  else
147  ESP_ENCRYPT_ERROR_DISCARD, 1);
148  /* Discard whole frame */
149  return n_left_from;
150  }
151 
152  next_index = ESP_ENCRYPT_NEXT_DROP;
153 
154  while (n_left_from > 0)
155  {
156  u32 n_left_to_next;
157 
158  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
159 
160  while (n_left_from > 0 && n_left_to_next > 0)
161  {
162  clib_error_t *error;
163  u32 bi0, bi1;
164  vlib_buffer_t *b0, *b1;
165  u32 sa_index0;
166  ip4_and_esp_header_t *ih0, *oh0 = 0;
167  ip6_and_esp_header_t *ih6_0, *oh6_0 = 0;
168  ip4_and_udp_and_esp_header_t *ouh0 = 0;
169  esp_header_t *esp0;
170  esp_footer_t *f0;
171  u8 next_hdr_type;
172  u32 iv_size;
173  u16 orig_sz;
174  u8 trunc_size;
175  u16 rewrite_len;
176  u16 udp_encap_adv = 0;
177  struct rte_mbuf *mb0;
178  struct rte_crypto_op *op;
179  u16 res_idx;
180 
181  bi0 = from[0];
182  from += 1;
183  n_left_from -= 1;
184 
185  b0 = vlib_get_buffer (vm, bi0);
186  ih0 = vlib_buffer_get_current (b0);
187  mb0 = rte_mbuf_from_vlib_buffer (b0);
188 
189  /* ih0/ih6_0 */
190  CLIB_PREFETCH (ih0, sizeof (ih6_0[0]), LOAD);
191  /* f0 */
192  CLIB_PREFETCH (vlib_buffer_get_tail (b0), 20, STORE);
193  /* mb0 */
194  CLIB_PREFETCH (mb0, CLIB_CACHE_LINE_BYTES, STORE);
195 
196  if (n_left_from > 1)
197  {
198  bi1 = from[1];
199  b1 = vlib_get_buffer (vm, bi1);
200 
203  CLIB_CACHE_LINE_BYTES, STORE);
204  }
205 
206  op = ops[0];
207  ops += 1;
208  ASSERT (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED);
209 
210  dpdk_op_priv_t *priv = crypto_op_get_priv (op);
211  /* store bi in op private */
212  priv->bi = bi0;
213  priv->encrypt = 1;
214 
215  u16 op_len =
216  sizeof (op[0]) + sizeof (op[0].sym[0]) + sizeof (priv[0]);
217  CLIB_PREFETCH (op, op_len, STORE);
218 
219  if (is_tun)
220  {
221  /* we are on a ipsec tunnel's feature arc */
222  vnet_buffer (b0)->ipsec.sad_index =
223  sa_index0 = ipsec_tun_protect_get_sa_out
224  (vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
225  }
226  else
227  sa_index0 = vnet_buffer (b0)->ipsec.sad_index;
228 
229  if (sa_index0 != last_sa_index)
230  {
231  sa0 = pool_elt_at_index (im->sad, sa_index0);
232 
233  cipher_alg =
235  auth_alg = vec_elt_at_index (dcm->auth_algs, sa0->integ_alg);
236 
237  is_aead = (cipher_alg->type == RTE_CRYPTO_SYM_XFORM_AEAD);
238 
239  if (is_aead)
240  auth_alg = cipher_alg;
241 
242  res_idx = get_resource (cwm, sa0);
243 
244  if (PREDICT_FALSE (res_idx == (u16) ~ 0))
245  {
246  clib_warning ("unsupported SA by thread index %u",
247  thread_idx);
248  if (is_ip6)
251  ESP_ENCRYPT_ERROR_NOSUP, 1);
252  else
255  ESP_ENCRYPT_ERROR_NOSUP, 1);
256  to_next[0] = bi0;
257  to_next += 1;
258  n_left_to_next -= 1;
259  goto trace;
260  }
261  res = vec_elt_at_index (dcm->resource, res_idx);
262 
263  error = crypto_get_session (&session, sa_index0, res, cwm, 1);
264  if (PREDICT_FALSE (error || !session))
265  {
266  clib_warning ("failed to get crypto session");
267  if (is_ip6)
270  ESP_ENCRYPT_ERROR_SESSION,
271  1);
272  else
275  ESP_ENCRYPT_ERROR_SESSION,
276  1);
277  to_next[0] = bi0;
278  to_next += 1;
279  n_left_to_next -= 1;
280  goto trace;
281  }
282 
283  last_sa_index = sa_index0;
284  }
285 
286  if (PREDICT_FALSE (esp_seq_advance (sa0)))
287  {
289  ("sequence number counter has cycled SPI %u (0x%08x)",
290  sa0->spi, sa0->spi);
291  if (is_ip6)
294  ESP_ENCRYPT_ERROR_SEQ_CYCLED, 1);
295  else
298  ESP_ENCRYPT_ERROR_SEQ_CYCLED, 1);
299  //TODO: rekey SA
300  to_next[0] = bi0;
301  to_next += 1;
302  n_left_to_next -= 1;
303  goto trace;
304  }
305 
306  orig_sz = b0->current_length;
307 
308  /* TODO multi-seg support - total_length_not_including_first_buffer */
310  (&ipsec_sa_counters, thread_index, sa_index0,
311  1, b0->current_length);
312 
313  /* Update tunnel interface tx counters */
314  if (is_tun)
317  thread_index, vnet_buffer (b0)->sw_if_index[VLIB_TX],
318  1, b0->current_length);
319 
320  res->ops[res->n_ops] = op;
321  res->bi[res->n_ops] = bi0;
322  res->n_ops += 1;
323 
324  dpdk_gcm_cnt_blk *icb = &priv->cb;
325 
326  crypto_set_icb (icb, sa0->salt, sa0->seq, sa0->seq_hi);
327 
328  iv_size = cipher_alg->iv_len;
329  trunc_size = auth_alg->trunc_size;
330 
331  /* if UDP encapsulation is used adjust the address of the IP header */
332  if (ipsec_sa_is_set_UDP_ENCAP (sa0) && !is_ip6)
333  udp_encap_adv = sizeof (udp_header_t);
334 
335  if (ipsec_sa_is_set_IS_TUNNEL (sa0))
336  {
337  rewrite_len = 0;
338  if (!ipsec_sa_is_set_IS_TUNNEL_V6 (sa0)) /* ip4 */
339  {
340  /* in tunnel mode send it back to FIB */
341  priv->next = DPDK_CRYPTO_INPUT_NEXT_IP4_LOOKUP;
342  u8 adv = sizeof (ip4_header_t) + udp_encap_adv +
343  sizeof (esp_header_t) + iv_size;
344  vlib_buffer_advance (b0, -adv);
345  oh0 = vlib_buffer_get_current (b0);
346  ouh0 = vlib_buffer_get_current (b0);
347  next_hdr_type = (is_ip6 ?
348  IP_PROTOCOL_IPV6 : IP_PROTOCOL_IP_IN_IP);
349  /*
350  * oh0->ip4.ip_version_and_header_length = 0x45;
351  * oh0->ip4.tos = ih0->ip4.tos;
352  * oh0->ip4.fragment_id = 0;
353  * oh0->ip4.flags_and_fragment_offset = 0;
354  */
355  oh0->ip4.checksum_data_64[0] =
356  clib_host_to_net_u64 (0x45ULL << 56);
357  /*
358  * oh0->ip4.ttl = 254;
359  * oh0->ip4.protocol = IP_PROTOCOL_IPSEC_ESP;
360  */
361  oh0->ip4.checksum_data_32[2] =
362  clib_host_to_net_u32 (0xfe320000);
363 
364  oh0->ip4.src_address.as_u32 =
365  sa0->tunnel_src_addr.ip4.as_u32;
366  oh0->ip4.dst_address.as_u32 =
367  sa0->tunnel_dst_addr.ip4.as_u32;
368 
369  if (ipsec_sa_is_set_UDP_ENCAP (sa0))
370  {
371  oh0->ip4.protocol = IP_PROTOCOL_UDP;
372  esp0 = &ouh0->esp;
373  }
374  else
375  esp0 = &oh0->esp;
376  esp0->spi = clib_host_to_net_u32 (sa0->spi);
377  esp0->seq = clib_host_to_net_u32 (sa0->seq);
378  }
379  else
380  {
381  /* ip6 */
382  /* in tunnel mode send it back to FIB */
383  priv->next = DPDK_CRYPTO_INPUT_NEXT_IP6_LOOKUP;
384 
385  u8 adv =
386  sizeof (ip6_header_t) + sizeof (esp_header_t) + iv_size;
387  vlib_buffer_advance (b0, -adv);
388  ih6_0 = (ip6_and_esp_header_t *) ih0;
389  oh6_0 = vlib_buffer_get_current (b0);
390 
391  next_hdr_type = (is_ip6 ?
392  IP_PROTOCOL_IPV6 : IP_PROTOCOL_IP_IN_IP);
393 
394  oh6_0->ip6.ip_version_traffic_class_and_flow_label =
395  ih6_0->ip6.ip_version_traffic_class_and_flow_label;
396 
397  oh6_0->ip6.protocol = IP_PROTOCOL_IPSEC_ESP;
398  oh6_0->ip6.hop_limit = 254;
399  oh6_0->ip6.src_address.as_u64[0] =
400  sa0->tunnel_src_addr.ip6.as_u64[0];
401  oh6_0->ip6.src_address.as_u64[1] =
402  sa0->tunnel_src_addr.ip6.as_u64[1];
403  oh6_0->ip6.dst_address.as_u64[0] =
404  sa0->tunnel_dst_addr.ip6.as_u64[0];
405  oh6_0->ip6.dst_address.as_u64[1] =
406  sa0->tunnel_dst_addr.ip6.as_u64[1];
407  esp0 = &oh6_0->esp;
408  oh6_0->esp.spi = clib_host_to_net_u32 (sa0->spi);
409  oh6_0->esp.seq = clib_host_to_net_u32 (sa0->seq);
410  }
411 
412  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
413  }
414  else /* transport mode */
415  {
416  if (is_tun)
417  {
418  rewrite_len = 0;
419  priv->next = DPDK_CRYPTO_INPUT_NEXT_MIDCHAIN;
420  }
421  else
422  {
423  priv->next = DPDK_CRYPTO_INPUT_NEXT_INTERFACE_OUTPUT;
424  rewrite_len = vnet_buffer (b0)->ip.save_rewrite_length;
425  }
426  u16 adv = sizeof (esp_header_t) + iv_size + udp_encap_adv;
427  vlib_buffer_advance (b0, -adv - rewrite_len);
428  u8 *src = ((u8 *) ih0) - rewrite_len;
430  oh0 = vlib_buffer_get_current (b0) + rewrite_len;
431  ouh0 = vlib_buffer_get_current (b0) + rewrite_len;
432 
433  if (is_ip6)
434  {
435  orig_sz -= sizeof (ip6_header_t);
436  ih6_0 = (ip6_and_esp_header_t *) ih0;
437  next_hdr_type = ih6_0->ip6.protocol;
438  memmove (dst, src, rewrite_len + sizeof (ip6_header_t));
439  oh6_0 = (ip6_and_esp_header_t *) oh0;
440  oh6_0->ip6.protocol = IP_PROTOCOL_IPSEC_ESP;
441  esp0 = &oh6_0->esp;
442  }
443  else /* ipv4 */
444  {
445  u16 ip_size = ip4_header_bytes (&ih0->ip4);
446  orig_sz -= ip_size;
447  next_hdr_type = ih0->ip4.protocol;
448  memmove (dst, src, rewrite_len + ip_size);
449  oh0->ip4.protocol = IP_PROTOCOL_IPSEC_ESP;
450  esp0 = (esp_header_t *) (((u8 *) oh0) + ip_size);
451  if (ipsec_sa_is_set_UDP_ENCAP (sa0))
452  {
453  oh0->ip4.protocol = IP_PROTOCOL_UDP;
454  esp0 = (esp_header_t *)
455  (((u8 *) oh0) + ip_size + udp_encap_adv);
456  }
457  else
458  {
459  oh0->ip4.protocol = IP_PROTOCOL_IPSEC_ESP;
460  esp0 = (esp_header_t *) (((u8 *) oh0) + ip_size);
461  }
462  }
463  esp0->spi = clib_host_to_net_u32 (sa0->spi);
464  esp0->seq = clib_host_to_net_u32 (sa0->seq);
465  }
466 
467  if (ipsec_sa_is_set_UDP_ENCAP (sa0) && ouh0)
468  {
469  ouh0->udp.src_port = clib_host_to_net_u16 (UDP_DST_PORT_ipsec);
470  ouh0->udp.dst_port = clib_host_to_net_u16 (UDP_DST_PORT_ipsec);
471  ouh0->udp.checksum = 0;
472  }
473  ASSERT (is_pow2 (cipher_alg->boundary));
474  u16 mask = cipher_alg->boundary - 1;
475  u16 pad_payload_len = ((orig_sz + 2) + mask) & ~mask;
476  u8 pad_bytes = pad_payload_len - 2 - orig_sz;
477 
478  u8 *padding =
479  vlib_buffer_put_uninit (b0, pad_bytes + 2 + trunc_size);
480 
481  /* The extra pad bytes would be overwritten by the digest */
482  if (pad_bytes)
483  clib_memcpy_fast (padding, pad_data, 16);
484 
485  f0 = (esp_footer_t *) (padding + pad_bytes);
486  f0->pad_length = pad_bytes;
487  f0->next_header = next_hdr_type;
488 
489  if (oh6_0)
490  {
491  u16 len = b0->current_length - sizeof (ip6_header_t);
492  oh6_0->ip6.payload_length =
493  clib_host_to_net_u16 (len - rewrite_len);
494  }
495  else if (oh0)
496  {
497  oh0->ip4.length =
498  clib_host_to_net_u16 (b0->current_length - rewrite_len);
499  oh0->ip4.checksum = ip4_header_checksum (&oh0->ip4);
500  if (ipsec_sa_is_set_UDP_ENCAP (sa0) && ouh0)
501  {
502  ouh0->udp.length =
503  clib_host_to_net_u16 (clib_net_to_host_u16
504  (ouh0->ip4.length) -
505  ip4_header_bytes (&ouh0->ip4));
506  }
507  }
508  else /* should never happen */
509  clib_warning ("No outer header found for ESP packet");
510 
511  b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
512 
513  /* mbuf packet starts at ESP header */
514  mb0->data_len = vlib_buffer_get_tail (b0) - ((u8 *) esp0);
515  mb0->pkt_len = vlib_buffer_get_tail (b0) - ((u8 *) esp0);
516  mb0->data_off = ((void *) esp0) - mb0->buf_addr;
517 
518  u32 cipher_off, cipher_len, auth_len = 0;
519  u32 *aad = NULL;
520 
521  u8 *digest = vlib_buffer_get_tail (b0) - trunc_size;
522  u64 digest_paddr =
523  mb0->buf_physaddr + digest - ((u8 *) mb0->buf_addr);
524 
525  if (!is_aead && (cipher_alg->alg == RTE_CRYPTO_CIPHER_AES_CBC ||
526  cipher_alg->alg == RTE_CRYPTO_CIPHER_NULL))
527  {
528  cipher_off = sizeof (esp_header_t);
529  cipher_len = iv_size + pad_payload_len;
530  }
531  else /* CTR/GCM */
532  {
533  u32 *esp_iv = (u32 *) (esp0 + 1);
534  esp_iv[0] = sa0->seq;
535  esp_iv[1] = sa0->seq_hi;
536 
537  cipher_off = sizeof (esp_header_t) + iv_size;
538  cipher_len = pad_payload_len;
539  }
540 
541  if (is_aead)
542  {
543  aad = (u32 *) priv->aad;
544  aad[0] = esp0->spi;
545 
546  /* aad[3] should always be 0 */
547  if (PREDICT_FALSE (ipsec_sa_is_set_USE_ESN (sa0)))
548  {
549  aad[1] = clib_host_to_net_u32 (sa0->seq_hi);
550  aad[2] = esp0->seq;
551  }
552  else
553  {
554  aad[1] = esp0->seq;
555  aad[2] = 0;
556  }
557  }
558  else
559  {
560  auth_len =
561  vlib_buffer_get_tail (b0) - ((u8 *) esp0) - trunc_size;
562  if (ipsec_sa_is_set_USE_ESN (sa0))
563  {
564  u32 *_digest = (u32 *) digest;
565  _digest[0] = clib_host_to_net_u32 (sa0->seq_hi);
566  auth_len += 4;
567  }
568  }
569 
570  crypto_op_setup (is_aead, mb0, op, session, cipher_off, cipher_len,
571  0, auth_len, (u8 *) aad, digest, digest_paddr);
572 
573  trace:
574  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
575  {
576  esp_encrypt_trace_t *tr =
577  vlib_add_trace (vm, node, b0, sizeof (*tr));
578  tr->crypto_alg = sa0->crypto_alg;
579  tr->integ_alg = sa0->integ_alg;
580  u8 *p = vlib_buffer_get_current (b0);
581  if (!ipsec_sa_is_set_IS_TUNNEL (sa0) && !is_tun)
582  p += vnet_buffer (b0)->ip.save_rewrite_length;
583  clib_memcpy_fast (tr->packet_data, p, sizeof (tr->packet_data));
584  }
585  }
586  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
587  }
588  if (is_ip6)
589  {
591  (is_tun ?
593  dpdk_esp6_encrypt_node.index),
594  ESP_ENCRYPT_ERROR_RX_PKTS,
595  from_frame->n_vectors);
596 
598  ESP_ENCRYPT_ERROR_ENQ_FAIL, numa, 1 /* encrypt */ );
599  }
600  else
601  {
603  (is_tun ?
605  dpdk_esp4_encrypt_node.index),
606  ESP_ENCRYPT_ERROR_RX_PKTS,
607  from_frame->n_vectors);
608 
610  ESP_ENCRYPT_ERROR_ENQ_FAIL, numa, 1 /* encrypt */ );
611  }
612 
613  crypto_free_ops (numa, ops, cwm->ops + from_frame->n_vectors - ops);
614 
615  return from_frame->n_vectors;
616 }
617 
620  vlib_frame_t * from_frame)
621 {
622  return dpdk_esp_encrypt_inline (vm, node, from_frame, 0 /*is_ip6 */ , 0);
623 }
624 
625 /* *INDENT-OFF* */
627  .name = "dpdk-esp4-encrypt",
628  .flags = VLIB_NODE_FLAG_IS_OUTPUT,
629  .vector_size = sizeof (u32),
630  .format_trace = format_esp_encrypt_trace,
631  .n_errors = ARRAY_LEN (esp_encrypt_error_strings),
632  .error_strings = esp_encrypt_error_strings,
633  .n_next_nodes = 1,
634  .next_nodes =
635  {
636  [ESP_ENCRYPT_NEXT_DROP] = "error-drop",
637  }
638 };
639 /* *INDENT-ON* */
640 
643  vlib_frame_t * from_frame)
644 {
645  return dpdk_esp_encrypt_inline (vm, node, from_frame, 1 /*is_ip6 */ , 0);
646 }
647 
648 /* *INDENT-OFF* */
650  .name = "dpdk-esp6-encrypt",
651  .flags = VLIB_NODE_FLAG_IS_OUTPUT,
652  .vector_size = sizeof (u32),
653  .format_trace = format_esp_encrypt_trace,
654  .n_errors = ARRAY_LEN (esp_encrypt_error_strings),
655  .error_strings = esp_encrypt_error_strings,
656  .n_next_nodes = 1,
657  .next_nodes =
658  {
659  [ESP_ENCRYPT_NEXT_DROP] = "error-drop",
660  }
661 };
662 /* *INDENT-ON* */
663 
666  vlib_frame_t * from_frame)
667 {
668  return dpdk_esp_encrypt_inline (vm, node, from_frame, 0 /*is_ip6 */ , 1);
669 }
670 
671 /* *INDENT-OFF* */
673  .name = "dpdk-esp4-encrypt-tun",
674  .flags = VLIB_NODE_FLAG_IS_OUTPUT,
675  .vector_size = sizeof (u32),
676  .format_trace = format_esp_encrypt_trace,
677  .n_errors = ARRAY_LEN (esp_encrypt_error_strings),
678  .error_strings = esp_encrypt_error_strings,
679  .n_next_nodes = 1,
680  .next_nodes =
681  {
682  [ESP_ENCRYPT_NEXT_DROP] = "error-drop",
683  }
684 };
685 /* *INDENT-ON* */
686 
689  vlib_frame_t * from_frame)
690 {
691  return dpdk_esp_encrypt_inline (vm, node, from_frame, 1 /*is_ip6 */ , 1);
692 }
693 
694 /* *INDENT-OFF* */
696  .name = "dpdk-esp6-encrypt-tun",
697  .flags = VLIB_NODE_FLAG_IS_OUTPUT,
698  .vector_size = sizeof (u32),
699  .format_trace = format_esp_encrypt_trace,
700  .n_errors = ARRAY_LEN (esp_encrypt_error_strings),
701  .error_strings = esp_encrypt_error_strings,
702  .n_next_nodes = 1,
703  .next_nodes =
704  {
705  [ESP_ENCRYPT_NEXT_DROP] = "error-drop",
706  }
707 };
708 /* *INDENT-ON* */
709 
710 /*
711  * fd.io coding-style-patch-verification: ON
712  *
713  * Local Variables:
714  * eval: (c-set-style "gnu")
715  * End:
716  */
u32 alg
Definition: ipsec.h:87
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
u8 * format_ipsec_integ_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:110
static_always_inline void crypto_op_setup(u8 is_aead, struct rte_mbuf *mb0, struct rte_crypto_op *op, void *session, u32 cipher_off, u32 cipher_len, u32 auth_off, u32 auth_len, u8 *aad, u8 *digest, u64 digest_paddr)
Definition: ipsec.h:357
static vlib_cli_command_t trace
(constructor) VLIB_CLI_COMMAND (trace)
Definition: vlib_api_cli.c:899
static u8 * vlib_buffer_get_tail(vlib_buffer_t *b)
Get pointer to the end of buffer&#39;s data.
Definition: buffer.h:310
#define CLIB_UNUSED(x)
Definition: clib.h:86
ip46_address_t tunnel_src_addr
Definition: ipsec_sa.h:175
vlib_node_registration_t dpdk_esp4_encrypt_node
(constructor) VLIB_REGISTER_NODE (dpdk_esp4_encrypt_node)
Definition: esp_encrypt.c:626
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
Definition: counter.h:220
vnet_interface_main_t interface_main
Definition: vnet.h:56
format_function_t format_ip4_header
Definition: format.h:81
ipsec_integ_alg_t
Definition: ipsec_sa.h:59
esp_encrypt_next_t
Definition: esp_encrypt.c:38
unsigned long u64
Definition: types.h:89
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
static_always_inline i32 crypto_alloc_ops(u8 numa, struct rte_crypto_op **ops, u32 n)
Definition: ipsec.h:284
ipsec_integ_alg_t integ_alg
Definition: ipsec_sa.h:171
#define thread_idx
Definition: tls_async.c:40
u32 thread_index
Definition: main.h:218
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
static u8 * format_esp_encrypt_trace(u8 *s, va_list *args)
Definition: esp_encrypt.c:82
static_always_inline void crypto_set_icb(dpdk_gcm_cnt_blk *icb, u32 salt, u32 seq, u32 seq_hi)
Definition: ipsec.h:349
vl_api_address_t src
Definition: gre.api:54
ipsec_integ_alg_t integ_alg
Definition: esp_encrypt.c:76
static u32 format_get_indent(u8 *s)
Definition: format.h:72
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
ipsec_crypto_alg_t crypto_alg
Definition: esp_encrypt.c:75
#define VLIB_NODE_FN(node)
Definition: node.h:202
unsigned char u8
Definition: types.h:56
u32 bi[VLIB_FRAME_SIZE]
Definition: ipsec.h:128
u32 seq_hi
Definition: ipsec_sa.h:122
static int esp_seq_advance(ipsec_sa_t *sa)
Definition: esp.h:84
static_always_inline clib_error_t * crypto_get_session(struct rte_cryptodev_sym_session **session, u32 sa_idx, crypto_resource_t *res, crypto_worker_main_t *cwm, u8 is_outbound)
Definition: ipsec.h:238
static_always_inline void crypto_free_ops(u8 numa, struct rte_crypto_op **ops, u32 n)
Definition: ipsec.h:300
dpdk_crypto_main_t dpdk_crypto_main
Definition: ipsec.c:25
vl_api_interface_index_t sw_if_index
Definition: gre.api:53
ipsec_main_t ipsec_main
Definition: ipsec.c:28
vlib_combined_counter_main_t * combined_sw_if_counters
Definition: interface.h:867
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:129
static_always_inline void crypto_enqueue_ops(vlib_main_t *vm, crypto_worker_main_t *cwm, u32 node_index, u32 error, u8 numa, u8 encrypt)
Definition: ipsec.h:312
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
u8 * format_ipsec_crypto_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:78
unsigned int u32
Definition: types.h:88
u32 padding
Definition: vhost_user.h:151
crypto_alg_t * auth_algs
Definition: ipsec.h:167
static const u8 pad_data[]
Definition: ipsec.h:176
bool is_ip6
Definition: ip.api:43
u32 next
Definition: ipsec.h:64
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:534
vlib_node_registration_t dpdk_esp6_encrypt_tun_node
(constructor) VLIB_REGISTER_NODE (dpdk_esp6_encrypt_tun_node)
Definition: esp_encrypt.c:695
static index_t ipsec_tun_protect_get_sa_out(adj_index_t ai)
Definition: ipsec_tun.h:159
crypto_alg_t * cipher_algs
Definition: ipsec.h:166
u32 salt
Definition: ipsec_sa.h:184
vnet_main_t * vnet_main
Definition: ipsec.h:118
unsigned short u16
Definition: types.h:57
#define rte_mbuf_from_vlib_buffer(x)
Definition: buffer.h:19
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
#define PREDICT_FALSE(x)
Definition: clib.h:118
#define always_inline
Definition: ipsec.h:28
vl_api_address_t dst
Definition: gre.api:55
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:338
vlib_main_t * vm
Definition: in2out_ed.c:1599
u8 encrypt
Definition: ipsec.h:66
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
Definition: node_funcs.h:1150
u8 len
Definition: ip_types.api:92
ip46_address_t tunnel_dst_addr
Definition: ipsec_sa.h:176
u8 aad[16]
Definition: ipsec.h:69
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
vlib_node_registration_t dpdk_esp4_encrypt_tun_node
(constructor) VLIB_REGISTER_NODE (dpdk_esp4_encrypt_tun_node)
Definition: esp_encrypt.c:672
u16 n_vectors
Definition: node.h:399
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:218
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:80
dpdk_gcm_cnt_blk cb
Definition: ipsec.h:68
#define foreach_esp_encrypt_error
Definition: esp_encrypt.c:45
vlib_node_registration_t dpdk_esp6_encrypt_node
(constructor) VLIB_REGISTER_NODE (dpdk_esp6_encrypt_node)
Definition: esp_encrypt.c:649
#define clib_warning(format, args...)
Definition: error.h:59
u8 data[]
Packet data.
Definition: buffer.h:181
static char * esp_encrypt_error_strings[]
Definition: esp_encrypt.c:62
#define ARRAY_LEN(x)
Definition: clib.h:66
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:483
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1599
u8 * format_esp_header(u8 *s, va_list *args)
Definition: esp_format.c:23
u8 iv_len
Definition: ipsec.h:89
vlib_combined_counter_main_t ipsec_sa_counters
SA packet & bytes counters.
Definition: ipsec_sa.c:27
#define ASSERT(truth)
ipsec_sa_t * sad
Definition: ipsec.h:107
esp_encrypt_error_t
Definition: esp_encrypt.c:54
crypto_worker_main_t * workers_main
Definition: ipsec.h:163
u32 seq
Definition: esp.h:29
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:248
format_function_t format_ip6_header
Definition: format.h:95
u32 spi
Definition: esp.h:26
u8 boundary
Definition: ipsec.h:91
crypto_resource_t * resource
Definition: ipsec.h:165
static uword is_pow2(uword x)
Definition: clib.h:250
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
struct _vlib_node_registration vlib_node_registration_t
template key/value backing page structure
Definition: bihash_doc.h:44
Definition: defs.h:47
static uword dpdk_esp_encrypt_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame, int is_ip6, int is_tun)
Definition: esp_encrypt.c:114
vl_api_address_t ip
Definition: l2.api:501
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
ipsec_crypto_alg_t
Definition: ipsec_sa.h:37
struct rte_crypto_op ** ops
Definition: ipsec.h:76
struct clib_bihash_value offset
template key/value backing page structure
enum rte_crypto_sym_xform_type type
Definition: ipsec.h:86
#define vnet_buffer(b)
Definition: buffer.h:417
static_always_inline u16 get_resource(crypto_worker_main_t *cwm, ipsec_sa_t *sa)
Definition: ipsec.h:259
ipsec_crypto_alg_t crypto_alg
Definition: ipsec_sa.h:167
#define VLIB_NODE_FLAG_IS_OUTPUT
Definition: node.h:298
#define foreach_esp_encrypt_next
Definition: esp_encrypt.c:31
static void * vlib_buffer_put_uninit(vlib_buffer_t *b, u16 size)
Append uninitialized data to buffer.
Definition: buffer.h:321
static int ip4_header_bytes(const ip4_header_t *i)
Definition: ip4_packet.h:235
u8 ip_version_and_header_length
Definition: ip4_packet.h:138
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
struct rte_crypto_op * ops[VLIB_FRAME_SIZE]
Definition: ipsec.h:127
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
static u16 ip4_header_checksum(ip4_header_t *i)
Definition: ip4_packet.h:247
static_always_inline dpdk_op_priv_t * crypto_op_get_priv(struct rte_crypto_op *op)
Definition: ipsec.h:208