FD.io VPP  v21.06-3-gbb25fbf28
Vector Packet Processing
esp_encrypt.c
Go to the documentation of this file.
1 /*
2  * esp_encrypt.c : IPSec ESP encrypt node
3  *
4  * Copyright (c) 2015 Cisco and/or its affiliates.
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at:
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  */
17 
18 #include <vnet/vnet.h>
19 #include <vnet/api_errno.h>
20 #include <vnet/ip/ip.h>
21 
22 #include <vnet/crypto/crypto.h>
23 
24 #include <vnet/ipsec/ipsec.h>
25 #include <vnet/ipsec/ipsec_tun.h>
26 #include <vnet/ipsec/esp.h>
27 #include <vnet/tunnel/tunnel_dp.h>
28 
29 #define foreach_esp_encrypt_next \
30  _ (DROP4, "ip4-drop") \
31  _ (DROP6, "ip6-drop") \
32  _ (DROP_MPLS, "mpls-drop") \
33  _ (HANDOFF4, "handoff4") \
34  _ (HANDOFF6, "handoff6") \
35  _ (HANDOFF_MPLS, "handoff-mpls") \
36  _ (INTERFACE_OUTPUT, "interface-output")
37 
38 #define _(v, s) ESP_ENCRYPT_NEXT_##v,
39 typedef enum
40 {
42 #undef _
45 
46 #define foreach_esp_encrypt_error \
47  _ (RX_PKTS, "ESP pkts received") \
48  _ (POST_RX_PKTS, "ESP-post pkts received") \
49  _ (HANDOFF, "Hand-off") \
50  _ (SEQ_CYCLED, "sequence number cycled (packet dropped)") \
51  _ (CRYPTO_ENGINE_ERROR, "crypto engine error (packet dropped)") \
52  _ (CRYPTO_QUEUE_FULL, "crypto queue full (packet dropped)") \
53  _ (NO_BUFFERS, "no buffers (packet dropped)")
54 
55 typedef enum
56 {
57 #define _(sym,str) ESP_ENCRYPT_ERROR_##sym,
59 #undef _
62 
63 static char *esp_encrypt_error_strings[] = {
64 #define _(sym,string) string,
66 #undef _
67 };
68 
69 typedef struct
70 {
76  ipsec_crypto_alg_t crypto_alg;
77  ipsec_integ_alg_t integ_alg;
79 
80 typedef struct
81 {
84 
85 /* packet trace format function */
86 static u8 *
87 format_esp_encrypt_trace (u8 * s, va_list * args)
88 {
89  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
90  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
91  esp_encrypt_trace_t *t = va_arg (*args, esp_encrypt_trace_t *);
92 
93  s =
94  format (s,
95  "esp: sa-index %d spi %u (0x%08x) seq %u sa-seq-hi %u crypto %U integrity %U%s",
96  t->sa_index, t->spi, t->spi, t->seq, t->sa_seq_hi,
99  t->udp_encap ? " udp-encap-enabled" : "");
100  return s;
101 }
102 
103 static u8 *
104 format_esp_post_encrypt_trace (u8 * s, va_list * args)
105 {
106  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
107  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
108  esp_encrypt_post_trace_t *t = va_arg (*args, esp_encrypt_post_trace_t *);
109 
110  s = format (s, "esp-post: next node index %u", t->next_index);
111  return s;
112 }
113 
114 /* pad packet in input buffer */
117  u8 icv_sz, vlib_node_runtime_t *node,
118  u16 buffer_data_size, uword total_len)
119 {
120  static const u8 pad_data[ESP_MAX_BLOCK_SIZE] = {
121  0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
122  0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x00,
123  };
124 
125  u16 min_length = total_len + sizeof (esp_footer_t);
126  u16 new_length = round_pow2 (min_length, esp_align);
127  u8 pad_bytes = new_length - min_length;
129  last[0]->current_length + pad_bytes);
130  u16 tail_sz = sizeof (esp_footer_t) + pad_bytes + icv_sz;
131 
132  if (last[0]->current_length + tail_sz > buffer_data_size)
133  {
134  u32 tmp_bi = 0;
135  if (vlib_buffer_alloc (vm, &tmp_bi, 1) != 1)
136  return 0;
137 
138  vlib_buffer_t *tmp = vlib_get_buffer (vm, tmp_bi);
139  last[0]->next_buffer = tmp_bi;
140  last[0]->flags |= VLIB_BUFFER_NEXT_PRESENT;
141  f = (esp_footer_t *) (vlib_buffer_get_current (tmp) + pad_bytes);
142  tmp->current_length += tail_sz;
143  last[0] = tmp;
144  }
145  else
146  last[0]->current_length += tail_sz;
147 
148  f->pad_length = pad_bytes;
149  if (pad_bytes)
150  {
151  ASSERT (pad_bytes <= ESP_MAX_BLOCK_SIZE);
152  pad_bytes = clib_min (ESP_MAX_BLOCK_SIZE, pad_bytes);
153  clib_memcpy_fast ((u8 *) f - pad_bytes, pad_data, pad_bytes);
154  }
155 
156  return &f->next_header;
157 }
158 
160 esp_update_ip4_hdr (ip4_header_t * ip4, u16 len, int is_transport, int is_udp)
161 {
162  ip_csum_t sum;
163  u16 old_len;
164 
165  len = clib_net_to_host_u16 (len);
166  old_len = ip4->length;
167 
168  if (is_transport)
169  {
170  u8 prot = is_udp ? IP_PROTOCOL_UDP : IP_PROTOCOL_IPSEC_ESP;
171 
172  sum = ip_csum_update (ip4->checksum, ip4->protocol,
173  prot, ip4_header_t, protocol);
174  ip4->protocol = prot;
175 
176  sum = ip_csum_update (sum, old_len, len, ip4_header_t, length);
177  }
178  else
179  sum = ip_csum_update (ip4->checksum, old_len, len, ip4_header_t, length);
180 
181  ip4->length = len;
182  ip4->checksum = ip_csum_fold (sum);
183 }
184 
187 {
188  clib_memcpy_fast (udp, &sa->udp_hdr, sizeof (udp_header_t));
189  udp->length = clib_net_to_host_u16 (len);
190 }
191 
194 {
195 #ifdef CLIB_HAVE_VEC128
196  static const u8x16 ext_hdr_types = {
197  IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS,
198  IP_PROTOCOL_IPV6_ROUTE,
199  IP_PROTOCOL_IPV6_FRAGMENTATION,
200  };
201 
202  return !u8x16_is_all_zero (ext_hdr_types == u8x16_splat (nexthdr));
203 #else
204  return ((nexthdr ^ IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) |
205  (nexthdr ^ IP_PROTOCOL_IPV6_ROUTE) |
206  (nexthdr ^ IP_PROTOCOL_IPV6_FRAGMENTATION) != 0);
207 #endif
208 }
209 
211 esp_get_ip6_hdr_len (ip6_header_t * ip6, ip6_ext_header_t ** ext_hdr)
212 {
213  /* this code assumes that HbH, route and frag headers will be before
214  others, if that is not the case, they will end up encrypted */
215  u8 len = sizeof (ip6_header_t);
216  ip6_ext_header_t *p;
217 
218  /* if next packet doesn't have ext header */
219  if (ext_hdr_is_pre_esp (ip6->protocol) == 0)
220  {
221  *ext_hdr = NULL;
222  return len;
223  }
224 
225  p = (void *) (ip6 + 1);
226  len += ip6_ext_header_len (p);
227 
228  while (ext_hdr_is_pre_esp (p->next_hdr))
229  {
230  len += ip6_ext_header_len (p);
231  p = ip6_ext_next_header (p);
232  }
233 
234  *ext_hdr = p;
235  return len;
236 }
237 
240  vnet_crypto_op_t * ops, vlib_buffer_t * b[],
241  u16 * nexts, vnet_crypto_op_chunk_t * chunks,
242  u16 drop_next)
243 {
244  u32 n_fail, n_ops = vec_len (ops);
245  vnet_crypto_op_t *op = ops;
246 
247  if (n_ops == 0)
248  return;
249 
250  n_fail = n_ops - vnet_crypto_process_chained_ops (vm, op, chunks, n_ops);
251 
252  while (n_fail)
253  {
254  ASSERT (op - ops < n_ops);
255 
256  if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED)
257  {
258  u32 bi = op->user_data;
259  b[bi]->error = node->errors[ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR];
260  nexts[bi] = drop_next;
261  n_fail--;
262  }
263  op++;
264  }
265 }
266 
269  vnet_crypto_op_t * ops, vlib_buffer_t * b[], u16 * nexts,
270  u16 drop_next)
271 {
272  u32 n_fail, n_ops = vec_len (ops);
273  vnet_crypto_op_t *op = ops;
274 
275  if (n_ops == 0)
276  return;
277 
278  n_fail = n_ops - vnet_crypto_process_ops (vm, op, n_ops);
279 
280  while (n_fail)
281  {
282  ASSERT (op - ops < n_ops);
283 
284  if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED)
285  {
286  u32 bi = op->user_data;
287  b[bi]->error = node->errors[ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR];
288  nexts[bi] = drop_next;
289  n_fail--;
290  }
291  op++;
292  }
293 }
294 
297  ipsec_sa_t * sa0, vlib_buffer_t * b,
298  vlib_buffer_t * lb, u8 icv_sz, u8 * start,
299  u32 start_len, u16 * n_ch)
300 {
302  vlib_buffer_t *cb = b;
303  u32 n_chunks = 1;
304  u32 total_len;
305  vec_add2 (ptd->chunks, ch, 1);
306  total_len = ch->len = start_len;
307  ch->src = ch->dst = start;
308  cb = vlib_get_buffer (vm, cb->next_buffer);
309 
310  while (1)
311  {
312  vec_add2 (ptd->chunks, ch, 1);
313  n_chunks += 1;
314  if (lb == cb)
315  total_len += ch->len = cb->current_length - icv_sz;
316  else
317  total_len += ch->len = cb->current_length;
318  ch->src = ch->dst = vlib_buffer_get_current (cb);
319 
320  if (!(cb->flags & VLIB_BUFFER_NEXT_PRESENT))
321  break;
322 
323  cb = vlib_get_buffer (vm, cb->next_buffer);
324  }
325 
326  if (n_ch)
327  *n_ch = n_chunks;
328 
329  return total_len;
330 }
331 
334  ipsec_sa_t * sa0, vlib_buffer_t * b,
335  vlib_buffer_t * lb, u8 icv_sz, u8 * start,
336  u32 start_len, u8 * digest, u16 * n_ch)
337 {
339  vlib_buffer_t *cb = b;
340  u32 n_chunks = 1;
341  u32 total_len;
342  vec_add2 (ptd->chunks, ch, 1);
343  total_len = ch->len = start_len;
344  ch->src = start;
345  cb = vlib_get_buffer (vm, cb->next_buffer);
346 
347  while (1)
348  {
349  vec_add2 (ptd->chunks, ch, 1);
350  n_chunks += 1;
351  if (lb == cb)
352  {
353  total_len += ch->len = cb->current_length - icv_sz;
354  if (ipsec_sa_is_set_USE_ESN (sa0))
355  {
356  u32 seq_hi = clib_net_to_host_u32 (sa0->seq_hi);
357  clib_memcpy_fast (digest, &seq_hi, sizeof (seq_hi));
358  ch->len += sizeof (seq_hi);
359  total_len += sizeof (seq_hi);
360  }
361  }
362  else
363  total_len += ch->len = cb->current_length;
364  ch->src = vlib_buffer_get_current (cb);
365 
366  if (!(cb->flags & VLIB_BUFFER_NEXT_PRESENT))
367  break;
368 
369  cb = vlib_get_buffer (vm, cb->next_buffer);
370  }
371 
372  if (n_ch)
373  *n_ch = n_chunks;
374 
375  return total_len;
376 }
377 
378 always_inline void
380  vnet_crypto_op_t **crypto_ops,
381  vnet_crypto_op_t **integ_ops, ipsec_sa_t *sa0,
382  u8 *payload, u16 payload_len, u8 iv_sz, u8 icv_sz, u32 bi,
383  vlib_buffer_t **b, vlib_buffer_t *lb, u32 hdr_len,
384  esp_header_t *esp)
385 {
386  if (sa0->crypto_enc_op_id)
387  {
388  vnet_crypto_op_t *op;
389  vec_add2_aligned (crypto_ops[0], op, 1, CLIB_CACHE_LINE_BYTES);
391 
392  op->src = op->dst = payload;
393  op->key_index = sa0->crypto_key_index;
394  op->len = payload_len - icv_sz;
395  op->user_data = bi;
396 
397  if (ipsec_sa_is_set_IS_CTR (sa0))
398  {
399  ASSERT (sizeof (u64) == iv_sz);
400  /* construct nonce in a scratch space in front of the IP header */
401  esp_ctr_nonce_t *nonce =
402  (esp_ctr_nonce_t *) (payload - sizeof (u64) - hdr_len -
403  sizeof (*nonce));
404  u64 *pkt_iv = (u64 *) (payload - sizeof (u64));
405 
406  if (ipsec_sa_is_set_IS_AEAD (sa0))
407  {
408  /* constuct aad in a scratch space in front of the nonce */
409  op->aad = (u8 *) nonce - sizeof (esp_aead_t);
410  op->aad_len = esp_aad_fill (op->aad, esp, sa0);
411  op->tag = payload + op->len;
412  op->tag_len = 16;
413  }
414  else
415  {
416  nonce->ctr = clib_host_to_net_u32 (1);
417  }
418 
419  nonce->salt = sa0->salt;
420  nonce->iv = *pkt_iv = clib_host_to_net_u64 (sa0->ctr_iv_counter++);
421  op->iv = (u8 *) nonce;
422  }
423  else
424  {
425  op->iv = payload - iv_sz;
427  }
428 
429  if (lb != b[0])
430  {
431  /* is chained */
433  op->chunk_index = vec_len (ptd->chunks);
434  op->tag = vlib_buffer_get_tail (lb) - icv_sz;
435  esp_encrypt_chain_crypto (vm, ptd, sa0, b[0], lb, icv_sz, payload,
436  payload_len, &op->n_chunks);
437  }
438  }
439 
440  if (sa0->integ_op_id)
441  {
442  vnet_crypto_op_t *op;
443  vec_add2_aligned (integ_ops[0], op, 1, CLIB_CACHE_LINE_BYTES);
444  vnet_crypto_op_init (op, sa0->integ_op_id);
445  op->src = payload - iv_sz - sizeof (esp_header_t);
446  op->digest = payload + payload_len - icv_sz;
447  op->key_index = sa0->integ_key_index;
448  op->digest_len = icv_sz;
449  op->len = payload_len - icv_sz + iv_sz + sizeof (esp_header_t);
450  op->user_data = bi;
451 
452  if (lb != b[0])
453  {
454  /* is chained */
456  op->chunk_index = vec_len (ptd->chunks);
457  op->digest = vlib_buffer_get_tail (lb) - icv_sz;
458 
459  esp_encrypt_chain_integ (vm, ptd, sa0, b[0], lb, icv_sz,
460  payload - iv_sz - sizeof (esp_header_t),
461  payload_len + iv_sz +
462  sizeof (esp_header_t), op->digest,
463  &op->n_chunks);
464  }
465  else if (ipsec_sa_is_set_USE_ESN (sa0))
466  {
467  u32 seq_hi = clib_net_to_host_u32 (sa0->seq_hi);
468  clib_memcpy_fast (op->digest, &seq_hi, sizeof (seq_hi));
469  op->len += sizeof (seq_hi);
470  }
471  }
472 }
473 
476  vnet_crypto_async_frame_t *async_frame,
478  u8 *payload, u32 payload_len, u8 iv_sz, u8 icv_sz,
479  u32 bi, u16 next, u32 hdr_len, u16 async_next,
480  vlib_buffer_t *lb)
481 {
482  esp_post_data_t *post = esp_post_data (b);
483  u8 *tag, *iv, *aad = 0;
484  u8 flag = 0;
485  u32 key_index;
486  i16 crypto_start_offset, integ_start_offset = 0;
487  u16 crypto_total_len, integ_total_len;
488 
489  post->next_index = next;
490 
491  /* crypto */
492  crypto_start_offset = payload - b->data;
493  crypto_total_len = integ_total_len = payload_len - icv_sz;
494  tag = payload + crypto_total_len;
495 
496  key_index = sa->linked_key_index;
497 
498  if (ipsec_sa_is_set_IS_CTR (sa))
499  {
500  ASSERT (sizeof (u64) == iv_sz);
501  /* construct nonce in a scratch space in front of the IP header */
502  esp_ctr_nonce_t *nonce = (esp_ctr_nonce_t *) (payload - sizeof (u64) -
503  hdr_len - sizeof (*nonce));
504  u64 *pkt_iv = (u64 *) (payload - sizeof (u64));
505 
506  if (ipsec_sa_is_set_IS_AEAD (sa))
507  {
508  /* constuct aad in a scratch space in front of the nonce */
509  aad = (u8 *) nonce - sizeof (esp_aead_t);
510  esp_aad_fill (aad, esp, sa);
511  key_index = sa->crypto_key_index;
512  }
513  else
514  {
515  nonce->ctr = clib_host_to_net_u32 (1);
516  }
517 
518  nonce->salt = sa->salt;
519  nonce->iv = *pkt_iv = clib_host_to_net_u64 (sa->ctr_iv_counter++);
520  iv = (u8 *) nonce;
521  }
522  else
523  {
524  iv = payload - iv_sz;
526  }
527 
528  if (lb != b)
529  {
530  /* chain */
532  tag = vlib_buffer_get_tail (lb) - icv_sz;
533  crypto_total_len = esp_encrypt_chain_crypto (vm, ptd, sa, b, lb, icv_sz,
534  payload, payload_len, 0);
535  }
536 
537  if (sa->integ_op_id)
538  {
539  integ_start_offset = crypto_start_offset - iv_sz - sizeof (esp_header_t);
540  integ_total_len += iv_sz + sizeof (esp_header_t);
541 
542  if (b != lb)
543  {
544  integ_total_len = esp_encrypt_chain_integ (
545  vm, ptd, sa, b, lb, icv_sz,
546  payload - iv_sz - sizeof (esp_header_t),
547  payload_len + iv_sz + sizeof (esp_header_t), tag, 0);
548  }
549  else if (ipsec_sa_is_set_USE_ESN (sa))
550  {
551  u32 seq_hi = clib_net_to_host_u32 (sa->seq_hi);
552  clib_memcpy_fast (tag, &seq_hi, sizeof (seq_hi));
553  integ_total_len += sizeof (seq_hi);
554  }
555  }
556 
557  /* this always succeeds because we know the frame is not full */
558  vnet_crypto_async_add_to_frame (vm, async_frame, key_index, crypto_total_len,
559  integ_total_len - crypto_total_len,
560  crypto_start_offset, integ_start_offset, bi,
561  async_next, iv, tag, aad, flag);
562 }
563 
566  vlib_frame_t *frame, vnet_link_t lt, int is_tun,
567  u16 async_next_node)
568 {
572  u32 n_left = frame->n_vectors;
575  u16 buffer_data_size = vlib_buffer_get_default_data_size (vm);
576  u32 current_sa_index = ~0, current_sa_packets = 0;
577  u32 current_sa_bytes = 0, spi = 0;
578  u8 esp_align = 4, iv_sz = 0, icv_sz = 0;
579  ipsec_sa_t *sa0 = 0;
580  vlib_buffer_t *lb;
581  vnet_crypto_op_t **crypto_ops = &ptd->crypto_ops;
582  vnet_crypto_op_t **integ_ops = &ptd->integ_ops;
584  int is_async = im->async_mode;
585  vnet_crypto_async_op_id_t async_op = ~0;
586  u16 drop_next =
587  (lt == VNET_LINK_IP6 ? ESP_ENCRYPT_NEXT_DROP6 :
588  (lt == VNET_LINK_IP4 ? ESP_ENCRYPT_NEXT_DROP4 :
589  ESP_ENCRYPT_NEXT_DROP_MPLS));
590  u16 handoff_next = (lt == VNET_LINK_IP6 ?
591  ESP_ENCRYPT_NEXT_HANDOFF6 :
592  (lt == VNET_LINK_IP4 ? ESP_ENCRYPT_NEXT_HANDOFF4 :
593  ESP_ENCRYPT_NEXT_HANDOFF_MPLS));
594  vlib_buffer_t *sync_bufs[VLIB_FRAME_SIZE];
595  u16 sync_nexts[VLIB_FRAME_SIZE], *sync_next = sync_nexts, n_sync = 0;
596  u16 async_nexts[VLIB_FRAME_SIZE], *async_next = async_nexts, n_async = 0;
597  u16 noop_nexts[VLIB_FRAME_SIZE], *noop_next = noop_nexts, n_noop = 0;
598  u32 sync_bi[VLIB_FRAME_SIZE];
599  u32 noop_bi[VLIB_FRAME_SIZE];
601 
603 
609  vec_reset_length (ptd->chunks);
610  clib_memset (async_frames, 0, sizeof (async_frames));
611 
612  while (n_left > 0)
613  {
614  u32 sa_index0;
615  dpo_id_t *dpo;
616  esp_header_t *esp;
617  u8 *payload, *next_hdr_ptr;
618  u16 payload_len, payload_len_total, n_bufs;
619  u32 hdr_len;
620 
621  err = ESP_ENCRYPT_ERROR_RX_PKTS;
622 
623  if (n_left > 2)
624  {
625  u8 *p;
626  vlib_prefetch_buffer_header (b[2], LOAD);
627  p = vlib_buffer_get_current (b[1]);
631  /* speculate that the trailer goes in the first buffer */
633  CLIB_CACHE_LINE_BYTES, LOAD);
634  }
635 
636  if (is_tun)
637  {
638  /* we are on a ipsec tunnel's feature arc */
639  vnet_buffer (b[0])->ipsec.sad_index =
640  sa_index0 = ipsec_tun_protect_get_sa_out
641  (vnet_buffer (b[0])->ip.adj_index[VLIB_TX]);
642  }
643  else
644  sa_index0 = vnet_buffer (b[0])->ipsec.sad_index;
645 
646  if (sa_index0 != current_sa_index)
647  {
648  if (current_sa_packets)
650  current_sa_index,
651  current_sa_packets,
652  current_sa_bytes);
653  current_sa_packets = current_sa_bytes = 0;
654 
655  sa0 = ipsec_sa_get (sa_index0);
656 
657  /* fetch the second cacheline ASAP */
658  CLIB_PREFETCH (sa0->cacheline1, CLIB_CACHE_LINE_BYTES, LOAD);
659 
660  current_sa_index = sa_index0;
661  spi = clib_net_to_host_u32 (sa0->spi);
662  esp_align = sa0->esp_block_align;
663  icv_sz = sa0->integ_icv_size;
664  iv_sz = sa0->crypto_iv_size;
665  is_async = im->async_mode | ipsec_sa_is_set_IS_ASYNC (sa0);
666  }
667 
668  if (is_async)
669  {
670  async_op = sa0->crypto_async_enc_op_id;
671 
672  /* get a frame for this op if we don't yet have one or it's full
673  */
674  if (NULL == async_frames[async_op] ||
675  vnet_crypto_async_frame_is_full (async_frames[async_op]))
676  {
677  async_frames[async_op] =
678  vnet_crypto_async_get_frame (vm, async_op);
679  /* Save the frame to the list we'll submit at the end */
680  vec_add1 (ptd->async_frames, async_frames[async_op]);
681  }
682  }
683 
684  if (PREDICT_FALSE (~0 == sa0->thread_index))
685  {
686  /* this is the first packet to use this SA, claim the SA
687  * for this thread. this could happen simultaneously on
688  * another thread */
691  }
692 
694  {
695  vnet_buffer (b[0])->ipsec.thread_index = sa0->thread_index;
696  err = ESP_ENCRYPT_ERROR_HANDOFF;
697  esp_set_next_index (b[0], node, err, n_noop, noop_nexts,
698  handoff_next);
699  goto trace;
700  }
701 
702  lb = b[0];
703  n_bufs = vlib_buffer_chain_linearize (vm, b[0]);
704  if (n_bufs == 0)
705  {
706  err = ESP_ENCRYPT_ERROR_NO_BUFFERS;
707  esp_set_next_index (b[0], node, err, n_noop, noop_nexts, drop_next);
708  goto trace;
709  }
710 
711  if (n_bufs > 1)
712  {
713  /* find last buffer in the chain */
714  while (lb->flags & VLIB_BUFFER_NEXT_PRESENT)
715  lb = vlib_get_buffer (vm, lb->next_buffer);
716  }
717 
718  if (PREDICT_FALSE (esp_seq_advance (sa0)))
719  {
720  err = ESP_ENCRYPT_ERROR_SEQ_CYCLED;
721  esp_set_next_index (b[0], node, err, n_noop, noop_nexts, drop_next);
722  goto trace;
723  }
724 
725  /* space for IV */
726  hdr_len = iv_sz;
727 
728  if (ipsec_sa_is_set_IS_TUNNEL (sa0))
729  {
730  payload = vlib_buffer_get_current (b[0]);
731  next_hdr_ptr = esp_add_footer_and_icv (
732  vm, &lb, esp_align, icv_sz, node, buffer_data_size,
734  if (!next_hdr_ptr)
735  {
736  err = ESP_ENCRYPT_ERROR_NO_BUFFERS;
737  esp_set_next_index (b[0], node, err, n_noop, noop_nexts,
738  drop_next);
739  goto trace;
740  }
741  b[0]->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
742  payload_len = b[0]->current_length;
743  payload_len_total = vlib_buffer_length_in_chain (vm, b[0]);
744 
745  /* ESP header */
746  hdr_len += sizeof (*esp);
747  esp = (esp_header_t *) (payload - hdr_len);
748 
749  /* optional UDP header */
750  if (ipsec_sa_is_set_UDP_ENCAP (sa0))
751  {
752  hdr_len += sizeof (udp_header_t);
753  esp_fill_udp_hdr (sa0, (udp_header_t *) (payload - hdr_len),
754  payload_len_total + hdr_len);
755  }
756 
757  /* IP header */
758  if (ipsec_sa_is_set_IS_TUNNEL_V6 (sa0))
759  {
760  ip6_header_t *ip6;
761  u16 len = sizeof (ip6_header_t);
762  hdr_len += len;
763  ip6 = (ip6_header_t *) (payload - hdr_len);
764  clib_memcpy_fast (ip6, &sa0->ip6_hdr, sizeof (ip6_header_t));
765 
766  if (VNET_LINK_IP6 == lt)
767  {
768  *next_hdr_ptr = IP_PROTOCOL_IPV6;
770  (const ip6_header_t *) payload,
771  ip6);
772  }
773  else if (VNET_LINK_IP4 == lt)
774  {
775  *next_hdr_ptr = IP_PROTOCOL_IP_IN_IP;
777  (const ip4_header_t *) payload, ip6);
778  }
779  else if (VNET_LINK_MPLS == lt)
780  {
781  *next_hdr_ptr = IP_PROTOCOL_MPLS_IN_IP;
783  sa0->tunnel_flags, b[0],
784  (const mpls_unicast_header_t *) payload, ip6);
785  }
786  else
787  ASSERT (0);
788 
789  len = payload_len_total + hdr_len - len;
790  ip6->payload_length = clib_net_to_host_u16 (len);
791  b[0]->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
792  }
793  else
794  {
795  ip4_header_t *ip4;
796  u16 len = sizeof (ip4_header_t);
797  hdr_len += len;
798  ip4 = (ip4_header_t *) (payload - hdr_len);
799  clib_memcpy_fast (ip4, &sa0->ip4_hdr, sizeof (ip4_header_t));
800 
801  if (VNET_LINK_IP6 == lt)
802  {
803  *next_hdr_ptr = IP_PROTOCOL_IPV6;
805  (const ip6_header_t *)
806  payload, ip4);
807  }
808  else if (VNET_LINK_IP4 == lt)
809  {
810  *next_hdr_ptr = IP_PROTOCOL_IP_IN_IP;
812  (const ip4_header_t *)
813  payload, ip4);
814  }
815  else if (VNET_LINK_MPLS == lt)
816  {
817  *next_hdr_ptr = IP_PROTOCOL_MPLS_IN_IP;
819  sa0->tunnel_flags, (const mpls_unicast_header_t *) payload,
820  ip4);
821  }
822  else
823  ASSERT (0);
824 
825  len = payload_len_total + hdr_len;
826  esp_update_ip4_hdr (ip4, len, /* is_transport */ 0, 0);
827  }
828 
829  dpo = &sa0->dpo;
830  if (!is_tun)
831  {
832  sync_next[0] = dpo->dpoi_next_node;
833  vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo->dpoi_index;
834  }
835  else
836  sync_next[0] = ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT;
837  b[0]->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
838  }
839  else /* transport mode */
840  {
841  u8 *l2_hdr, l2_len, *ip_hdr, ip_len;
842  ip6_ext_header_t *ext_hdr;
843  udp_header_t *udp = 0;
844  u16 udp_len = 0;
845  u8 *old_ip_hdr = vlib_buffer_get_current (b[0]);
846 
847  ip_len =
848  (VNET_LINK_IP6 == lt ?
849  esp_get_ip6_hdr_len ((ip6_header_t *) old_ip_hdr, &ext_hdr) :
850  ip4_header_bytes ((ip4_header_t *) old_ip_hdr));
851 
852  vlib_buffer_advance (b[0], ip_len);
853  payload = vlib_buffer_get_current (b[0]);
854  next_hdr_ptr = esp_add_footer_and_icv (
855  vm, &lb, esp_align, icv_sz, node, buffer_data_size,
857  if (!next_hdr_ptr)
858  {
859  err = ESP_ENCRYPT_ERROR_NO_BUFFERS;
860  esp_set_next_index (b[0], node, err, n_noop, noop_nexts,
861  drop_next);
862  goto trace;
863  }
864 
865  b[0]->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
866  payload_len = b[0]->current_length;
867  payload_len_total = vlib_buffer_length_in_chain (vm, b[0]);
868 
869  /* ESP header */
870  hdr_len += sizeof (*esp);
871  esp = (esp_header_t *) (payload - hdr_len);
872 
873  /* optional UDP header */
874  if (ipsec_sa_is_set_UDP_ENCAP (sa0))
875  {
876  hdr_len += sizeof (udp_header_t);
877  udp = (udp_header_t *) (payload - hdr_len);
878  }
879 
880  /* IP header */
881  hdr_len += ip_len;
882  ip_hdr = payload - hdr_len;
883 
884  /* L2 header */
885  if (!is_tun)
886  {
887  l2_len = vnet_buffer (b[0])->ip.save_rewrite_length;
888  hdr_len += l2_len;
889  l2_hdr = payload - hdr_len;
890 
891  /* copy l2 and ip header */
892  clib_memcpy_le32 (l2_hdr, old_ip_hdr - l2_len, l2_len);
893  }
894  else
895  l2_len = 0;
896 
897  if (VNET_LINK_IP6 == lt)
898  {
899  ip6_header_t *ip6 = (ip6_header_t *) (old_ip_hdr);
900  if (PREDICT_TRUE (NULL == ext_hdr))
901  {
902  *next_hdr_ptr = ip6->protocol;
903  ip6->protocol = IP_PROTOCOL_IPSEC_ESP;
904  }
905  else
906  {
907  *next_hdr_ptr = ext_hdr->next_hdr;
908  ext_hdr->next_hdr = IP_PROTOCOL_IPSEC_ESP;
909  }
910  ip6->payload_length =
911  clib_host_to_net_u16 (payload_len_total + hdr_len - l2_len -
912  sizeof (ip6_header_t));
913  }
914  else if (VNET_LINK_IP4 == lt)
915  {
916  u16 len;
917  ip4_header_t *ip4 = (ip4_header_t *) (old_ip_hdr);
918  *next_hdr_ptr = ip4->protocol;
919  len = payload_len_total + hdr_len - l2_len;
920  if (udp)
921  {
922  esp_update_ip4_hdr (ip4, len, /* is_transport */ 1, 1);
923  udp_len = len - ip_len;
924  }
925  else
926  esp_update_ip4_hdr (ip4, len, /* is_transport */ 1, 0);
927  }
928 
929  clib_memcpy_le64 (ip_hdr, old_ip_hdr, ip_len);
930 
931  if (udp)
932  {
933  esp_fill_udp_hdr (sa0, udp, udp_len);
934  }
935 
936  sync_next[0] = ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT;
937  }
938 
939  if (lb != b[0])
940  {
941  crypto_ops = &ptd->chained_crypto_ops;
942  integ_ops = &ptd->chained_integ_ops;
943  }
944  else
945  {
946  crypto_ops = &ptd->crypto_ops;
947  integ_ops = &ptd->integ_ops;
948  }
949 
950  esp->spi = spi;
951  esp->seq = clib_net_to_host_u32 (sa0->seq);
952 
953  if (is_async)
954  esp_prepare_async_frame (vm, ptd, async_frames[async_op], sa0, b[0],
955  esp, payload, payload_len, iv_sz, icv_sz,
956  from[b - bufs], sync_next[0], hdr_len,
957  async_next_node, lb);
958  else
959  esp_prepare_sync_op (vm, ptd, crypto_ops, integ_ops, sa0, payload,
960  payload_len, iv_sz, icv_sz, n_sync, b, lb,
961  hdr_len, esp);
962 
963  vlib_buffer_advance (b[0], 0LL - hdr_len);
964 
965  current_sa_packets += 1;
966  current_sa_bytes += payload_len_total;
967 
968  trace:
969  if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED))
970  {
972  sizeof (*tr));
973  tr->sa_index = sa_index0;
974  tr->spi = sa0->spi;
975  tr->seq = sa0->seq;
976  tr->sa_seq_hi = sa0->seq_hi;
977  tr->udp_encap = ipsec_sa_is_set_UDP_ENCAP (sa0);
978  tr->crypto_alg = sa0->crypto_alg;
979  tr->integ_alg = sa0->integ_alg;
980  }
981 
982  /* next */
983  if (ESP_ENCRYPT_ERROR_RX_PKTS != err)
984  {
985  noop_bi[n_noop] = from[b - bufs];
986  n_noop++;
987  noop_next++;
988  }
989  else if (!is_async)
990  {
991  sync_bi[n_sync] = from[b - bufs];
992  sync_bufs[n_sync] = b[0];
993  n_sync++;
994  sync_next++;
995  }
996  else
997  {
998  n_async++;
999  async_next++;
1000  }
1001  n_left -= 1;
1002  b += 1;
1003  }
1004 
1006  current_sa_index, current_sa_packets,
1007  current_sa_bytes);
1008  if (n_sync)
1009  {
1010  esp_process_ops (vm, node, ptd->crypto_ops, sync_bufs, sync_nexts,
1011  drop_next);
1013  sync_nexts, ptd->chunks, drop_next);
1014 
1015  esp_process_ops (vm, node, ptd->integ_ops, sync_bufs, sync_nexts,
1016  drop_next);
1017  esp_process_chained_ops (vm, node, ptd->chained_integ_ops, sync_bufs,
1018  sync_nexts, ptd->chunks, drop_next);
1019 
1020  vlib_buffer_enqueue_to_next (vm, node, sync_bi, sync_nexts, n_sync);
1021  }
1022  if (n_async)
1023  {
1024  /* submit all of the open frames */
1025  vnet_crypto_async_frame_t **async_frame;
1026 
1027  vec_foreach (async_frame, ptd->async_frames)
1028  {
1029  if (vnet_crypto_async_submit_open_frame (vm, *async_frame) < 0)
1030  {
1032  vm, *async_frame, node, ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR,
1033  n_sync, noop_bi, noop_nexts, drop_next);
1034  vnet_crypto_async_reset_frame (*async_frame);
1035  vnet_crypto_async_free_frame (vm, *async_frame);
1036  }
1037  }
1038  }
1039  if (n_noop)
1040  vlib_buffer_enqueue_to_next (vm, node, noop_bi, noop_nexts, n_noop);
1041 
1042  vlib_node_increment_counter (vm, node->node_index, ESP_ENCRYPT_ERROR_RX_PKTS,
1043  frame->n_vectors);
1044 
1045  return frame->n_vectors;
1046 }
1047 
1050  vlib_frame_t * frame)
1051 {
1055  u32 n_left = frame->n_vectors;
1056 
1058 
1059  if (n_left >= 4)
1060  {
1061  vlib_prefetch_buffer_header (b[0], LOAD);
1062  vlib_prefetch_buffer_header (b[1], LOAD);
1063  vlib_prefetch_buffer_header (b[2], LOAD);
1064  vlib_prefetch_buffer_header (b[3], LOAD);
1065  }
1066 
1067  while (n_left > 8)
1068  {
1069  vlib_prefetch_buffer_header (b[4], LOAD);
1070  vlib_prefetch_buffer_header (b[5], LOAD);
1071  vlib_prefetch_buffer_header (b[6], LOAD);
1072  vlib_prefetch_buffer_header (b[7], LOAD);
1073 
1074  next[0] = (esp_post_data (b[0]))->next_index;
1075  next[1] = (esp_post_data (b[1]))->next_index;
1076  next[2] = (esp_post_data (b[2]))->next_index;
1077  next[3] = (esp_post_data (b[3]))->next_index;
1078 
1079  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
1080  {
1081  if (b[0]->flags & VLIB_BUFFER_IS_TRACED)
1082  {
1084  sizeof (*tr));
1085  tr->next_index = next[0];
1086  }
1087  if (b[1]->flags & VLIB_BUFFER_IS_TRACED)
1088  {
1090  sizeof (*tr));
1091  tr->next_index = next[1];
1092  }
1093  if (b[2]->flags & VLIB_BUFFER_IS_TRACED)
1094  {
1096  sizeof (*tr));
1097  tr->next_index = next[2];
1098  }
1099  if (b[3]->flags & VLIB_BUFFER_IS_TRACED)
1100  {
1102  sizeof (*tr));
1103  tr->next_index = next[3];
1104  }
1105  }
1106 
1107  b += 4;
1108  next += 4;
1109  n_left -= 4;
1110  }
1111 
1112  while (n_left > 0)
1113  {
1114  next[0] = (esp_post_data (b[0]))->next_index;
1115  if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED))
1116  {
1118  sizeof (*tr));
1119  tr->next_index = next[0];
1120  }
1121 
1122  b += 1;
1123  next += 1;
1124  n_left -= 1;
1125  }
1126 
1127  vlib_node_increment_counter (vm, node->node_index,
1128  ESP_ENCRYPT_ERROR_POST_RX_PKTS,
1129  frame->n_vectors);
1131  return frame->n_vectors;
1132 }
1133 
1137 {
1140 }
1141 
1142 /* *INDENT-OFF* */
1144  .name = "esp4-encrypt",
1145  .vector_size = sizeof (u32),
1146  .format_trace = format_esp_encrypt_trace,
1148 
1149  .n_errors = ARRAY_LEN (esp_encrypt_error_strings),
1150  .error_strings = esp_encrypt_error_strings,
1151 
1152  .n_next_nodes = ESP_ENCRYPT_N_NEXT,
1153  .next_nodes = { [ESP_ENCRYPT_NEXT_DROP4] = "ip4-drop",
1154  [ESP_ENCRYPT_NEXT_DROP6] = "ip6-drop",
1155  [ESP_ENCRYPT_NEXT_DROP_MPLS] = "mpls-drop",
1156  [ESP_ENCRYPT_NEXT_HANDOFF4] = "esp4-encrypt-handoff",
1157  [ESP_ENCRYPT_NEXT_HANDOFF6] = "esp6-encrypt-handoff",
1158  [ESP_ENCRYPT_NEXT_HANDOFF_MPLS] = "error-drop",
1159  [ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT] = "interface-output" },
1160 };
1161 /* *INDENT-ON* */
1162 
1166 {
1168 }
1169 
1170 /* *INDENT-OFF* */
1172  .name = "esp4-encrypt-post",
1173  .vector_size = sizeof (u32),
1174  .format_trace = format_esp_post_encrypt_trace,
1176  .sibling_of = "esp4-encrypt",
1177 
1178  .n_errors = ARRAY_LEN(esp_encrypt_error_strings),
1179  .error_strings = esp_encrypt_error_strings,
1180 };
1181 /* *INDENT-ON* */
1182 
1186 {
1189 }
1190 
1191 /* *INDENT-OFF* */
1193  .name = "esp6-encrypt",
1194  .vector_size = sizeof (u32),
1195  .format_trace = format_esp_encrypt_trace,
1197  .sibling_of = "esp4-encrypt",
1198 
1199  .n_errors = ARRAY_LEN(esp_encrypt_error_strings),
1200  .error_strings = esp_encrypt_error_strings,
1201 };
1202 /* *INDENT-ON* */
1203 
1207 {
1209 }
1210 
1211 /* *INDENT-OFF* */
1213  .name = "esp6-encrypt-post",
1214  .vector_size = sizeof (u32),
1215  .format_trace = format_esp_post_encrypt_trace,
1217  .sibling_of = "esp4-encrypt",
1218 
1219  .n_errors = ARRAY_LEN(esp_encrypt_error_strings),
1220  .error_strings = esp_encrypt_error_strings,
1221 };
1222 /* *INDENT-ON* */
1223 
1227 {
1230 }
1231 
1232 /* *INDENT-OFF* */
1234  .name = "esp4-encrypt-tun",
1235  .vector_size = sizeof (u32),
1236  .format_trace = format_esp_encrypt_trace,
1238 
1239  .n_errors = ARRAY_LEN(esp_encrypt_error_strings),
1240  .error_strings = esp_encrypt_error_strings,
1241 
1242  .n_next_nodes = ESP_ENCRYPT_N_NEXT,
1243  .next_nodes = {
1244  [ESP_ENCRYPT_NEXT_DROP4] = "ip4-drop",
1245  [ESP_ENCRYPT_NEXT_DROP6] = "ip6-drop",
1246  [ESP_ENCRYPT_NEXT_DROP_MPLS] = "mpls-drop",
1247  [ESP_ENCRYPT_NEXT_HANDOFF4] = "esp4-encrypt-tun-handoff",
1248  [ESP_ENCRYPT_NEXT_HANDOFF6] = "esp6-encrypt-tun-handoff",
1249  [ESP_ENCRYPT_NEXT_HANDOFF_MPLS] = "esp-mpls-encrypt-tun-handoff",
1250  [ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT] = "adj-midchain-tx",
1251  },
1252 };
1253 
1257 {
1259 }
1260 
1261 /* *INDENT-OFF* */
1263  .name = "esp4-encrypt-tun-post",
1264  .vector_size = sizeof (u32),
1265  .format_trace = format_esp_post_encrypt_trace,
1267  .sibling_of = "esp4-encrypt-tun",
1268 
1269  .n_errors = ARRAY_LEN(esp_encrypt_error_strings),
1270  .error_strings = esp_encrypt_error_strings,
1271 };
1272 /* *INDENT-ON* */
1273 
1277 {
1280 }
1281 
1282 /* *INDENT-OFF* */
1284  .name = "esp6-encrypt-tun",
1285  .vector_size = sizeof (u32),
1286  .format_trace = format_esp_encrypt_trace,
1288 
1289  .n_errors = ARRAY_LEN(esp_encrypt_error_strings),
1290  .error_strings = esp_encrypt_error_strings,
1291 
1292  .n_next_nodes = ESP_ENCRYPT_N_NEXT,
1293  .next_nodes = {
1294  [ESP_ENCRYPT_NEXT_DROP4] = "ip4-drop",
1295  [ESP_ENCRYPT_NEXT_DROP6] = "ip6-drop",
1296  [ESP_ENCRYPT_NEXT_DROP_MPLS] = "mpls-drop",
1297  [ESP_ENCRYPT_NEXT_HANDOFF4] = "esp4-encrypt-tun-handoff",
1298  [ESP_ENCRYPT_NEXT_HANDOFF6] = "esp6-encrypt-tun-handoff",
1299  [ESP_ENCRYPT_NEXT_HANDOFF_MPLS] = "esp-mpls-encrypt-tun-handoff",
1300  [ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT] = "adj-midchain-tx",
1301  },
1302 };
1303 
1304 /* *INDENT-ON* */
1305 
1309 {
1311 }
1312 
1313 /* *INDENT-OFF* */
1315  .name = "esp6-encrypt-tun-post",
1316  .vector_size = sizeof (u32),
1317  .format_trace = format_esp_post_encrypt_trace,
1319  .sibling_of = "esp-mpls-encrypt-tun",
1320 
1321  .n_errors = ARRAY_LEN (esp_encrypt_error_strings),
1322  .error_strings = esp_encrypt_error_strings,
1323 };
1324 /* *INDENT-ON* */
1325 
1331 }
1332 
1334  .name = "esp-mpls-encrypt-tun",
1335  .vector_size = sizeof (u32),
1336  .format_trace = format_esp_encrypt_trace,
1338 
1339  .n_errors = ARRAY_LEN(esp_encrypt_error_strings),
1340  .error_strings = esp_encrypt_error_strings,
1341 
1342  .n_next_nodes = ESP_ENCRYPT_N_NEXT,
1343  .next_nodes = {
1344  [ESP_ENCRYPT_NEXT_DROP4] = "ip4-drop",
1345  [ESP_ENCRYPT_NEXT_DROP6] = "ip6-drop",
1346  [ESP_ENCRYPT_NEXT_DROP_MPLS] = "mpls-drop",
1347  [ESP_ENCRYPT_NEXT_HANDOFF4] = "esp4-encrypt-tun-handoff",
1348  [ESP_ENCRYPT_NEXT_HANDOFF6] = "esp6-encrypt-tun-handoff",
1349  [ESP_ENCRYPT_NEXT_HANDOFF_MPLS] = "esp-mpls-encrypt-tun-handoff",
1350  [ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT] = "adj-midchain-tx",
1351  },
1352 };
1353 
1356 {
1358 }
1359 
1361  .name = "esp-mpls-encrypt-tun-post",
1362  .vector_size = sizeof (u32),
1363  .format_trace = format_esp_post_encrypt_trace,
1365  .sibling_of = "esp-mpls-encrypt-tun",
1366 
1367  .n_errors = ARRAY_LEN (esp_encrypt_error_strings),
1368  .error_strings = esp_encrypt_error_strings,
1369 };
1370 
1371 typedef struct
1372 {
1375 
1376 static u8 *
1377 format_esp_no_crypto_trace (u8 * s, va_list * args)
1378 {
1379  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1380  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1381  esp_no_crypto_trace_t *t = va_arg (*args, esp_no_crypto_trace_t *);
1382 
1383  s = format (s, "esp-no-crypto: sa-index %u", t->sa_index);
1384 
1385  return s;
1386 }
1387 
1388 enum
1389 {
1392 };
1393 
1394 enum
1395 {
1397 };
1398 
1400  "Outbound ESP packets received",
1401 };
1402 
1405  vlib_frame_t * frame)
1406 {
1409  u32 n_left = frame->n_vectors;
1410 
1412 
1413  while (n_left > 0)
1414  {
1415  u32 sa_index0;
1416 
1417  /* packets are always going to be dropped, but get the sa_index */
1418  sa_index0 = ipsec_tun_protect_get_sa_out
1419  (vnet_buffer (b[0])->ip.adj_index[VLIB_TX]);
1420 
1421  if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED))
1422  {
1424  sizeof (*tr));
1425  tr->sa_index = sa_index0;
1426  }
1427 
1428  n_left -= 1;
1429  b += 1;
1430  }
1431 
1432  vlib_node_increment_counter (vm, node->node_index,
1433  ESP_NO_CRYPTO_ERROR_RX_PKTS, frame->n_vectors);
1434 
1437  frame->n_vectors);
1438 
1439  return frame->n_vectors;
1440 }
1441 
1445 {
1447 }
1448 
1449 /* *INDENT-OFF* */
1451 {
1452  .name = "esp4-no-crypto",
1453  .vector_size = sizeof (u32),
1454  .format_trace = format_esp_no_crypto_trace,
1456  .error_strings = esp_no_crypto_error_strings,
1457  .n_next_nodes = ESP_NO_CRYPTO_N_NEXT,
1458  .next_nodes = {
1459  [ESP_NO_CRYPTO_NEXT_DROP] = "ip4-drop",
1460  },
1461 };
1462 
1466 {
1468 }
1469 
1470 /* *INDENT-OFF* */
1472 {
1473  .name = "esp6-no-crypto",
1474  .vector_size = sizeof (u32),
1475  .format_trace = format_esp_no_crypto_trace,
1477  .error_strings = esp_no_crypto_error_strings,
1478  .n_next_nodes = ESP_NO_CRYPTO_N_NEXT,
1479  .next_nodes = {
1480  [ESP_NO_CRYPTO_NEXT_DROP] = "ip6-drop",
1481  },
1482 };
1483 /* *INDENT-ON* */
1484 
1485 #ifndef CLIB_MARCH_VARIANT
1486 
1487 static clib_error_t *
1489 {
1491 
1492  im->esp4_enc_fq_index =
1494  im->esp6_enc_fq_index =
1496  im->esp4_enc_tun_fq_index =
1498  im->esp6_enc_tun_fq_index =
1500  im->esp_mpls_enc_tun_fq_index =
1502 
1503  return 0;
1504 }
1505 
1507 
1508 #endif
1509 
1510 /*
1511  * fd.io coding-style-patch-verification: ON
1512  *
1513  * Local Variables:
1514  * eval: (c-set-style "gnu")
1515  * End:
1516  */
esp_post_data
#define esp_post_data(b)
Definition: esp.h:227
vec_reset_length
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
Definition: vec_bootstrap.h:194
esp_encrypt_chain_crypto
static_always_inline u32 esp_encrypt_chain_crypto(vlib_main_t *vm, ipsec_per_thread_data_t *ptd, ipsec_sa_t *sa0, vlib_buffer_t *b, vlib_buffer_t *lb, u8 icv_sz, u8 *start, u32 start_len, u16 *n_ch)
Definition: esp_encrypt.c:296
ipsec.h
ipsec_per_thread_data_t::integ_ops
vnet_crypto_op_t * integ_ops
Definition: ipsec.h:101
vnet_crypto_op_t::digest
u8 * digest
Definition: crypto.h:299
tmp
u32 * tmp
Definition: interface_output.c:1078
vlib_buffer_t::next_buffer
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:149
vnet_crypto_async_free_frame
static_always_inline void vnet_crypto_async_free_frame(vlib_main_t *vm, vnet_crypto_async_frame_t *frame)
Definition: crypto.h:580
dpo_id_t_::dpoi_next_node
u16 dpoi_next_node
The next VLIB node to follow.
Definition: dpo.h:186
vnet_crypto_op_init
static_always_inline void vnet_crypto_op_init(vnet_crypto_op_t *op, vnet_crypto_op_id_t type)
Definition: crypto.h:528
im
vnet_interface_main_t * im
Definition: interface_output.c:395
esp_encrypt_trace_t::crypto_alg
ipsec_crypto_alg_t crypto_alg
Definition: esp_encrypt.c:76
ESP_NO_CRYPTO_N_NEXT
@ ESP_NO_CRYPTO_N_NEXT
Definition: esp_encrypt.c:1391
vnet_crypto_op_t::digest_len
u8 digest_len
Definition: crypto.h:268
ipsec_tun.h
udp_header_t::length
u16 length
Definition: udp_packet.h:51
trace
static vlib_cli_command_t trace
(constructor) VLIB_CLI_COMMAND (trace)
Definition: vlib_api_cli.c:899
ipsec_sa_t::integ_key_index
vnet_crypto_key_index_t integ_key_index
Definition: ipsec_sa.h:140
dpo_id_t_::dpoi_index
index_t dpoi_index
the index of objects of that type
Definition: dpo.h:190
ext_hdr_is_pre_esp
static_always_inline u8 ext_hdr_is_pre_esp(u8 nexthdr)
Definition: esp_encrypt.c:193
thread_index
u32 thread_index
Definition: nat44_ei_hairpinning.c:492
format_esp_no_crypto_trace
static u8 * format_esp_no_crypto_trace(u8 *s, va_list *args)
Definition: esp_encrypt.c:1377
bufs
vlib_buffer_t * bufs[VLIB_FRAME_SIZE]
Definition: nat44_ei_out2in.c:717
vlib_prefetch_buffer_header
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:231
frame
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
Definition: nat44_ei.c:3048
vnet_crypto_op_t::tag
u8 * tag
Definition: crypto.h:298
ipsec_sa_t::ctr_iv_counter
u64 ctr_iv_counter
Definition: ipsec_sa.h:165
crypto.h
esp4_encrypt_tun_node
vlib_node_registration_t esp4_encrypt_tun_node
(constructor) VLIB_REGISTER_NODE (esp4_encrypt_tun_node)
Definition: esp_encrypt.c:1233
format_ipsec_integ_alg
u8 * format_ipsec_integ_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:111
ip4
vl_api_ip4_address_t ip4
Definition: one.api:376
next_index
nat44_ei_hairpin_src_next_t next_index
Definition: nat44_ei_hairpinning.c:412
VNET_CRYPTO_ASYNC_OP_N_IDS
@ VNET_CRYPTO_ASYNC_OP_N_IDS
Definition: crypto.h:195
ipsec_sa_t::thread_index
u32 thread_index
Definition: ipsec_sa.h:129
vlib_get_buffer
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:111
f
vlib_frame_t * f
Definition: interface_output.c:1080
ipsec_tun_protect_get_sa_out
static index_t ipsec_tun_protect_get_sa_out(adj_index_t ai)
Definition: ipsec_tun.h:179
node
vlib_main_t vlib_node_runtime_t * node
Definition: esp_encrypt.c:1327
ipsec_per_thread_data_t::chunks
vnet_crypto_op_chunk_t * chunks
Definition: ipsec.h:104
esp_header_t::spi
u32 spi
Definition: esp.h:26
vnet_crypto_op_t::status
vnet_crypto_op_status_t status
Definition: crypto.h:260
vlib_get_buffers
vlib_get_buffers(vm, from, b, n_left_from)
next
u16 * next
Definition: nat44_ei_out2in.c:718
esp_async_post_next_t::esp_mpls_tun_post_next
u32 esp_mpls_tun_post_next
Definition: esp.h:246
VLIB_NODE_TYPE_INTERNAL
@ VLIB_NODE_TYPE_INTERNAL
Definition: node.h:72
ip6_ext_header_len
#define ip6_ext_header_len(p)
Definition: ip6_packet.h:545
VLIB_FRAME_SIZE
#define VLIB_FRAME_SIZE
Definition: node.h:368
esp4_no_crypto_tun_node
vlib_node_registration_t esp4_no_crypto_tun_node
(constructor) VLIB_REGISTER_NODE (esp4_no_crypto_tun_node)
Definition: esp_encrypt.c:1450
esp_encrypt_trace_t::udp_encap
u8 udp_encap
Definition: esp_encrypt.c:75
ipsec_sa_t::ip4_hdr
ip4_header_t ip4_hdr
Definition: ipsec_sa.h:168
esp_encrypt_error_strings
static char * esp_encrypt_error_strings[]
Definition: esp_encrypt.c:63
esp_mpls_encrypt_tun_post_node
vlib_node_registration_t esp_mpls_encrypt_tun_post_node
(constructor) VLIB_REGISTER_NODE (esp_mpls_encrypt_tun_post_node)
Definition: esp_encrypt.c:1360
tunnel_encap_fixup_mplso6
static_always_inline void tunnel_encap_fixup_mplso6(tunnel_encap_decap_flags_t flags, const vlib_buffer_t *b, const mpls_unicast_header_t *inner, ip6_header_t *outer)
Definition: tunnel_dp.h:163
esp_encrypt_trace_t::seq
u32 seq
Definition: esp_encrypt.c:73
vnet_crypto_op_t::src
u8 * src
Definition: crypto.h:277
u16
unsigned short u16
Definition: types.h:57
esp6_no_crypto_tun_node
vlib_node_registration_t esp6_no_crypto_tun_node
(constructor) VLIB_REGISTER_NODE (esp6_no_crypto_tun_node)
Definition: esp_encrypt.c:1471
vnet_crypto_op_t::dst
u8 * dst
Definition: crypto.h:278
esp_process_chained_ops
static_always_inline void esp_process_chained_ops(vlib_main_t *vm, vlib_node_runtime_t *node, vnet_crypto_op_t *ops, vlib_buffer_t *b[], u16 *nexts, vnet_crypto_op_chunk_t *chunks, u16 drop_next)
Definition: esp_encrypt.c:239
vnet_crypto_async_frame_t
Definition: crypto.h:358
esp_set_next_index
static void esp_set_next_index(vlib_buffer_t *b, vlib_node_runtime_t *node, u32 err, u16 index, u16 *nexts, u16 drop_next)
Definition: esp.h:149
vnet_crypto_async_add_to_frame
static_always_inline void vnet_crypto_async_add_to_frame(vlib_main_t *vm, vnet_crypto_async_frame_t *f, u32 key_index, u32 crypto_len, i16 integ_len_adj, i16 crypto_start_offset, u16 integ_start_offset, u32 buffer_index, u16 next_node, u8 *iv, u8 *tag, u8 *aad, u8 flags)
Definition: crypto.h:621
vnet_crypto_op_t::user_data
uword user_data
Definition: crypto.h:258
tunnel_encap_fixup_mplso4_w_chksum
static_always_inline void tunnel_encap_fixup_mplso4_w_chksum(tunnel_encap_decap_flags_t flags, const mpls_unicast_header_t *inner, ip4_header_t *outer)
Definition: tunnel_dp.h:75
mpls_unicast_header_t
Definition: packet.h:28
clib_memcpy_le32
static_always_inline void clib_memcpy_le32(u8 *dst, u8 *src, u8 len)
Definition: string.h:306
esp_aead_t_
AES GCM Additional Authentication data.
Definition: esp.h:76
esp_encrypt_trace_t::integ_alg
ipsec_integ_alg_t integ_alg
Definition: esp_encrypt.c:77
from_frame
vlib_main_t vlib_node_runtime_t vlib_frame_t * from_frame
Definition: esp_encrypt.c:1328
vlib_buffer_enqueue_to_next
vlib_buffer_enqueue_to_next(vm, node, from,(u16 *) nexts, frame->n_vectors)
ipsec_sa_t::integ_icv_size
u8 integ_icv_size
Definition: ipsec_sa.h:125
vnet_crypto_op_t::iv
u8 * iv
Definition: crypto.h:293
VNET_CRYPTO_OP_FLAG_INIT_IV
#define VNET_CRYPTO_OP_FLAG_INIT_IV
Definition: crypto.h:262
vnet_crypto_op_t::len
u32 len
Definition: crypto.h:287
ipsec_sa_t::esp_block_align
u8 esp_block_align
Definition: ipsec_sa.h:124
vlib_frame_t
Definition: node.h:372
esp_header_t::seq
u32 seq
Definition: esp.h:29
vlib_buffer_length_in_chain
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:433
clib_memcpy_fast
static_always_inline void * clib_memcpy_fast(void *restrict dst, const void *restrict src, size_t n)
Definition: string.h:92
vnet_crypto_async_submit_open_frame
static_always_inline int vnet_crypto_async_submit_open_frame(vlib_main_t *vm, vnet_crypto_async_frame_t *frame)
Definition: crypto.h:589
udp_header_t
Definition: udp_packet.h:45
ip4_header_t
Definition: ip4_packet.h:87
ESP_NO_CRYPTO_NEXT_DROP
@ ESP_NO_CRYPTO_NEXT_DROP
Definition: esp_encrypt.c:1390
ipsec_sa_t::udp_hdr
udp_header_t udp_hdr
Definition: ipsec_sa.h:171
vnet_crypto_op_t::aad_len
u16 aad_len
Definition: crypto.h:271
esp6_encrypt_post_node
vlib_node_registration_t esp6_encrypt_post_node
(constructor) VLIB_REGISTER_NODE (esp6_encrypt_post_node)
Definition: esp_encrypt.c:1212
vnet_crypto_op_t
Definition: crypto.h:255
vnet_crypto_op_t::flags
u8 flags
Definition: crypto.h:261
vlib_frame_queue_main_init
u32 vlib_frame_queue_main_init(u32 node_index, u32 frame_queue_nelts)
Definition: threads.c:1572
vnet_crypto_op_t::key_index
u32 key_index
Definition: crypto.h:292
u8x16
u8x16
Definition: vector_sse42.h:194
format_esp_post_encrypt_trace
static u8 * format_esp_post_encrypt_trace(u8 *s, va_list *args)
Definition: esp_encrypt.c:104
esp_no_crypto_inline
static uword esp_no_crypto_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: esp_encrypt.c:1404
esp_encrypt_trace_t::spi
u32 spi
Definition: esp_encrypt.c:72
esp.h
vnet_crypto_op_t::chunk_index
u32 chunk_index
Definition: crypto.h:289
round_pow2
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:279
CLIB_PREFETCH
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:80
i16
signed short i16
Definition: types.h:46
esp_encrypt_trace_t::sa_index
u32 sa_index
Definition: esp_encrypt.c:71
vlib_buffer_enqueue_to_single_next
static_always_inline void vlib_buffer_enqueue_to_single_next(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 next_index, u32 count)
Definition: buffer_node.h:348
tunnel_encap_fixup_6o6
static_always_inline void tunnel_encap_fixup_6o6(tunnel_encap_decap_flags_t flags, const ip6_header_t *inner, ip6_header_t *outer)
Definition: tunnel_dp.h:129
esp_encrypt_trace_t::sa_seq_hi
u32 sa_seq_hi
Definition: esp_encrypt.c:74
vlib_buffer_advance
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:276
vnet_crypto_process_ops
u32 vnet_crypto_process_ops(vlib_main_t *vm, vnet_crypto_op_t ops[], u32 n_ops)
Definition: crypto.c:99
esp_encrypt_async_next
esp_async_post_next_t esp_encrypt_async_next
Definition: ipsec.c:29
vec_len
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
Definition: vec_bootstrap.h:142
esp_add_footer_and_icv
static_always_inline u8 * esp_add_footer_and_icv(vlib_main_t *vm, vlib_buffer_t **last, u8 esp_align, u8 icv_sz, vlib_node_runtime_t *node, u16 buffer_data_size, uword total_len)
Definition: esp_encrypt.c:116
vlib_buffer_t::error
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:145
len
u8 len
Definition: ip_types.api:103
VNET_LINK_IP4
@ VNET_LINK_IP4
Definition: interface.h:344
VLIB_NODE_FN
#define VLIB_NODE_FN(node)
Definition: node.h:202
vec_add2
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:644
vec_add1
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:606
format_ipsec_crypto_alg
u8 * format_ipsec_crypto_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:79
vlib_buffer_alloc
static __clib_warn_unused_result u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:708
ipsec_sa_get
static ipsec_sa_t * ipsec_sa_get(u32 sa_index)
Definition: ipsec_sa.h:519
foreach_esp_encrypt_error
#define foreach_esp_encrypt_error
Definition: esp_encrypt.c:46
CLIB_UNUSED
#define CLIB_UNUSED(x)
Definition: clib.h:90
vnet_buffer
#define vnet_buffer(b)
Definition: buffer.h:437
ESP_NO_CRYPTO_ERROR_RX_PKTS
@ ESP_NO_CRYPTO_ERROR_RX_PKTS
Definition: esp_encrypt.c:1396
vec_elt_at_index
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
Definition: vec_bootstrap.h:203
esp_update_ip4_hdr
static_always_inline void esp_update_ip4_hdr(ip4_header_t *ip4, u16 len, int is_transport, int is_udp)
Definition: esp_encrypt.c:160
VLIB_NODE_FLAG_TRACE
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:291
ipsec_per_thread_data_t
Definition: ipsec.h:97
vnet_crypto_process_chained_ops
u32 vnet_crypto_process_chained_ops(vlib_main_t *vm, vnet_crypto_op_t ops[], vnet_crypto_op_chunk_t *chunks, u32 n_ops)
Definition: crypto.c:105
ipsec_sa_counters
vlib_combined_counter_main_t ipsec_sa_counters
SA packet & bytes counters.
Definition: ipsec_sa.c:27
PREDICT_FALSE
#define PREDICT_FALSE(x)
Definition: clib.h:124
esp4_encrypt_post_node
vlib_node_registration_t esp4_encrypt_post_node
(constructor) VLIB_REGISTER_NODE (esp4_encrypt_post_node)
Definition: esp_encrypt.c:1171
clib_memcpy_le64
static_always_inline void clib_memcpy_le64(u8 *dst, u8 *src, u8 len)
Definition: string.h:300
ipsec_per_thread_data_t::async_frames
vnet_crypto_async_frame_t ** async_frames
Definition: ipsec.h:105
ARRAY_LEN
#define ARRAY_LEN(x)
Definition: clib.h:70
vlib_frame_vector_args
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:301
ipsec_main_t
Definition: ipsec.h:108
ipsec_sa_t::crypto_iv_size
u8 crypto_iv_size
Definition: ipsec_sa.h:123
esp_async_post_next_t::esp4_tun_post_next
u32 esp4_tun_post_next
Definition: esp.h:244
esp_encrypt_init
static clib_error_t * esp_encrypt_init(vlib_main_t *vm)
Definition: esp_encrypt.c:1488
vnet_crypto_op_chunk_t::src
u8 * src
Definition: crypto.h:250
VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS
#define VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS
Definition: crypto.h:264
ipsec_sa_t::tunnel_flags
tunnel_encap_decap_flags_t tunnel_flags
Definition: ipsec_sa.h:177
static_always_inline
#define static_always_inline
Definition: clib.h:112
ipsec_sa_t::seq
u32 seq
Definition: ipsec_sa.h:132
tunnel_encap_fixup_6o4_w_chksum
static_always_inline void tunnel_encap_fixup_6o4_w_chksum(tunnel_encap_decap_flags_t flags, const ip6_header_t *inner, ip4_header_t *outer)
Definition: tunnel_dp.h:107
esp_prepare_sync_op
static void esp_prepare_sync_op(vlib_main_t *vm, ipsec_per_thread_data_t *ptd, vnet_crypto_op_t **crypto_ops, vnet_crypto_op_t **integ_ops, ipsec_sa_t *sa0, u8 *payload, u16 payload_len, u8 iv_sz, u8 icv_sz, u32 bi, vlib_buffer_t **b, vlib_buffer_t *lb, u32 hdr_len, esp_header_t *esp)
Definition: esp_encrypt.c:379
esp_async_post_next_t::esp6_tun_post_next
u32 esp6_tun_post_next
Definition: esp.h:245
esp_encrypt_next_t
esp_encrypt_next_t
Definition: esp_encrypt.c:39
uword
u64 uword
Definition: types.h:112
last
static heap_elt_t * last(heap_header_t *h)
Definition: heap.c:53
esp_process_ops
static_always_inline void esp_process_ops(vlib_main_t *vm, vlib_node_runtime_t *node, vnet_crypto_op_t *ops, vlib_buffer_t *b[], u16 *nexts, u16 drop_next)
Definition: esp_encrypt.c:268
esp_no_crypto_error_strings
static char * esp_no_crypto_error_strings[]
Definition: esp_encrypt.c:1399
vnet_crypto_async_reset_frame
static_always_inline void vnet_crypto_async_reset_frame(vnet_crypto_async_frame_t *f)
Definition: crypto.h:650
ipsec_main
ipsec_main_t ipsec_main
Definition: ipsec.c:28
VNET_LINK_MPLS
@ VNET_LINK_MPLS
Definition: interface.h:349
esp_mpls_encrypt_tun_node
vlib_node_registration_t esp_mpls_encrypt_tun_node
(constructor) VLIB_REGISTER_NODE (esp_mpls_encrypt_tun_node)
Definition: esp_encrypt.c:1333
vlib_main_t::thread_index
u32 thread_index
Definition: main.h:213
vlib_node_increment_counter
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
Definition: node_funcs.h:1244
esp_header_t
Definition: esp.h:22
esp_fill_udp_hdr
static_always_inline void esp_fill_udp_hdr(ipsec_sa_t *sa, udp_header_t *udp, u16 len)
Definition: esp_encrypt.c:186
vnet_crypto_op_chunk_t
Definition: crypto.h:248
vnet_crypto_async_frame_is_full
static_always_inline u8 vnet_crypto_async_frame_is_full(const vnet_crypto_async_frame_t *f)
Definition: crypto.h:665
clib_min
#define clib_min(x, y)
Definition: clib.h:342
vlib_buffer_chain_linearize
static u32 vlib_buffer_chain_linearize(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:1472
ipsec_sa_assign_thread
static u32 ipsec_sa_assign_thread(u32 thread_id)
Definition: ipsec_sa.h:512
CLIB_CACHE_LINE_BYTES
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
esp_get_ip6_hdr_len
static_always_inline u8 esp_get_ip6_hdr_len(ip6_header_t *ip6, ip6_ext_header_t **ext_hdr)
Definition: esp_encrypt.c:211
esp_no_crypto_trace_t
Definition: esp_encrypt.c:1371
ESP_ENCRYPT_N_ERROR
@ ESP_ENCRYPT_N_ERROR
Definition: esp_encrypt.c:60
vlib_node_registration_t
struct _vlib_node_registration vlib_node_registration_t
esp_aad_fill
static u16 esp_aad_fill(u8 *data, const esp_header_t *esp, const ipsec_sa_t *sa)
Definition: esp.h:121
ipsec_sa_t::crypto_alg
ipsec_crypto_alg_t crypto_alg
Definition: ipsec_sa.h:217
ipsec_sa_t::crypto_async_enc_op_id
vnet_crypto_async_op_id_t crypto_async_enc_op_id
Definition: ipsec_sa.h:155
vlib_buffer_t::current_length
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:122
ESP_MAX_BLOCK_SIZE
#define ESP_MAX_BLOCK_SIZE
Definition: esp.h:88
ESP_ENCRYPT_N_NEXT
@ ESP_ENCRYPT_N_NEXT
Definition: esp_encrypt.c:43
vnet_crypto_op_t::n_chunks
u16 n_chunks
Definition: crypto.h:282
vec_add2_aligned
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:656
ipsec_sa_t
Definition: ipsec_sa.h:116
esp_encrypt_error_t
esp_encrypt_error_t
Definition: esp_encrypt.c:55
vnet_crypto_op_chunk_t::dst
u8 * dst
Definition: crypto.h:251
foreach_esp_encrypt_next
#define foreach_esp_encrypt_next
Definition: esp_encrypt.c:29
always_inline
#define always_inline
Definition: rdma_mlx5dv.h:23
vnet_crypto_op_t::tag_len
u8 tag_len
Definition: crypto.h:269
spi
u32 spi
Definition: flow_types.api:140
u64
unsigned long u64
Definition: types.h:89
esp_encrypt_inline
static uword esp_encrypt_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, vnet_link_t lt, int is_tun, u16 async_next_node)
Definition: esp_encrypt.c:565
vnet_crypto_op_t::aad
u8 * aad
Definition: crypto.h:294
esp_encrypt_post_trace_t::next_index
u32 next_index
Definition: esp_encrypt.c:82
format
description fragment has unexpected format
Definition: map.api:433
ASSERT
#define ASSERT(truth)
Definition: error_bootstrap.h:69
vlib_buffer_get_tail
static u8 * vlib_buffer_get_tail(vlib_buffer_t *b)
Get pointer to the end of buffer's data.
Definition: buffer.h:338
ipsec_per_thread_data_t::chained_crypto_ops
vnet_crypto_op_t * chained_crypto_ops
Definition: ipsec.h:102
vlib_buffer_get_default_data_size
static_always_inline u32 vlib_buffer_get_default_data_size(vlib_main_t *vm)
Definition: buffer_funcs.h:122
ip.h
u32
unsigned int u32
Definition: types.h:88
VLIB_INIT_FUNCTION
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:172
format_esp_encrypt_trace
static u8 * format_esp_encrypt_trace(u8 *s, va_list *args)
Definition: esp_encrypt.c:87
clib_atomic_cmp_and_swap
#define clib_atomic_cmp_and_swap(addr, old, new)
Definition: atomics.h:37
ip6
vl_api_ip6_address_t ip6
Definition: one.api:424
protocol
vl_api_ip_proto_t protocol
Definition: lb_types.api:72
esp_encrypt_post_inline
static uword esp_encrypt_post_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: esp_encrypt.c:1049
pad_data
static const u8 pad_data[]
Definition: ipsec.h:177
vnet_crypto_async_get_frame
static_always_inline vnet_crypto_async_frame_t * vnet_crypto_async_get_frame(vlib_main_t *vm, vnet_crypto_async_op_id_t opt)
async crypto inline functions
Definition: crypto.h:563
vec_foreach
#define vec_foreach(var, vec)
Vector iterator.
Definition: vec_bootstrap.h:213
ipsec_sa_t::integ_op_id
vnet_crypto_op_id_t integ_op_id
Definition: ipsec_sa.h:150
n_left
u32 n_left
Definition: interface_output.c:1078
iv
static u8 iv[]
Definition: aes_cbc.c:24
ipsec_sa_t::salt
u32 salt
Definition: ipsec_sa.h:174
ipsec_sa_t::seq_hi
u32 seq_hi
Definition: ipsec_sa.h:133
ip6_header_t
Definition: ip6_packet.h:294
ipsec_sa_t::linked_key_index
vnet_crypto_key_index_t linked_key_index
Definition: ipsec_sa.h:157
esp_prepare_async_frame
static_always_inline void esp_prepare_async_frame(vlib_main_t *vm, ipsec_per_thread_data_t *ptd, vnet_crypto_async_frame_t *async_frame, ipsec_sa_t *sa, vlib_buffer_t *b, esp_header_t *esp, u8 *payload, u32 payload_len, u8 iv_sz, u8 icv_sz, u32 bi, u16 next, u32 hdr_len, u16 async_next, vlib_buffer_t *lb)
Definition: esp_encrypt.c:475
ipsec_per_thread_data_t::chained_integ_ops
vnet_crypto_op_t * chained_integ_ops
Definition: ipsec.h:103
esp6_encrypt_node
vlib_node_registration_t esp6_encrypt_node
(constructor) VLIB_REGISTER_NODE (esp6_encrypt_node)
Definition: esp_encrypt.c:1192
esp_no_crypto_trace_t::sa_index
u32 sa_index
Definition: esp_encrypt.c:1373
length
char const int length
Definition: cJSON.h:163
clib_memset
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
vlib_main_t
Definition: main.h:102
ip_csum_update
#define ip_csum_update(sum, old, new, type, field)
Definition: ip_packet.h:295
vnet_link_t
enum vnet_link_t_ vnet_link_t
Link Type: A description of the protocol of packets on the link.
vlib_node_t
Definition: node.h:247
vlib_add_trace
void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace.c:628
b
vlib_buffer_t ** b
Definition: nat44_ei_out2in.c:717
esp_async_post_next_t::esp4_post_next
u32 esp4_post_next
Definition: esp.h:242
u8
unsigned char u8
Definition: types.h:56
clib_error_t
Definition: clib_error.h:21
vnet_crypto_async_op_id_t
vnet_crypto_async_op_id_t
Definition: crypto.h:182
VNET_LINK_IP6
@ VNET_LINK_IP6
Definition: interface.h:348
ipsec_sa_t::ip6_hdr
ip6_header_t ip6_hdr
Definition: ipsec_sa.h:169
vlib_buffer_get_current
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:257
ip
vl_api_address_t ip
Definition: l2.api:558
esp_encrypt_post_trace_t
Definition: esp_encrypt.c:80
esp_encrypt_chain_integ
static_always_inline u32 esp_encrypt_chain_integ(vlib_main_t *vm, ipsec_per_thread_data_t *ptd, ipsec_sa_t *sa0, vlib_buffer_t *b, vlib_buffer_t *lb, u8 icv_sz, u8 *start, u32 start_len, u8 *digest, u16 *n_ch)
Definition: esp_encrypt.c:333
vlib_init_function_t
clib_error_t *() vlib_init_function_t(struct vlib_main_t *vm)
Definition: init.h:51
ip_csum_t
uword ip_csum_t
Definition: ip_packet.h:245
vlib_buffer_t::data
u8 data[]
Packet data.
Definition: buffer.h:204
ipsec_sa_t::integ_alg
ipsec_integ_alg_t integ_alg
Definition: ipsec_sa.h:218
vm
vlib_main_t * vm
Definition: esp_encrypt.c:1327
dpo_id_t_
The identity of a DPO is a combination of its type and its instance number/index of objects of that t...
Definition: dpo.h:172
nexts
u16 nexts[VLIB_FRAME_SIZE]
Definition: nat44_ei_out2in.c:718
esp_encrypt_trace_t
Definition: esp_encrypt.c:69
ip4_header_bytes
static int ip4_header_bytes(const ip4_header_t *i)
Definition: ip4_packet.h:190
tunnel_encap_fixup_4o4_w_chksum
static_always_inline void tunnel_encap_fixup_4o4_w_chksum(tunnel_encap_decap_flags_t flags, const ip4_header_t *inner, ip4_header_t *outer)
Definition: tunnel_dp.h:40
vnet.h
api_errno.h
ip6_ext_next_header
static void * ip6_ext_next_header(ip6_ext_header_t *ext_hdr)
Definition: ip6_packet.h:549
esp_post_data_t::next_index
u16 next_index
Definition: esp.h:219
vlib_node_runtime_t
Definition: node.h:454
tunnel_encap_fixup_4o6
static_always_inline void tunnel_encap_fixup_4o6(tunnel_encap_decap_flags_t flags, const vlib_buffer_t *b, const ip4_header_t *inner, ip6_header_t *outer)
Definition: tunnel_dp.h:145
esp_async_recycle_failed_submit
static u32 esp_async_recycle_failed_submit(vlib_main_t *vm, vnet_crypto_async_frame_t *f, vlib_node_runtime_t *node, u32 err, u16 index, u32 *from, u16 *nexts, u16 drop_next_index)
Definition: esp.h:158
esp4_encrypt_node
vlib_node_registration_t esp4_encrypt_node
(constructor) VLIB_REGISTER_NODE (esp4_encrypt_node)
Definition: esp_encrypt.c:1143
from
from
Definition: nat44_ei_hairpinning.c:415
PREDICT_TRUE
#define PREDICT_TRUE(x)
Definition: clib.h:125
vnet_crypto_op_chunk_t::len
u32 len
Definition: crypto.h:252
ip_csum_fold
static u16 ip_csum_fold(ip_csum_t c)
Definition: ip_packet.h:301
esp_post_data_t
Definition: esp.h:217
VLIB_TX
@ VLIB_TX
Definition: defs.h:47
ipsec_sa_t::spi
u32 spi
Definition: ipsec_sa.h:131
esp4_encrypt_tun_post_node
vlib_node_registration_t esp4_encrypt_tun_post_node
(constructor) VLIB_REGISTER_NODE (esp4_encrypt_tun_post_node)
Definition: esp_encrypt.c:1262
ipsec_sa_t::crypto_enc_op_id
vnet_crypto_op_id_t crypto_enc_op_id
Definition: ipsec_sa.h:148
esp_async_post_next_t::esp6_post_next
u32 esp6_post_next
Definition: esp.h:243
esp6_encrypt_tun_node
vlib_node_registration_t esp6_encrypt_tun_node
(constructor) VLIB_REGISTER_NODE (esp6_encrypt_tun_node)
Definition: esp_encrypt.c:1283
type
vl_api_fib_path_type_t type
Definition: fib_types.api:123
vlib_increment_combined_counter
vlib_increment_combined_counter(ccm, ti, sw_if_index, n_buffers, n_bytes)
tunnel_dp.h
esp6_encrypt_tun_post_node
vlib_node_registration_t esp6_encrypt_tun_post_node
(constructor) VLIB_REGISTER_NODE (esp6_encrypt_tun_post_node)
Definition: esp_encrypt.c:1314
vlib_buffer_t::flags
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index,...
Definition: buffer.h:133
ipsec_sa_t::crypto_key_index
vnet_crypto_key_index_t crypto_key_index
Definition: ipsec_sa.h:139
esp_seq_advance
static int esp_seq_advance(ipsec_sa_t *sa)
Definition: esp.h:96
ipsec_per_thread_data_t::crypto_ops
vnet_crypto_op_t * crypto_ops
Definition: ipsec.h:100
ipsec_sa_t::dpo
dpo_id_t dpo
Definition: ipsec_sa.h:137
vlib_buffer_t
VLIB buffer representation.
Definition: buffer.h:111
VLIB_REGISTER_NODE
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
flags
vl_api_wireguard_peer_flags_t flags
Definition: wireguard.api:105