FD.io VPP  v20.05.1-6-gf53edbc3b
Vector Packet Processing
device.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2017 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #define _GNU_SOURCE
19 #include <stdint.h>
20 #include <vnet/ethernet/ethernet.h>
21 #include <vnet/ip/ip4_packet.h>
22 #include <vnet/ip/ip6_packet.h>
24 #include <vnet/bonding/node.h>
25 #include <vppinfra/lb_hash_hash.h>
26 #include <vnet/ip/ip.h>
28 
29 #define foreach_bond_tx_error \
30  _(NONE, "no error") \
31  _(IF_DOWN, "interface down") \
32  _(NO_SLAVE, "no slave")
33 
34 typedef enum
35 {
36 #define _(f,s) BOND_TX_ERROR_##f,
38 #undef _
41 
42 static char *bond_tx_error_strings[] = {
43 #define _(n,s) s,
45 #undef _
46 };
47 
48 static u8 *
49 format_bond_tx_trace (u8 * s, va_list * args)
50 {
51  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
52  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
53  bond_packet_trace_t *t = va_arg (*args, bond_packet_trace_t *);
54  vnet_hw_interface_t *hw, *hw1;
55  vnet_main_t *vnm = vnet_get_main ();
56 
59  s = format (s, "src %U, dst %U, %s -> %s",
62  hw->name, hw1->name);
63 
64  return s;
65 }
66 
67 #ifndef CLIB_MARCH_VARIANT
68 u8 *
69 format_bond_interface_name (u8 * s, va_list * args)
70 {
71  u32 dev_instance = va_arg (*args, u32);
72  bond_main_t *bm = &bond_main;
73  bond_if_t *bif = pool_elt_at_index (bm->interfaces, dev_instance);
74 
75  s = format (s, "BondEthernet%lu", bif->id);
76 
77  return s;
78 }
79 #endif
80 
81 static __clib_unused clib_error_t *
83  struct vnet_hw_interface_t *bif_hw,
84  i32 l2_if_adjust)
85 {
86  bond_if_t *bif;
88  struct vnet_hw_interface_t *sif_hw;
89 
91  if (!bif)
92  return 0;
93 
94  if ((bif_hw->l2_if_count == 1) && (l2_if_adjust == 1))
95  {
96  /* Just added first L2 interface on this port */
97  vec_foreach (sw_if_index, bif->slaves)
98  {
99  sif_hw = vnet_get_sup_hw_interface (vnm, *sw_if_index);
100  ethernet_set_flags (vnm, sif_hw->hw_if_index,
102  }
103  }
104  else if ((bif_hw->l2_if_count == 0) && (l2_if_adjust == -1))
105  {
106  /* Just removed last L2 subinterface on this port */
107  vec_foreach (sw_if_index, bif->slaves)
108  {
109  sif_hw = vnet_get_sup_hw_interface (vnm, *sw_if_index);
110  ethernet_set_flags (vnm, sif_hw->hw_if_index,
111  /*ETHERNET_INTERFACE_FLAG_DEFAULT_L3 */ 0);
112  }
113  }
114 
115  return 0;
116 }
117 
118 static __clib_unused clib_error_t *
120  struct vnet_sw_interface_t *st, int is_add)
121 {
122  /* Nothing for now */
123  return 0;
124 }
125 
126 static clib_error_t *
128 {
129  vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index);
130  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
131  bond_main_t *bm = &bond_main;
133 
134  bif->admin_up = is_up;
135  if (is_up && vec_len (bif->active_slaves))
138  return 0;
139 }
140 
141 static clib_error_t *
143  u8 is_add)
144 {
145  vnet_main_t *vnm = vnet_get_main ();
146  bond_if_t *bif;
147  clib_error_t *error = 0;
148  vnet_hw_interface_t *s_hi;
149  int i;
150 
151 
153  if (!bif)
154  {
155  return clib_error_return (0, "No bond master found for sw_if_index %u",
156  hi->sw_if_index);
157  }
158 
159  /* Add/del address on each slave hw intf, they control the hardware */
160  vec_foreach_index (i, bif->slaves)
161  {
162  s_hi = vnet_get_sup_hw_interface (vnm, vec_elt (bif->slaves, i));
164  address, is_add);
165 
166  if (error)
167  {
168  int j;
169 
170  /* undo any that were completed before the failure */
171  for (j = i - 1; j > -1; j--)
172  {
173  s_hi = vnet_get_sup_hw_interface (vnm, vec_elt (bif->slaves, j));
175  address, !(is_add));
176  }
177 
178  return error;
179  }
180  }
181 
182  return 0;
183 }
184 
187 {
188  u32 idx = ptd->per_port_queue[port].n_buffers++;
189  ptd->per_port_queue[port].buffers[idx] = bi;
190 }
191 
194  bond_if_t * bif, vlib_buffer_t * b0, uword n_slaves)
195 {
196  bond_main_t *bm = &bond_main;
197  vlib_buffer_t *c0;
198  int port;
200  u16 thread_index = vm->thread_index;
202  thread_index);
203 
204  for (port = 1; port < n_slaves; port++)
205  {
206  sw_if_index = *vec_elt_at_index (bif->active_slaves, port);
207  c0 = vlib_buffer_copy (vm, b0);
208  if (PREDICT_TRUE (c0 != 0))
209  {
210  vnet_buffer (c0)->sw_if_index[VLIB_TX] = sw_if_index;
211  bond_tx_add_to_queue (ptd, port, vlib_get_buffer_index (vm, c0));
212  }
213  }
214 
215  return 0;
216 }
217 
220 {
222  u64 *dst = (u64 *) & eth->dst_address[0];
223  u64 a = clib_mem_unaligned (dst, u64);
224  u32 *src = (u32 *) & eth->src_address[2];
225  u32 b = clib_mem_unaligned (src, u32);
226 
227  return lb_hash_hash_2_tuples (a, b);
228 }
229 
232 {
233  u16 *ethertype_p;
235 
236  if (!ethernet_frame_is_tagged (clib_net_to_host_u16 (eth->type)))
237  {
238  ethertype_p = &eth->type;
239  }
240  else
241  {
242  vlan = (void *) (eth + 1);
243  ethertype_p = &vlan->type;
244  if (*ethertype_p == ntohs (ETHERNET_TYPE_VLAN))
245  {
246  vlan++;
247  ethertype_p = &vlan->type;
248  }
249  }
250  return ethertype_p;
251 }
252 
255 {
257  u8 ip_version;
258  ip4_header_t *ip4;
259  u16 ethertype, *ethertype_p;
260  u32 *mac1, *mac2, *mac3;
261 
262  ethertype_p = bond_locate_ethertype (eth);
263  ethertype = clib_mem_unaligned (ethertype_p, u16);
264 
265  if ((ethertype != htons (ETHERNET_TYPE_IP4)) &&
266  (ethertype != htons (ETHERNET_TYPE_IP6)))
267  return bond_lb_l2 (b0);
268 
269  ip4 = (ip4_header_t *) (ethertype_p + 1);
270  ip_version = (ip4->ip_version_and_header_length >> 4);
271 
272  if (ip_version == 0x4)
273  {
274  u32 a, c;
275 
276  mac1 = (u32 *) & eth->dst_address[0];
277  mac2 = (u32 *) & eth->dst_address[4];
278  mac3 = (u32 *) & eth->src_address[2];
279 
280  a = clib_mem_unaligned (mac1, u32) ^ clib_mem_unaligned (mac2, u32) ^
281  clib_mem_unaligned (mac3, u32);
282  c =
284  a);
285  return c;
286  }
287  else if (ip_version == 0x6)
288  {
289  u64 a;
290  u32 c;
291  ip6_header_t *ip6 = (ip6_header_t *) (eth + 1);
292 
293  mac1 = (u32 *) & eth->dst_address[0];
294  mac2 = (u32 *) & eth->dst_address[4];
295  mac3 = (u32 *) & eth->src_address[2];
296 
297  a = clib_mem_unaligned (mac1, u32) ^ clib_mem_unaligned (mac2, u32) ^
298  clib_mem_unaligned (mac3, u32);
299  c =
301  (&ip6->src_address.as_uword[0], uword),
302  clib_mem_unaligned (&ip6->src_address.as_uword[1],
303  uword),
304  clib_mem_unaligned (&ip6->dst_address.as_uword[0],
305  uword),
306  clib_mem_unaligned (&ip6->dst_address.as_uword[1],
307  uword), a);
308  return c;
309  }
310  return bond_lb_l2 (b0);
311 }
312 
315 {
317  u8 ip_version;
318  uword is_tcp_udp;
319  ip4_header_t *ip4;
320  u16 ethertype, *ethertype_p;
321 
322  ethertype_p = bond_locate_ethertype (eth);
323  ethertype = clib_mem_unaligned (ethertype_p, u16);
324 
325  if ((ethertype != htons (ETHERNET_TYPE_IP4)) &&
326  (ethertype != htons (ETHERNET_TYPE_IP6)))
327  return (bond_lb_l2 (b0));
328 
329  ip4 = (ip4_header_t *) (ethertype_p + 1);
330  ip_version = (ip4->ip_version_and_header_length >> 4);
331 
332  if (ip_version == 0x4)
333  {
334  u32 a, t1, t2;
335  tcp_header_t *tcp = (void *) (ip4 + 1);
336 
337  is_tcp_udp = (ip4->protocol == IP_PROTOCOL_TCP) ||
338  (ip4->protocol == IP_PROTOCOL_UDP);
339  t1 = is_tcp_udp ? clib_mem_unaligned (&tcp->src, u16) : 0;
340  t2 = is_tcp_udp ? clib_mem_unaligned (&tcp->dst, u16) : 0;
341  a = t1 ^ t2;
342  return
343  lb_hash_hash_2_tuples (clib_mem_unaligned (&ip4->address_pair, u64),
344  a);
345  }
346  else if (ip_version == 0x6)
347  {
348  u64 a;
349  u32 c, t1, t2;
350  ip6_header_t *ip6 = (ip6_header_t *) (eth + 1);
351  tcp_header_t *tcp = (void *) (ip6 + 1);
352 
353  is_tcp_udp = 0;
354  if (PREDICT_TRUE ((ip6->protocol == IP_PROTOCOL_TCP) ||
355  (ip6->protocol == IP_PROTOCOL_UDP)))
356  {
357  is_tcp_udp = 1;
358  tcp = (void *) (ip6 + 1);
359  }
360  else if (ip6->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)
361  {
363  (ip6_hop_by_hop_header_t *) (ip6 + 1);
364  if ((hbh->protocol == IP_PROTOCOL_TCP)
365  || (hbh->protocol == IP_PROTOCOL_UDP))
366  {
367  is_tcp_udp = 1;
368  tcp = (tcp_header_t *) ((u8 *) hbh + ((hbh->length + 1) << 3));
369  }
370  }
371  t1 = is_tcp_udp ? clib_mem_unaligned (&tcp->src, u16) : 0;
372  t2 = is_tcp_udp ? clib_mem_unaligned (&tcp->dst, u16) : 0;
373  a = t1 ^ t2;
374  c =
375  lb_hash_hash (clib_mem_unaligned
376  (&ip6->src_address.as_uword[0], uword),
377  clib_mem_unaligned (&ip6->src_address.as_uword[1],
378  uword),
379  clib_mem_unaligned (&ip6->dst_address.as_uword[0],
380  uword),
381  clib_mem_unaligned (&ip6->dst_address.as_uword[1],
382  uword), a);
383  return c;
384  }
385 
386  return bond_lb_l2 (b0);
387 }
388 
391 {
392  bif->lb_rr_last_index++;
393  if (bif->lb_rr_last_index >= n_slaves)
394  bif->lb_rr_last_index = 0;
395 
396  return bif->lb_rr_last_index;
397 }
398 
401  u32 * h, u32 n_left, uword n_slaves, u32 lb_alg)
402 {
403  while (n_left >= 4)
404  {
405  // Prefetch next iteration
406  if (n_left >= 8)
407  {
408  vlib_buffer_t **pb = b + 4;
409 
410  vlib_prefetch_buffer_header (pb[0], LOAD);
411  vlib_prefetch_buffer_header (pb[1], LOAD);
412  vlib_prefetch_buffer_header (pb[2], LOAD);
413  vlib_prefetch_buffer_header (pb[3], LOAD);
414 
415  CLIB_PREFETCH (pb[0]->data, CLIB_CACHE_LINE_BYTES, LOAD);
416  CLIB_PREFETCH (pb[1]->data, CLIB_CACHE_LINE_BYTES, LOAD);
417  CLIB_PREFETCH (pb[2]->data, CLIB_CACHE_LINE_BYTES, LOAD);
418  CLIB_PREFETCH (pb[3]->data, CLIB_CACHE_LINE_BYTES, LOAD);
419  }
420 
425 
426  if (lb_alg == BOND_LB_L2)
427  {
428  h[0] = bond_lb_l2 (b[0]);
429  h[1] = bond_lb_l2 (b[1]);
430  h[2] = bond_lb_l2 (b[2]);
431  h[3] = bond_lb_l2 (b[3]);
432  }
433  else if (lb_alg == BOND_LB_L34)
434  {
435  h[0] = bond_lb_l34 (b[0]);
436  h[1] = bond_lb_l34 (b[1]);
437  h[2] = bond_lb_l34 (b[2]);
438  h[3] = bond_lb_l34 (b[3]);
439  }
440  else if (lb_alg == BOND_LB_L23)
441  {
442  h[0] = bond_lb_l23 (b[0]);
443  h[1] = bond_lb_l23 (b[1]);
444  h[2] = bond_lb_l23 (b[2]);
445  h[3] = bond_lb_l23 (b[3]);
446  }
447  else if (lb_alg == BOND_LB_RR)
448  {
449  h[0] = bond_lb_round_robin (bif, b[0], n_slaves);
450  h[1] = bond_lb_round_robin (bif, b[1], n_slaves);
451  h[2] = bond_lb_round_robin (bif, b[2], n_slaves);
452  h[3] = bond_lb_round_robin (bif, b[3], n_slaves);
453  }
454  else if (lb_alg == BOND_LB_BC)
455  {
456  h[0] = bond_lb_broadcast (vm, bif, b[0], n_slaves);
457  h[1] = bond_lb_broadcast (vm, bif, b[1], n_slaves);
458  h[2] = bond_lb_broadcast (vm, bif, b[2], n_slaves);
459  h[3] = bond_lb_broadcast (vm, bif, b[3], n_slaves);
460  }
461  else
462  {
463  ASSERT (0);
464  }
465 
466  n_left -= 4;
467  b += 4;
468  h += 4;
469  }
470 
471  while (n_left > 0)
472  {
474 
475  if (bif->lb == BOND_LB_L2)
476  h[0] = bond_lb_l2 (b[0]);
477  else if (bif->lb == BOND_LB_L34)
478  h[0] = bond_lb_l34 (b[0]);
479  else if (bif->lb == BOND_LB_L23)
480  h[0] = bond_lb_l23 (b[0]);
481  else if (bif->lb == BOND_LB_RR)
482  h[0] = bond_lb_round_robin (bif, b[0], n_slaves);
483  else if (bif->lb == BOND_LB_BC)
484  h[0] = bond_lb_broadcast (vm, bif, b[0], n_slaves);
485  else
486  {
487  ASSERT (0);
488  }
489 
490  n_left -= 1;
491  b += 1;
492  h += 1;
493  }
494 }
495 
497 bond_hash_to_port (u32 * h, u32 n_left, u32 n_slaves, int use_modulo_shortcut)
498 {
499  u32 mask = n_slaves - 1;
500 
501 #ifdef CLIB_HAVE_VEC256
502  /* only lower 16 bits of hash due to single precision fp arithmetic */
503  u32x8 mask8, sc8u, h8a, h8b;
504  f32x8 sc8f;
505 
506  if (use_modulo_shortcut)
507  {
508  mask8 = u32x8_splat (mask);
509  }
510  else
511  {
512  mask8 = u32x8_splat (0xffff);
513  sc8u = u32x8_splat (n_slaves);
514  sc8f = f32x8_from_u32x8 (sc8u);
515  }
516 
517  while (n_left > 16)
518  {
519  h8a = u32x8_load_unaligned (h) & mask8;
520  h8b = u32x8_load_unaligned (h + 8) & mask8;
521 
522  if (use_modulo_shortcut == 0)
523  {
524  h8a -= sc8u * u32x8_from_f32x8 (f32x8_from_u32x8 (h8a) / sc8f);
525  h8b -= sc8u * u32x8_from_f32x8 (f32x8_from_u32x8 (h8b) / sc8f);
526  }
527 
528  u32x8_store_unaligned (h8a, h);
529  u32x8_store_unaligned (h8b, h + 8);
530  n_left -= 16;
531  h += 16;
532  }
533 #endif
534 
535  while (n_left > 4)
536  {
537  if (use_modulo_shortcut)
538  {
539  h[0] &= mask;
540  h[1] &= mask;
541  h[2] &= mask;
542  h[3] &= mask;
543  }
544  else
545  {
546  h[0] %= n_slaves;
547  h[1] %= n_slaves;
548  h[2] %= n_slaves;
549  h[3] %= n_slaves;
550  }
551  n_left -= 4;
552  h += 4;
553  }
554  while (n_left)
555  {
556  if (use_modulo_shortcut)
557  h[0] &= mask;
558  else
559  h[0] %= n_slaves;
560  n_left -= 1;
561  h += 1;
562  }
563 }
564 
567  u32 * bi, vlib_buffer_t ** b, u32 * data, u32 n_left,
568  int single_sw_if_index)
569 {
570  u32 sw_if_index = data[0];
571  u32 *h = data;
572 
573  while (n_left >= 4)
574  {
575  // Prefetch next iteration
576  if (n_left >= 8)
577  {
578  vlib_buffer_t **pb = b + 4;
579  vlib_prefetch_buffer_header (pb[0], LOAD);
580  vlib_prefetch_buffer_header (pb[1], LOAD);
581  vlib_prefetch_buffer_header (pb[2], LOAD);
582  vlib_prefetch_buffer_header (pb[3], LOAD);
583  }
584 
585  if (PREDICT_FALSE (single_sw_if_index))
586  {
587  vnet_buffer (b[0])->sw_if_index[VLIB_TX] = sw_if_index;
588  vnet_buffer (b[1])->sw_if_index[VLIB_TX] = sw_if_index;
589  vnet_buffer (b[2])->sw_if_index[VLIB_TX] = sw_if_index;
590  vnet_buffer (b[3])->sw_if_index[VLIB_TX] = sw_if_index;
591 
592  bond_tx_add_to_queue (ptd, 0, bi[0]);
593  bond_tx_add_to_queue (ptd, 0, bi[1]);
594  bond_tx_add_to_queue (ptd, 0, bi[2]);
595  bond_tx_add_to_queue (ptd, 0, bi[3]);
596  }
597  else
598  {
599  u32 sw_if_index[4];
600 
601  sw_if_index[0] = *vec_elt_at_index (bif->active_slaves, h[0]);
602  sw_if_index[1] = *vec_elt_at_index (bif->active_slaves, h[1]);
603  sw_if_index[2] = *vec_elt_at_index (bif->active_slaves, h[2]);
604  sw_if_index[3] = *vec_elt_at_index (bif->active_slaves, h[3]);
605 
606  vnet_buffer (b[0])->sw_if_index[VLIB_TX] = sw_if_index[0];
607  vnet_buffer (b[1])->sw_if_index[VLIB_TX] = sw_if_index[1];
608  vnet_buffer (b[2])->sw_if_index[VLIB_TX] = sw_if_index[2];
609  vnet_buffer (b[3])->sw_if_index[VLIB_TX] = sw_if_index[3];
610 
611  bond_tx_add_to_queue (ptd, h[0], bi[0]);
612  bond_tx_add_to_queue (ptd, h[1], bi[1]);
613  bond_tx_add_to_queue (ptd, h[2], bi[2]);
614  bond_tx_add_to_queue (ptd, h[3], bi[3]);
615  }
616 
617  bi += 4;
618  h += 4;
619  b += 4;
620  n_left -= 4;
621  }
622  while (n_left)
623  {
624  if (PREDICT_FALSE (single_sw_if_index))
625  {
626  vnet_buffer (b[0])->sw_if_index[VLIB_TX] = sw_if_index;
627  bond_tx_add_to_queue (ptd, 0, bi[0]);
628  }
629  else
630  {
631  u32 sw_if_index0 = *vec_elt_at_index (bif->active_slaves, h[0]);
632 
633  vnet_buffer (b[0])->sw_if_index[VLIB_TX] = sw_if_index0;
634  bond_tx_add_to_queue (ptd, h[0], bi[0]);
635  }
636 
637  bi += 1;
638  h += 1;
639  b += 1;
640  n_left -= 1;
641  }
642 }
643 
646  vlib_buffer_t ** b, u32 n_left, u32 * h)
647 {
648  uword n_trace = vlib_get_trace_count (vm, node);
649 
650  while (n_trace > 0 && n_left > 0)
651  {
653  ethernet_header_t *eth;
654  u32 next0 = 0;
655 
656  vlib_trace_buffer (vm, node, next0, b[0], 0 /* follow_chain */ );
657  vlib_set_trace_count (vm, node, --n_trace);
658  t0 = vlib_add_trace (vm, node, b[0], sizeof (*t0));
659  eth = vlib_buffer_get_current (b[0]);
660  t0->ethernet = *eth;
661  t0->sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_TX];
662  if (!h)
663  {
665  }
666  else
667  {
668  t0->bond_sw_if_index = *vec_elt_at_index (bif->active_slaves, h[0]);
669  h++;
670  }
671  b++;
672  n_left--;
673  }
674 }
675 
679 {
680  vnet_interface_output_runtime_t *rund = (void *) node->runtime_data;
681  bond_main_t *bm = &bond_main;
682  u16 thread_index = vm->thread_index;
683  bond_if_t *bif = pool_elt_at_index (bm->interfaces, rund->dev_instance);
684  uword n_slaves;
686  u32 *from = vlib_frame_vector_args (frame);
687  u32 n_left = frame->n_vectors;
688  u32 hashes[VLIB_FRAME_SIZE], *h;
689  vnet_main_t *vnm = vnet_get_main ();
690  bond_per_thread_data_t *ptd = vec_elt_at_index (bm->per_thread_data,
691  thread_index);
692  u32 p, sw_if_index;
693 
694  if (PREDICT_FALSE (bif->admin_up == 0))
695  {
699  thread_index, bif->sw_if_index,
700  frame->n_vectors);
701  vlib_error_count (vm, node->node_index, BOND_TX_ERROR_IF_DOWN,
702  frame->n_vectors);
703  return frame->n_vectors;
704  }
705 
706  n_slaves = vec_len (bif->active_slaves);
707  if (PREDICT_FALSE (n_slaves == 0))
708  {
712  thread_index, bif->sw_if_index,
713  frame->n_vectors);
714  vlib_error_count (vm, node->node_index, BOND_TX_ERROR_NO_SLAVE,
715  frame->n_vectors);
716  return frame->n_vectors;
717  }
718 
719  vlib_get_buffers (vm, from, bufs, n_left);
720 
721  /* active-backup mode, ship everything to first sw if index */
722  if ((bif->lb == BOND_LB_AB) || PREDICT_FALSE (n_slaves == 1))
723  {
724  sw_if_index = *vec_elt_at_index (bif->active_slaves, 0);
725 
726  bond_tx_trace (vm, node, bif, bufs, frame->n_vectors, 0);
727  bond_update_sw_if_index (ptd, bif, from, bufs, &sw_if_index, n_left,
728  /* single_sw_if_index */ 1);
729  goto done;
730  }
731 
732  if (bif->lb == BOND_LB_BC)
733  {
734  sw_if_index = *vec_elt_at_index (bif->active_slaves, 0);
735 
736  bond_tx_inline (vm, bif, bufs, hashes, n_left, n_slaves, BOND_LB_BC);
737  bond_tx_trace (vm, node, bif, bufs, frame->n_vectors, 0);
738  bond_update_sw_if_index (ptd, bif, from, bufs, &sw_if_index, n_left,
739  /* single_sw_if_index */ 1);
740  goto done;
741  }
742 
743  /* if have at least one slave on local numa node, only slaves on local numa
744  node will transmit pkts when bif->local_numa_only is enabled */
745  if (bif->n_numa_slaves >= 1)
746  n_slaves = bif->n_numa_slaves;
747 
748  if (bif->lb == BOND_LB_L2)
749  bond_tx_inline (vm, bif, bufs, hashes, n_left, n_slaves, BOND_LB_L2);
750  else if (bif->lb == BOND_LB_L34)
751  bond_tx_inline (vm, bif, bufs, hashes, n_left, n_slaves, BOND_LB_L34);
752  else if (bif->lb == BOND_LB_L23)
753  bond_tx_inline (vm, bif, bufs, hashes, n_left, n_slaves, BOND_LB_L23);
754  else if (bif->lb == BOND_LB_RR)
755  bond_tx_inline (vm, bif, bufs, hashes, n_left, n_slaves, BOND_LB_RR);
756  else
757  ASSERT (0);
758 
759  /* calculate port out of hash */
760  h = hashes;
761  if (BOND_MODULO_SHORTCUT (n_slaves))
762  bond_hash_to_port (h, frame->n_vectors, n_slaves, 1);
763  else
764  bond_hash_to_port (h, frame->n_vectors, n_slaves, 0);
765 
766  bond_tx_trace (vm, node, bif, bufs, frame->n_vectors, h);
767 
768  bond_update_sw_if_index (ptd, bif, from, bufs, hashes, frame->n_vectors,
769  /* single_sw_if_index */ 0);
770 
771 done:
772  for (p = 0; p < n_slaves; p++)
773  {
774  vlib_frame_t *f;
775  u32 *to_next;
776 
777  sw_if_index = *vec_elt_at_index (bif->active_slaves, p);
778  if (PREDICT_TRUE (ptd->per_port_queue[p].n_buffers))
779  {
780  f = vnet_get_frame_to_sw_interface (vnm, sw_if_index);
781  f->n_vectors = ptd->per_port_queue[p].n_buffers;
782  to_next = vlib_frame_vector_args (f);
783  clib_memcpy_fast (to_next, ptd->per_port_queue[p].buffers,
784  f->n_vectors * sizeof (u32));
785  vnet_put_frame_to_sw_interface (vnm, sw_if_index, f);
786  ptd->per_port_queue[p].n_buffers = 0;
787  }
788  }
789  return frame->n_vectors;
790 }
791 
792 static walk_rc_t
794  void *arg)
795 {
796  bond_main_t *bm = &bond_main;
797 
798  ip_neighbor_advertise (bm->vlib_main, IP46_TYPE_BOTH, NULL, sw_if_index);
799 
800  return (WALK_CONTINUE);
801 }
802 
803 static uword
805 {
806  vnet_main_t *vnm = vnet_get_main ();
807  uword event_type, *event_data = 0;
808 
809  while (1)
810  {
811  u32 i;
813 
815  event_type = vlib_process_get_events (vm, &event_data);
816  ASSERT (event_type == BOND_SEND_GARP_NA);
817  for (i = 0; i < vec_len (event_data); i++)
818  {
819  hw_if_index = event_data[i];
820  if (vnet_get_hw_interface_or_null (vnm, hw_if_index))
821  /* walk hw interface to process all subinterfaces */
822  vnet_hw_interface_walk_sw (vnm, hw_if_index,
824  }
825  vec_reset_length (event_data);
826  }
827  return 0;
828 }
829 
830 /* *INDENT-OFF* */
832  .function = bond_process,
834  .type = VLIB_NODE_TYPE_PROCESS,
835  .name = "bond-process",
836 };
837 /* *INDENT-ON* */
838 
839 /* *INDENT-OFF* */
841  .name = "bond",
842  .tx_function_n_errors = BOND_TX_N_ERROR,
843  .tx_function_error_strings = bond_tx_error_strings,
844  .format_device_name = format_bond_interface_name,
845  .set_l2_mode_function = bond_set_l2_mode_function,
846  .admin_up_down_function = bond_interface_admin_up_down,
847  .subif_add_del_function = bond_subif_add_del_function,
848  .format_tx_trace = format_bond_tx_trace,
849  .mac_addr_add_del_function = bond_add_del_mac_address,
850 };
851 
852 /* *INDENT-ON* */
853 
854 static clib_error_t *
856 {
857  bond_main_t *bm = &bond_main;
858  slave_if_t *sif;
859  bond_detach_slave_args_t args = { 0 };
860 
861  if (is_add)
862  return 0;
863  sif = bond_get_slave_by_sw_if_index (sw_if_index);
864  if (!sif)
865  return 0;
866  args.slave = sw_if_index;
867  bond_detach_slave (bm->vlib_main, &args);
868  return args.error;
869 }
870 
872 
873 /*
874  * fd.io coding-style-patch-verification: ON
875  *
876  * Local Variables:
877  * eval: (c-set-style "gnu")
878  * End:
879  */
#define foreach_bond_tx_error
Definition: device.c:29
#define vec_foreach_index(var, v)
Iterate over vector indices.
u32 hw_if_index
Definition: node.h:183
#define CLIB_UNUSED(x)
Definition: clib.h:86
static walk_rc_t bond_active_interface_switch_cb(vnet_main_t *vnm, u32 sw_if_index, void *arg)
Definition: device.c:793
static u32 vlib_get_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt)
Definition: trace_funcs.h:187
a
Definition: bitmap.h:538
static_always_inline u32 bond_lb_broadcast(vlib_main_t *vm, bond_if_t *bif, vlib_buffer_t *b0, uword n_slaves)
Definition: device.c:193
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:937
static uword * vlib_process_wait_for_event(vlib_main_t *vm)
Definition: node_funcs.h:593
vnet_main_t * vnet_get_main(void)
Definition: misc.c:46
static_always_inline void bond_hash_to_port(u32 *h, u32 n_left, u32 n_slaves, int use_modulo_shortcut)
Definition: device.c:497
static vnet_hw_interface_t * vnet_get_sup_hw_interface(vnet_main_t *vnm, u32 sw_if_index)
u8 lb
Definition: node.h:172
static clib_error_t * bond_add_del_mac_address(vnet_hw_interface_t *hi, const u8 *address, u8 is_add)
Definition: device.c:142
vnet_interface_main_t interface_main
Definition: vnet.h:56
#define PREDICT_TRUE(x)
Definition: clib.h:119
static __clib_unused clib_error_t * bond_subif_add_del_function(vnet_main_t *vnm, u32 hw_if_index, struct vnet_sw_interface_t *st, int is_add)
Definition: device.c:119
unsigned long u64
Definition: types.h:89
u32 buffers[VLIB_FRAME_SIZE]
Definition: node.h:159
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
static bond_if_t * bond_get_master_by_sw_if_index(u32 sw_if_index)
Definition: node.h:502
static_always_inline u32 bond_lb_l34(vlib_buffer_t *b0)
Definition: device.c:314
void vnet_hw_interface_walk_sw(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_sw_interface_walk_t fn, void *ctx)
Walk the SW interfaces on a HW interface - this is the super interface and any sub-interfaces.
Definition: interface.c:1049
#define VLIB_NODE_FLAG_TRACE_SUPPORTED
Definition: node.h:308
u8 src_address[6]
Definition: packet.h:56
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u32 thread_index
Definition: main.h:218
static_always_inline void bond_update_sw_if_index(bond_per_thread_data_t *ptd, bond_if_t *bif, u32 *bi, vlib_buffer_t **b, u32 *data, u32 n_left, int single_sw_if_index)
Definition: device.c:566
vl_api_address_t src
Definition: gre.api:54
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
static void vlib_increment_simple_counter(vlib_simple_counter_main_t *cm, u32 thread_index, u32 index, u64 increment)
Increment a simple counter.
Definition: counter.h:78
void bond_detach_slave(vlib_main_t *vm, bond_detach_slave_args_t *args)
Definition: cli.c:826
bond_tx_error_t
Definition: device.c:34
static_always_inline u32 lb_hash_hash_2_tuples(u64 k0, u32 k1)
Definition: lb_hash_hash.h:54
#define BOND_MODULO_SHORTCUT(a)
Definition: node.h:34
u32 id
Definition: node.h:181
struct _tcp_header tcp_header_t
ip6_address_t src_address
Definition: ip6_packet.h:310
unsigned char u8
Definition: types.h:56
bond_per_thread_data_t * per_thread_data
Definition: node.h:386
static vlib_buffer_t * vlib_buffer_copy(vlib_main_t *vm, vlib_buffer_t *b)
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
VNET_DEVICE_CLASS_TX_FN() bond_dev_class(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: device.c:676
static void vlib_trace_buffer(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, vlib_buffer_t *b, int follow_chain)
Definition: trace_funcs.h:130
static __clib_unused clib_error_t * bond_set_l2_mode_function(vnet_main_t *vnm, struct vnet_hw_interface_t *bif_hw, i32 l2_if_adjust)
Definition: device.c:82
enum walk_rc_t_ walk_rc_t
Walk return code.
#define static_always_inline
Definition: clib.h:106
vlib_node_registration_t bond_process_node
(constructor) VLIB_REGISTER_NODE (bond_process_node)
Definition: device.c:831
u8 * format_ethernet_address(u8 *s, va_list *args)
Definition: format.c:44
vl_api_interface_index_t sw_if_index
Definition: gre.api:53
u8 * format_bond_interface_name(u8 *s, va_list *args)
Definition: device.c:69
static uword vlib_process_get_events(vlib_main_t *vm, uword **data_vector)
Return the first event type which has occurred and a vector of per-event data of that type...
Definition: node_funcs.h:516
vl_api_ip6_address_t ip6
Definition: one.api:424
u8 dst_address[6]
Definition: packet.h:55
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:203
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
vlib_main_t * vlib_main
Definition: node.h:376
#define VNET_DEVICE_CLASS_TX_FN(devclass)
Definition: interface.h:305
unsigned int u32
Definition: types.h:88
u32 lb_rr_last_index
Definition: node.h:175
#define VLIB_FRAME_SIZE
Definition: node.h:380
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:293
static char * bond_tx_error_strings[]
Definition: device.c:42
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:534
static_always_inline u32 bond_lb_round_robin(bond_if_t *bif, vlib_buffer_t *b0, uword n_slaves)
Definition: device.c:390
static u8 * format_bond_tx_trace(u8 *s, va_list *args)
Definition: device.c:49
u8 admin_up
Definition: node.h:170
static_always_inline u32 bond_lb_l23(vlib_buffer_t *b0)
Definition: device.c:254
unsigned short u16
Definition: types.h:57
vec_header_t h
Definition: buffer.c:322
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
ip4_address_pair_t address_pair
Definition: ip4_packet.h:172
#define PREDICT_FALSE(x)
Definition: clib.h:118
vl_api_ip4_address_t ip4
Definition: one.api:376
vnet_main_t vnet_main
Definition: misc.c:43
static clib_error_t * bond_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: device.c:127
vlib_simple_counter_main_t * sw_if_counters
Definition: interface.h:866
vl_api_address_t dst
Definition: gre.api:55
bond_if_t * interfaces
Definition: node.h:364
static_always_inline void bond_tx_trace(vlib_main_t *vm, vlib_node_runtime_t *node, bond_if_t *bif, vlib_buffer_t **b, u32 n_left, u32 *h)
Definition: device.c:645
vlib_main_t * vm
Definition: in2out_ed.c:1599
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
u32 flags
Definition: vhost_user.h:248
static clib_error_t * bond_slave_interface_add_del(vnet_main_t *vnm, u32 sw_if_index, u32 is_add)
Definition: device.c:855
u32 * slaves
Definition: node.h:187
svmdb_client_t * c
u16 n_vectors
Definition: node.h:399
static void vnet_put_frame_to_sw_interface(vnet_main_t *vnm, u32 sw_if_index, vlib_frame_t *f)
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:80
VNET_SW_INTERFACE_ADD_DEL_FUNCTION(bond_slave_interface_add_del)
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:317
ethernet_header_t ethernet
Definition: node.h:394
static uword bond_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: device.c:804
static_always_inline void bond_tx_add_to_queue(bond_per_thread_data_t *ptd, u32 port, u32 bi)
Definition: device.c:186
#define ETHERNET_INTERFACE_FLAG_ACCEPT_ALL
Definition: ethernet.h:152
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1599
static_always_inline u32 bond_lb_l2(vlib_buffer_t *b0)
Definition: device.c:219
signed int i32
Definition: types.h:77
#define ASSERT(truth)
VNET_DEVICE_CLASS(avf_device_class,)
manual_print typedef address
Definition: ip_types.api:85
u8 data[128]
Definition: ipsec_types.api:89
bond_main_t bond_main
Definition: node.c:25
static_always_inline int ethernet_frame_is_tagged(u16 type)
Definition: ethernet.h:78
u32 * active_slaves
Definition: node.h:190
#define clib_mem_unaligned(pointer, type)
Definition: types.h:155
static vlib_frame_t * vnet_get_frame_to_sw_interface(vnet_main_t *vnm, u32 sw_if_index)
vl_api_ip4_address_t hi
Definition: arp.api:37
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
clib_error_t * error
Definition: node.h:111
#define vec_elt(v, i)
Get vector value at index i.
static_always_inline u32x8 u32x8_from_f32x8(f32x8 v)
Definition: vector_avx2.h:206
Definition: defs.h:47
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
Definition: interface.c:498
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:492
void ip_neighbor_advertise(vlib_main_t *vm, ip46_type_t type, const ip46_address_t *addr, u32 sw_if_index)
Definition: ip_neighbor.c:1021
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
u16 port
Definition: lb_types.api:72
#define vnet_buffer(b)
Definition: buffer.h:417
static vnet_hw_interface_t * vnet_get_hw_interface_or_null(vnet_main_t *vnm, u32 hw_if_index)
#define vec_foreach(var, vec)
Vector iterator.
static_always_inline u32 lb_hash_hash(u64 k0, u64 k1, u64 k2, u64 k3, u64 k4)
Definition: lb_hash_hash.h:46
static_always_inline void bond_tx_inline(vlib_main_t *vm, bond_if_t *bif, vlib_buffer_t **b, u32 *h, u32 n_left, uword n_slaves, u32 lb_alg)
Definition: device.c:400
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
Definition: in2out_ed.c:1600
clib_error_t * vnet_hw_interface_add_del_mac_address(vnet_main_t *vnm, u32 hw_if_index, const u8 *mac_address, u8 is_add)
Definition: interface.c:1456
u8 ip_version_and_header_length
Definition: ip4_packet.h:138
static slave_if_t * bond_get_slave_by_sw_if_index(u32 sw_if_index)
Definition: node.h:524
static void vlib_set_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 count)
Definition: trace_funcs.h:203
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:280
u32 bond_sw_if_index
Definition: node.h:396
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
static_always_inline u16 * bond_locate_ethertype(ethernet_header_t *eth)
Definition: device.c:231
static_always_inline f32x8 f32x8_from_u32x8(u32x8 v)
Definition: vector_avx2.h:200
ip6_address_t dst_address
Definition: ip6_packet.h:310
bond_per_port_queue_t * per_port_queue
Definition: node.h:165
u32 ethernet_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: interface.c:426