FD.io VPP  v20.05.1-6-gf53edbc3b
Vector Packet Processing
threads.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #define _GNU_SOURCE
16 
17 #include <signal.h>
18 #include <math.h>
19 #include <vppinfra/format.h>
20 #include <vppinfra/time_range.h>
21 #include <vppinfra/linux/sysfs.h>
22 #include <vlib/vlib.h>
23 
24 #include <vlib/threads.h>
25 
26 #include <vlib/stat_weak_inlines.h>
27 
28 u32
29 vl (void *p)
30 {
31  return vec_len (p);
32 }
33 
36 
37 /*
38  * Barrier tracing can be enabled on a normal build to collect information
39  * on barrier use, including timings and call stacks. Deliberately not
40  * keyed off CLIB_DEBUG, because that can add significant overhead which
41  * imapacts observed timings.
42  */
43 
44 static inline void
45 barrier_trace_sync (f64 t_entry, f64 t_open, f64 t_closed)
46 {
47  if (!vlib_worker_threads->barrier_elog_enabled)
48  return;
49 
50  /* *INDENT-OFF* */
51  ELOG_TYPE_DECLARE (e) =
52  {
53  .format = "bar-trace-%s-#%d",
54  .format_args = "T4i4",
55  };
56  /* *INDENT-ON* */
57  struct
58  {
59  u32 caller, count, t_entry, t_open, t_closed;
60  } *ed = 0;
61 
63  ed->count = (int) vlib_worker_threads[0].barrier_sync_count;
64  ed->caller = elog_string (&vlib_global_main.elog_main,
65  (char *) vlib_worker_threads[0].barrier_caller);
66  ed->t_entry = (int) (1000000.0 * t_entry);
67  ed->t_open = (int) (1000000.0 * t_open);
68  ed->t_closed = (int) (1000000.0 * t_closed);
69 }
70 
71 static inline void
73 {
74  if (!vlib_worker_threads->barrier_elog_enabled)
75  return;
76 
77  /* *INDENT-OFF* */
78  ELOG_TYPE_DECLARE (e) =
79  {
80  .format = "bar-syncrec-%s-#%d",
81  .format_args = "T4i4",
82  };
83  /* *INDENT-ON* */
84  struct
85  {
86  u32 caller, depth;
87  } *ed = 0;
88 
90  ed->depth = (int) vlib_worker_threads[0].recursion_level - 1;
91  ed->caller = elog_string (&vlib_global_main.elog_main,
92  (char *) vlib_worker_threads[0].barrier_caller);
93 }
94 
95 static inline void
97 {
98  if (!vlib_worker_threads->barrier_elog_enabled)
99  return;
100 
101  /* *INDENT-OFF* */
102  ELOG_TYPE_DECLARE (e) =
103  {
104  .format = "bar-relrrec-#%d",
105  .format_args = "i4",
106  };
107  /* *INDENT-ON* */
108  struct
109  {
110  u32 depth;
111  } *ed = 0;
112 
114  ed->depth = (int) vlib_worker_threads[0].recursion_level;
115 }
116 
117 static inline void
118 barrier_trace_release (f64 t_entry, f64 t_closed_total, f64 t_update_main)
119 {
120  if (!vlib_worker_threads->barrier_elog_enabled)
121  return;
122 
123  /* *INDENT-OFF* */
124  ELOG_TYPE_DECLARE (e) =
125  {
126  .format = "bar-rel-#%d-e%d-u%d-t%d",
127  .format_args = "i4i4i4i4",
128  };
129  /* *INDENT-ON* */
130  struct
131  {
132  u32 count, t_entry, t_update_main, t_closed_total;
133  } *ed = 0;
134 
136  ed->t_entry = (int) (1000000.0 * t_entry);
137  ed->t_update_main = (int) (1000000.0 * t_update_main);
138  ed->t_closed_total = (int) (1000000.0 * t_closed_total);
139  ed->count = (int) vlib_worker_threads[0].barrier_sync_count;
140 
141  /* Reset context for next trace */
142  vlib_worker_threads[0].barrier_context = NULL;
143 }
144 
145 uword
147 {
148  return vec_len (vlib_thread_stacks);
149 }
150 
151 void
153 {
154  int pthread_setname_np (pthread_t __target_thread, const char *__name);
155  int rv;
156  pthread_t thread = pthread_self ();
157 
158  if (thread)
159  {
160  rv = pthread_setname_np (thread, name);
161  if (rv)
162  clib_warning ("pthread_setname_np returned %d", rv);
163  }
164 }
165 
166 static int
167 sort_registrations_by_no_clone (void *a0, void *a1)
168 {
169  vlib_thread_registration_t **tr0 = a0;
170  vlib_thread_registration_t **tr1 = a1;
171 
172  return ((i32) ((*tr0)->no_data_structure_clone)
173  - ((i32) ((*tr1)->no_data_structure_clone)));
174 }
175 
176 static uword *
178 {
179  FILE *fp;
180  uword *r = 0;
181 
182  fp = fopen (filename, "r");
183 
184  if (fp != NULL)
185  {
186  u8 *buffer = 0;
187  vec_validate (buffer, 256 - 1);
188  if (fgets ((char *) buffer, 256, fp))
189  {
190  unformat_input_t in;
191  unformat_init_string (&in, (char *) buffer,
192  strlen ((char *) buffer));
193  if (unformat (&in, "%U", unformat_bitmap_list, &r) != 1)
194  clib_warning ("unformat_bitmap_list failed");
195  unformat_free (&in);
196  }
197  vec_free (buffer);
198  fclose (fp);
199  }
200  return r;
201 }
202 
203 
204 /* Called early in the init sequence */
205 
206 clib_error_t *
208 {
212  u32 n_vlib_mains = 1;
213  u32 first_index = 1;
214  u32 i;
215  uword *avail_cpu;
216 
217  /* get bitmaps of active cpu cores and sockets */
218  tm->cpu_core_bitmap =
219  clib_sysfs_list_to_bitmap ("/sys/devices/system/cpu/online");
220  tm->cpu_socket_bitmap =
221  clib_sysfs_list_to_bitmap ("/sys/devices/system/node/online");
222 
223  avail_cpu = clib_bitmap_dup (tm->cpu_core_bitmap);
224 
225  /* skip cores */
226  for (i = 0; i < tm->skip_cores; i++)
227  {
228  uword c = clib_bitmap_first_set (avail_cpu);
229  if (c == ~0)
230  return clib_error_return (0, "no available cpus to skip");
231 
232  avail_cpu = clib_bitmap_set (avail_cpu, c, 0);
233  }
234 
235  /* grab cpu for main thread */
236  if (tm->main_lcore == ~0)
237  {
238  /* if main-lcore is not set, we try to use lcore 1 */
239  if (clib_bitmap_get (avail_cpu, 1))
240  tm->main_lcore = 1;
241  else
242  tm->main_lcore = clib_bitmap_first_set (avail_cpu);
243  if (tm->main_lcore == (u8) ~ 0)
244  return clib_error_return (0, "no available cpus to be used for the"
245  " main thread");
246  }
247  else
248  {
249  if (clib_bitmap_get (avail_cpu, tm->main_lcore) == 0)
250  return clib_error_return (0, "cpu %u is not available to be used"
251  " for the main thread", tm->main_lcore);
252  }
253  avail_cpu = clib_bitmap_set (avail_cpu, tm->main_lcore, 0);
254 
255  /* assume that there is socket 0 only if there is no data from sysfs */
256  if (!tm->cpu_socket_bitmap)
257  tm->cpu_socket_bitmap = clib_bitmap_set (0, 0, 1);
258 
259  /* pin main thread to main_lcore */
261  {
263  }
264  else
265  {
266  cpu_set_t cpuset;
267  CPU_ZERO (&cpuset);
268  CPU_SET (tm->main_lcore, &cpuset);
269  pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset);
270  }
271 
272  /* Set up thread 0 */
273  vec_validate_aligned (vlib_worker_threads, 0, CLIB_CACHE_LINE_BYTES);
274  _vec_len (vlib_worker_threads) = 1;
278  w->cpu_id = tm->main_lcore;
279  w->lwp = syscall (SYS_gettid);
280  w->thread_id = pthread_self ();
281  tm->n_vlib_mains = 1;
282 
284 
285  if (tm->sched_policy != ~0)
286  {
287  struct sched_param sched_param;
288  if (!sched_getparam (w->lwp, &sched_param))
289  {
290  if (tm->sched_priority != ~0)
291  sched_param.sched_priority = tm->sched_priority;
292  sched_setscheduler (w->lwp, tm->sched_policy, &sched_param);
293  }
294  }
295 
296  /* assign threads to cores and set n_vlib_mains */
297  tr = tm->next;
298 
299  while (tr)
300  {
301  vec_add1 (tm->registrations, tr);
302  tr = tr->next;
303  }
304 
306 
307  for (i = 0; i < vec_len (tm->registrations); i++)
308  {
309  int j;
310  tr = tm->registrations[i];
311  tr->first_index = first_index;
312  first_index += tr->count;
313  n_vlib_mains += (tr->no_data_structure_clone == 0) ? tr->count : 0;
314 
315  /* construct coremask */
316  if (tr->use_pthreads || !tr->count)
317  continue;
318 
319  if (tr->coremask)
320  {
321  uword c;
322  /* *INDENT-OFF* */
323  clib_bitmap_foreach (c, tr->coremask, ({
324  if (clib_bitmap_get(avail_cpu, c) == 0)
325  return clib_error_return (0, "cpu %u is not available to be used"
326  " for the '%s' thread",c, tr->name);
327 
328  avail_cpu = clib_bitmap_set(avail_cpu, c, 0);
329  }));
330  /* *INDENT-ON* */
331  }
332  else
333  {
334  for (j = 0; j < tr->count; j++)
335  {
336  /* Do not use CPU 0 by default - leave it to the host and IRQs */
337  uword avail_c0 = clib_bitmap_get (avail_cpu, 0);
338  avail_cpu = clib_bitmap_set (avail_cpu, 0, 0);
339 
340  uword c = clib_bitmap_first_set (avail_cpu);
341  /* Use CPU 0 as a last resort */
342  if (c == ~0 && avail_c0)
343  {
344  c = 0;
345  avail_c0 = 0;
346  }
347 
348  if (c == ~0)
349  return clib_error_return (0,
350  "no available cpus to be used for"
351  " the '%s' thread", tr->name);
352 
353  avail_cpu = clib_bitmap_set (avail_cpu, 0, avail_c0);
354  avail_cpu = clib_bitmap_set (avail_cpu, c, 0);
355  tr->coremask = clib_bitmap_set (tr->coremask, c, 1);
356  }
357  }
358  }
359 
360  clib_bitmap_free (avail_cpu);
361 
362  tm->n_vlib_mains = n_vlib_mains;
363 
364  /*
365  * Allocate the remaining worker threads, and thread stack vector slots
366  * from now on, calls to os_get_nthreads() will return the correct
367  * answer.
368  */
369  vec_validate_aligned (vlib_worker_threads, first_index - 1,
371  vec_validate (vlib_thread_stacks, vec_len (vlib_worker_threads) - 1);
372  return 0;
373 }
374 
377 {
378  vlib_frame_queue_t *fq;
379 
380  fq = clib_mem_alloc_aligned (sizeof (*fq), CLIB_CACHE_LINE_BYTES);
381  clib_memset (fq, 0, sizeof (*fq));
382  fq->nelts = nelts;
383  fq->vector_threshold = 128; // packets
385 
386  if (1)
387  {
388  if (((uword) & fq->tail) & (CLIB_CACHE_LINE_BYTES - 1))
389  fformat (stderr, "WARNING: fq->tail unaligned\n");
390  if (((uword) & fq->head) & (CLIB_CACHE_LINE_BYTES - 1))
391  fformat (stderr, "WARNING: fq->head unaligned\n");
392  if (((uword) fq->elts) & (CLIB_CACHE_LINE_BYTES - 1))
393  fformat (stderr, "WARNING: fq->elts unaligned\n");
394 
395  if (sizeof (fq->elts[0]) % CLIB_CACHE_LINE_BYTES)
396  fformat (stderr, "WARNING: fq->elts[0] size %d\n",
397  sizeof (fq->elts[0]));
398  if (nelts & (nelts - 1))
399  {
400  fformat (stderr, "FATAL: nelts MUST be a power of 2\n");
401  abort ();
402  }
403  }
404 
405  return (fq);
406 }
407 
408 void vl_msg_api_handler_no_free (void *) __attribute__ ((weak));
409 void
411 {
412 }
413 
414 /* Turned off, save as reference material... */
415 #if 0
416 static inline int
417 vlib_frame_queue_dequeue_internal (int thread_id,
418  vlib_main_t * vm, vlib_node_main_t * nm)
419 {
420  vlib_frame_queue_t *fq = vlib_frame_queues[thread_id];
422  vlib_frame_t *f;
425  u32 node_runtime_index;
426  int msg_type;
427  u64 before;
428  int processed = 0;
429 
430  ASSERT (vm == vlib_mains[thread_id]);
431 
432  while (1)
433  {
434  if (fq->head == fq->tail)
435  return processed;
436 
437  elt = fq->elts + ((fq->head + 1) & (fq->nelts - 1));
438 
439  if (!elt->valid)
440  return processed;
441 
442  before = clib_cpu_time_now ();
443 
444  f = elt->frame;
445  node_runtime_index = elt->node_runtime_index;
446  msg_type = elt->msg_type;
447 
448  switch (msg_type)
449  {
450  case VLIB_FRAME_QUEUE_ELT_FREE_BUFFERS:
452  /* note fallthrough... */
453  case VLIB_FRAME_QUEUE_ELT_FREE_FRAME:
455  node_runtime_index);
456  vlib_frame_free (vm, r, f);
457  break;
459  vec_add2 (vm->node_main.pending_frames, p, 1);
461  p->node_runtime_index = elt->node_runtime_index;
462  p->frame_index = vlib_frame_index (vm, f);
464  fq->dequeue_vectors += (u64) f->n_vectors;
465  break;
466  case VLIB_FRAME_QUEUE_ELT_API_MSG:
468  break;
469  default:
470  clib_warning ("bogus frame queue message, type %d", msg_type);
471  break;
472  }
473  elt->valid = 0;
474  fq->dequeues++;
475  fq->dequeue_ticks += clib_cpu_time_now () - before;
477  fq->head++;
478  processed++;
479  }
480  ASSERT (0);
481  return processed;
482 }
483 
484 int
485 vlib_frame_queue_dequeue (int thread_id,
486  vlib_main_t * vm, vlib_node_main_t * nm)
487 {
488  return vlib_frame_queue_dequeue_internal (thread_id, vm, nm);
489 }
490 
491 int
492 vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index,
493  u32 frame_queue_index, vlib_frame_t * frame,
495 {
496  vlib_frame_queue_t *fq = vlib_frame_queues[frame_queue_index];
498  u32 save_count;
499  u64 new_tail;
500  u64 before = clib_cpu_time_now ();
501 
502  ASSERT (fq);
503 
504  new_tail = clib_atomic_add_fetch (&fq->tail, 1);
505 
506  /* Wait until a ring slot is available */
507  while (new_tail >= fq->head + fq->nelts)
508  {
509  f64 b4 = vlib_time_now_ticks (vm, before);
511  /* Bad idea. Dequeue -> enqueue -> dequeue -> trouble */
512  // vlib_frame_queue_dequeue (vm->thread_index, vm, nm);
513  }
514 
515  elt = fq->elts + (new_tail & (fq->nelts - 1));
516 
517  /* this would be very bad... */
518  while (elt->valid)
519  {
520  }
521 
522  /* Once we enqueue the frame, frame->n_vectors is owned elsewhere... */
523  save_count = frame->n_vectors;
524 
525  elt->frame = frame;
526  elt->node_runtime_index = node_runtime_index;
527  elt->msg_type = type;
529  elt->valid = 1;
530 
531  return save_count;
532 }
533 #endif /* 0 */
534 
535 /* To be called by vlib worker threads upon startup */
536 void
538 {
540 
541  /*
542  * Note: disabling signals in worker threads as follows
543  * prevents the api post-mortem dump scheme from working
544  * {
545  * sigset_t s;
546  * sigfillset (&s);
547  * pthread_sigmask (SIG_SETMASK, &s, 0);
548  * }
549  */
550 
552 
553  if (vec_len (tm->thread_prefix) && w->registration->short_name)
554  {
555  w->name = format (0, "%v_%s_%d%c", tm->thread_prefix,
556  w->registration->short_name, w->instance_id, '\0');
557  vlib_set_thread_name ((char *) w->name);
558  }
559 
560  if (!w->registration->use_pthreads)
561  {
562 
563  /* Initial barrier sync, for both worker and i/o threads */
564  clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, 1);
565 
566  while (*vlib_worker_threads->wait_at_barrier)
567  ;
568 
569  clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, -1);
570  }
571 }
572 
573 void *
575 {
576  void *rv;
577  vlib_worker_thread_t *w = arg;
578 
579  w->lwp = syscall (SYS_gettid);
580  w->thread_id = pthread_self ();
581 
582  __os_thread_index = w - vlib_worker_threads;
583 
584  rv = (void *) clib_calljmp
585  ((uword (*)(uword)) w->thread_function,
587  /* NOTREACHED, we hope */
588  return rv;
589 }
590 
591 void
593 {
594  const char *sys_cpu_path = "/sys/devices/system/cpu/cpu";
595  const char *sys_node_path = "/sys/devices/system/node/node";
596  clib_bitmap_t *nbmp = 0, *cbmp = 0;
597  u32 node;
598  u8 *p = 0;
599  int core_id = -1, numa_id = -1;
600 
601  p = format (p, "%s%u/topology/core_id%c", sys_cpu_path, cpu_id, 0);
602  clib_sysfs_read ((char *) p, "%d", &core_id);
603  vec_reset_length (p);
604 
605  /* *INDENT-OFF* */
606  clib_sysfs_read ("/sys/devices/system/node/online", "%U",
607  unformat_bitmap_list, &nbmp);
608  clib_bitmap_foreach (node, nbmp, ({
609  p = format (p, "%s%u/cpulist%c", sys_node_path, node, 0);
610  clib_sysfs_read ((char *) p, "%U", unformat_bitmap_list, &cbmp);
611  if (clib_bitmap_get (cbmp, cpu_id))
612  numa_id = node;
613  vec_reset_length (cbmp);
614  vec_reset_length (p);
615  }));
616  /* *INDENT-ON* */
617  vec_free (nbmp);
618  vec_free (cbmp);
619  vec_free (p);
620 
621  w->core_id = core_id;
622  w->numa_id = numa_id;
623 }
624 
625 static clib_error_t *
627 {
629  void *(*fp_arg) (void *) = fp;
630  void *numa_heap;
631 
632  w->cpu_id = cpu_id;
633  vlib_get_thread_core_numa (w, cpu_id);
634 
635  /* Set up NUMA-bound heap if indicated */
636  if (clib_per_numa_mheaps[w->numa_id] == 0)
637  {
638  /* If the user requested a NUMA heap, create it... */
639  if (tm->numa_heap_size)
640  {
642  (0 /* DIY */ , tm->numa_heap_size, w->numa_id);
643  clib_per_numa_mheaps[w->numa_id] = numa_heap;
644  }
645  else
646  {
647  /* Or, use the main heap */
649  }
650  }
651 
653  return tm->cb.vlib_launch_thread_cb (fp, (void *) w, cpu_id);
654  else
655  {
656  pthread_t worker;
657  cpu_set_t cpuset;
658  CPU_ZERO (&cpuset);
659  CPU_SET (cpu_id, &cpuset);
660 
661  if (pthread_create (&worker, NULL /* attr */ , fp_arg, (void *) w))
662  return clib_error_return_unix (0, "pthread_create");
663 
664  if (pthread_setaffinity_np (worker, sizeof (cpu_set_t), &cpuset))
665  return clib_error_return_unix (0, "pthread_setaffinity_np");
666 
667  return 0;
668  }
669 }
670 
671 static clib_error_t *
673 {
674  int i, j;
676  vlib_main_t *vm_clone;
677  void *oldheap;
681  u32 n_vlib_mains = tm->n_vlib_mains;
682  u32 worker_thread_index;
683  u8 *main_heap = clib_mem_get_per_cpu_heap ();
684 
685  vec_reset_length (vlib_worker_threads);
686 
687  /* Set up the main thread */
688  vec_add2_aligned (vlib_worker_threads, w, 1, CLIB_CACHE_LINE_BYTES);
689  w->elog_track.name = "main thread";
691 
692  if (vec_len (tm->thread_prefix))
693  {
694  w->name = format (0, "%v_main%c", tm->thread_prefix, '\0');
695  vlib_set_thread_name ((char *) w->name);
696  }
697 
698  vm->elog_main.lock =
700  vm->elog_main.lock[0] = 0;
701 
702  if (n_vlib_mains > 1)
703  {
704  /* Replace hand-crafted length-1 vector with a real vector */
705  vlib_mains = 0;
706 
709  _vec_len (vlib_mains) = 0;
711 
712  vlib_worker_threads->wait_at_barrier =
714  vlib_worker_threads->workers_at_barrier =
716 
717  vlib_worker_threads->node_reforks_required =
719 
720  /* We'll need the rpc vector lock... */
722 
723  /* Ask for an initial barrier sync */
724  *vlib_worker_threads->workers_at_barrier = 0;
725  *vlib_worker_threads->wait_at_barrier = 1;
726 
727  /* Without update or refork */
728  *vlib_worker_threads->node_reforks_required = 0;
730 
731  /* init timing */
732  vm->barrier_epoch = 0;
733  vm->barrier_no_close_before = 0;
734 
735  worker_thread_index = 1;
736 
737  for (i = 0; i < vec_len (tm->registrations); i++)
738  {
739  vlib_node_main_t *nm, *nm_clone;
740  int k;
741 
742  tr = tm->registrations[i];
743 
744  if (tr->count == 0)
745  continue;
746 
747  for (k = 0; k < tr->count; k++)
748  {
749  vlib_node_t *n;
750 
751  vec_add2 (vlib_worker_threads, w, 1);
752  /* Currently unused, may not really work */
753  if (tr->mheap_size)
755  0 /* unlocked */ );
756  else
757  w->thread_mheap = main_heap;
758 
759  w->thread_stack =
760  vlib_thread_stack_init (w - vlib_worker_threads);
761  w->thread_function = tr->function;
762  w->thread_function_arg = w;
763  w->instance_id = k;
764  w->registration = tr;
765 
766  w->elog_track.name =
767  (char *) format (0, "%s %d", tr->name, k + 1);
768  vec_add1 (w->elog_track.name, 0);
770 
771  if (tr->no_data_structure_clone)
772  continue;
773 
774  /* Fork vlib_global_main et al. Look for bugs here */
775  oldheap = clib_mem_set_heap (w->thread_mheap);
776 
777  vm_clone = clib_mem_alloc_aligned (sizeof (*vm_clone),
779  clib_memcpy (vm_clone, vlib_mains[0], sizeof (*vm_clone));
780 
781  vm_clone->thread_index = worker_thread_index;
782  vm_clone->heap_base = w->thread_mheap;
783  vm_clone->heap_aligned_base = (void *)
784  (((uword) w->thread_mheap) & ~(VLIB_FRAME_ALIGN - 1));
785  vm_clone->init_functions_called =
786  hash_create (0, /* value bytes */ 0);
787  vm_clone->pending_rpc_requests = 0;
788  vec_validate (vm_clone->pending_rpc_requests, 0);
789  _vec_len (vm_clone->pending_rpc_requests) = 0;
790  clib_memset (&vm_clone->random_buffer, 0,
791  sizeof (vm_clone->random_buffer));
792 
793  nm = &vlib_mains[0]->node_main;
794  nm_clone = &vm_clone->node_main;
795  /* fork next frames array, preserving node runtime indices */
796  nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
798  for (j = 0; j < vec_len (nm_clone->next_frames); j++)
799  {
800  vlib_next_frame_t *nf = &nm_clone->next_frames[j];
801  u32 save_node_runtime_index;
802  u32 save_flags;
803 
804  save_node_runtime_index = nf->node_runtime_index;
805  save_flags = nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH;
807  nf->node_runtime_index = save_node_runtime_index;
808  nf->flags = save_flags;
809  }
810 
811  /* fork the frame dispatch queue */
812  nm_clone->pending_frames = 0;
813  vec_validate (nm_clone->pending_frames, 10);
814  _vec_len (nm_clone->pending_frames) = 0;
815 
816  /* fork nodes */
817  nm_clone->nodes = 0;
818 
819  /* Allocate all nodes in single block for speed */
820  n = clib_mem_alloc_no_fail (vec_len (nm->nodes) * sizeof (*n));
821 
822  for (j = 0; j < vec_len (nm->nodes); j++)
823  {
824  clib_memcpy (n, nm->nodes[j], sizeof (*n));
825  /* none of the copied nodes have enqueue rights given out */
827  clib_memset (&n->stats_total, 0, sizeof (n->stats_total));
829  sizeof (n->stats_last_clear));
830  vec_add1 (nm_clone->nodes, n);
831  n++;
832  }
836  vec_foreach (rt,
838  {
839  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
840  rt->thread_index = vm_clone->thread_index;
841  /* copy initial runtime_data from node */
842  if (n->runtime_data && n->runtime_data_bytes > 0)
845  n->runtime_data_bytes));
846  }
847 
852  {
853  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
854  rt->thread_index = vm_clone->thread_index;
855  /* copy initial runtime_data from node */
856  if (n->runtime_data && n->runtime_data_bytes > 0)
859  n->runtime_data_bytes));
860  }
861 
865  vec_foreach (rt,
867  {
868  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
869  rt->thread_index = vm_clone->thread_index;
870  /* copy initial runtime_data from node */
871  if (n->runtime_data && n->runtime_data_bytes > 0)
874  n->runtime_data_bytes));
875  }
876 
877  nm_clone->processes = vec_dup_aligned (nm->processes,
879 
880  /* Create per-thread frame freelist */
881  nm_clone->frame_sizes = vec_new (vlib_frame_size_t, 1);
882 #ifdef VLIB_SUPPORTS_ARBITRARY_SCALAR_SIZES
883  nm_clone->frame_size_hash = hash_create (0, sizeof (uword));
884 #endif
885  nm_clone->node_by_error = nm->node_by_error;
886 
887  /* Packet trace buffers are guaranteed to be empty, nothing to do here */
888 
889  clib_mem_set_heap (oldheap);
891 
892  /* Switch to the stats segment ... */
893  void *oldheap = vlib_stats_push_heap (0);
895  (vlib_mains[0]->error_main.counters, CLIB_CACHE_LINE_BYTES);
897  worker_thread_index, oldheap, 1);
898 
900  (vlib_mains[0]->error_main.counters_last_clear,
902 
903  worker_thread_index++;
904  }
905  }
906  }
907  else
908  {
909  /* only have non-data-structure copy threads to create... */
910  for (i = 0; i < vec_len (tm->registrations); i++)
911  {
912  tr = tm->registrations[i];
913 
914  for (j = 0; j < tr->count; j++)
915  {
916  vec_add2 (vlib_worker_threads, w, 1);
917  if (tr->mheap_size)
918  {
919  w->thread_mheap =
920  create_mspace (tr->mheap_size, 0 /* locked */ );
921  }
922  else
923  w->thread_mheap = main_heap;
924  w->thread_stack =
925  vlib_thread_stack_init (w - vlib_worker_threads);
926  w->thread_function = tr->function;
927  w->thread_function_arg = w;
928  w->instance_id = j;
929  w->elog_track.name =
930  (char *) format (0, "%s %d", tr->name, j + 1);
931  w->registration = tr;
932  vec_add1 (w->elog_track.name, 0);
934  }
935  }
936  }
937 
938  worker_thread_index = 1;
939 
940  for (i = 0; i < vec_len (tm->registrations); i++)
941  {
942  clib_error_t *err;
943  int j;
944 
945  tr = tm->registrations[i];
946 
947  if (tr->use_pthreads || tm->use_pthreads)
948  {
949  for (j = 0; j < tr->count; j++)
950  {
951  w = vlib_worker_threads + worker_thread_index++;
953  w, 0);
954  if (err)
955  clib_error_report (err);
956  }
957  }
958  else
959  {
960  uword c;
961  /* *INDENT-OFF* */
962  clib_bitmap_foreach (c, tr->coremask, ({
963  w = vlib_worker_threads + worker_thread_index++;
964  err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn,
965  w, c);
966  if (err)
967  clib_error_report (err);
968  }));
969  /* *INDENT-ON* */
970  }
971  }
974  return 0;
975 }
976 
978 
979 
980 static inline void
982 {
983  int i, j;
984  vlib_main_t *vm;
985  vlib_node_main_t *nm, *nm_clone;
986  vlib_main_t *vm_clone;
988  never_inline void
991  uword n_calls,
992  uword n_vectors, uword n_clocks);
993 
994  ASSERT (vlib_get_thread_index () == 0);
995 
996  vm = vlib_mains[0];
997  nm = &vm->node_main;
998 
999  ASSERT (*vlib_worker_threads->wait_at_barrier == 1);
1000 
1001  /*
1002  * Scrape all runtime stats, so we don't lose node runtime(s) with
1003  * pending counts, or throw away worker / io thread counts.
1004  */
1005  for (j = 0; j < vec_len (nm->nodes); j++)
1006  {
1007  vlib_node_t *n;
1008  n = nm->nodes[j];
1009  vlib_node_sync_stats (vm, n);
1010  }
1011 
1012  for (i = 1; i < vec_len (vlib_mains); i++)
1013  {
1014  vlib_node_t *n;
1015 
1016  vm_clone = vlib_mains[i];
1017  nm_clone = &vm_clone->node_main;
1018 
1019  for (j = 0; j < vec_len (nm_clone->nodes); j++)
1020  {
1021  n = nm_clone->nodes[j];
1022 
1023  rt = vlib_node_get_runtime (vm_clone, n->index);
1024  vlib_node_runtime_sync_stats (vm_clone, rt, 0, 0, 0);
1025  }
1026  }
1027 
1028  /* Per-worker clone rebuilds are now done on each thread */
1029 }
1030 
1031 
1032 void
1034 {
1035  vlib_main_t *vm, *vm_clone;
1036  vlib_node_main_t *nm, *nm_clone;
1037  vlib_node_t **old_nodes_clone;
1038  vlib_node_runtime_t *rt, *old_rt;
1039 
1040  vlib_node_t *new_n_clone;
1041 
1042  int j;
1043 
1044  vm = vlib_mains[0];
1045  nm = &vm->node_main;
1046  vm_clone = vlib_get_main ();
1047  nm_clone = &vm_clone->node_main;
1048 
1049  /* Re-clone error heap */
1050  u64 *old_counters = vm_clone->error_main.counters;
1051  u64 *old_counters_all_clear = vm_clone->error_main.counters_last_clear;
1052 
1053  clib_memcpy_fast (&vm_clone->error_main, &vm->error_main,
1054  sizeof (vm->error_main));
1055  j = vec_len (vm->error_main.counters) - 1;
1056 
1057  /* Switch to the stats segment ... */
1058  void *oldheap = vlib_stats_push_heap (0);
1059  vec_validate_aligned (old_counters, j, CLIB_CACHE_LINE_BYTES);
1060  vm_clone->error_main.counters = old_counters;
1061  vlib_stats_pop_heap2 (vm_clone->error_main.counters, vm_clone->thread_index,
1062  oldheap, 0);
1063 
1064  vec_validate_aligned (old_counters_all_clear, j, CLIB_CACHE_LINE_BYTES);
1065  vm_clone->error_main.counters_last_clear = old_counters_all_clear;
1066 
1067  nm_clone = &vm_clone->node_main;
1068  vec_free (nm_clone->next_frames);
1069  nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
1071 
1072  for (j = 0; j < vec_len (nm_clone->next_frames); j++)
1073  {
1074  vlib_next_frame_t *nf = &nm_clone->next_frames[j];
1075  u32 save_node_runtime_index;
1076  u32 save_flags;
1077 
1078  save_node_runtime_index = nf->node_runtime_index;
1079  save_flags = nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH;
1080  vlib_next_frame_init (nf);
1081  nf->node_runtime_index = save_node_runtime_index;
1082  nf->flags = save_flags;
1083  }
1084 
1085  old_nodes_clone = nm_clone->nodes;
1086  nm_clone->nodes = 0;
1087 
1088  /* re-fork nodes */
1089 
1090  /* Allocate all nodes in single block for speed */
1091  new_n_clone =
1092  clib_mem_alloc_no_fail (vec_len (nm->nodes) * sizeof (*new_n_clone));
1093  for (j = 0; j < vec_len (nm->nodes); j++)
1094  {
1095  vlib_node_t *new_n = nm->nodes[j];
1096 
1097  clib_memcpy_fast (new_n_clone, new_n, sizeof (*new_n));
1098  /* none of the copied nodes have enqueue rights given out */
1100 
1101  if (j >= vec_len (old_nodes_clone))
1102  {
1103  /* new node, set to zero */
1104  clib_memset (&new_n_clone->stats_total, 0,
1105  sizeof (new_n_clone->stats_total));
1106  clib_memset (&new_n_clone->stats_last_clear, 0,
1107  sizeof (new_n_clone->stats_last_clear));
1108  }
1109  else
1110  {
1111  vlib_node_t *old_n_clone = old_nodes_clone[j];
1112  /* Copy stats if the old data is valid */
1113  clib_memcpy_fast (&new_n_clone->stats_total,
1114  &old_n_clone->stats_total,
1115  sizeof (new_n_clone->stats_total));
1116  clib_memcpy_fast (&new_n_clone->stats_last_clear,
1117  &old_n_clone->stats_last_clear,
1118  sizeof (new_n_clone->stats_last_clear));
1119 
1120  /* keep previous node state */
1121  new_n_clone->state = old_n_clone->state;
1122  }
1123  vec_add1 (nm_clone->nodes, new_n_clone);
1124  new_n_clone++;
1125  }
1126  /* Free the old node clones */
1127  clib_mem_free (old_nodes_clone[0]);
1128 
1129  vec_free (old_nodes_clone);
1130 
1131 
1132  /* re-clone internal nodes */
1133  old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL];
1137 
1139  {
1140  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
1141  rt->thread_index = vm_clone->thread_index;
1142  /* copy runtime_data, will be overwritten later for existing rt */
1143  if (n->runtime_data && n->runtime_data_bytes > 0)
1146  n->runtime_data_bytes));
1147  }
1148 
1149  for (j = 0; j < vec_len (old_rt); j++)
1150  {
1151  rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
1152  rt->state = old_rt[j].state;
1153  clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
1155  }
1156 
1157  vec_free (old_rt);
1158 
1159  /* re-clone input nodes */
1160  old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT];
1161  nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] =
1164 
1166  {
1167  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
1168  rt->thread_index = vm_clone->thread_index;
1169  /* copy runtime_data, will be overwritten later for existing rt */
1170  if (n->runtime_data && n->runtime_data_bytes > 0)
1173  n->runtime_data_bytes));
1174  }
1175 
1176  for (j = 0; j < vec_len (old_rt); j++)
1177  {
1178  rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
1179  rt->state = old_rt[j].state;
1180  clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
1182  }
1183 
1184  vec_free (old_rt);
1185 
1186  /* re-clone pre-input nodes */
1187  old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT];
1191 
1193  {
1194  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
1195  rt->thread_index = vm_clone->thread_index;
1196  /* copy runtime_data, will be overwritten later for existing rt */
1197  if (n->runtime_data && n->runtime_data_bytes > 0)
1200  n->runtime_data_bytes));
1201  }
1202 
1203  for (j = 0; j < vec_len (old_rt); j++)
1204  {
1205  rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
1206  rt->state = old_rt[j].state;
1207  clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
1209  }
1210 
1211  vec_free (old_rt);
1212 
1213  nm_clone->processes = vec_dup_aligned (nm->processes,
1215  nm_clone->node_by_error = nm->node_by_error;
1216 }
1217 
1218 void
1220 {
1221  /*
1222  * Make a note that we need to do a node runtime update
1223  * prior to releasing the barrier.
1224  */
1226 }
1227 
1228 u32
1229 unformat_sched_policy (unformat_input_t * input, va_list * args)
1230 {
1231  u32 *r = va_arg (*args, u32 *);
1232 
1233  if (0);
1234 #define _(v,f,s) else if (unformat (input, s)) *r = SCHED_POLICY_##f;
1236 #undef _
1237  else
1238  return 0;
1239  return 1;
1240 }
1241 
1242 static clib_error_t *
1244 {
1246  uword *p;
1248  u8 *name;
1249  uword *bitmap;
1250  u32 count;
1251 
1253 
1254  tm->n_thread_stacks = 1; /* account for main thread */
1255  tm->sched_policy = ~0;
1256  tm->sched_priority = ~0;
1257  tm->main_lcore = ~0;
1258 
1259  tr = tm->next;
1260 
1261  while (tr)
1262  {
1264  tr = tr->next;
1265  }
1266 
1267  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1268  {
1269  if (unformat (input, "use-pthreads"))
1270  tm->use_pthreads = 1;
1271  else if (unformat (input, "thread-prefix %v", &tm->thread_prefix))
1272  ;
1273  else if (unformat (input, "main-core %u", &tm->main_lcore))
1274  ;
1275  else if (unformat (input, "skip-cores %u", &tm->skip_cores))
1276  ;
1277  else if (unformat (input, "numa-heap-size %U",
1279  ;
1280  else if (unformat (input, "coremask-%s %U", &name,
1281  unformat_bitmap_mask, &bitmap) ||
1282  unformat (input, "corelist-%s %U", &name,
1283  unformat_bitmap_list, &bitmap))
1284  {
1286  if (p == 0)
1287  return clib_error_return (0, "no such thread type '%s'", name);
1288 
1289  tr = (vlib_thread_registration_t *) p[0];
1290 
1291  if (tr->use_pthreads)
1292  return clib_error_return (0,
1293  "corelist cannot be set for '%s' threads",
1294  name);
1295  if (tr->count)
1296  return clib_error_return
1297  (0, "core placement of '%s' threads is already configured",
1298  name);
1299 
1300  tr->coremask = bitmap;
1302  }
1303  else
1304  if (unformat
1305  (input, "scheduler-policy %U", unformat_sched_policy,
1306  &tm->sched_policy))
1307  ;
1308  else if (unformat (input, "scheduler-priority %u", &tm->sched_priority))
1309  ;
1310  else if (unformat (input, "%s %u", &name, &count))
1311  {
1313  if (p == 0)
1314  return clib_error_return (0, "no such thread type 3 '%s'", name);
1315 
1316  tr = (vlib_thread_registration_t *) p[0];
1317 
1318  if (tr->fixed_count)
1319  return clib_error_return
1320  (0, "number of '%s' threads not configurable", name);
1321  if (tr->count)
1322  return clib_error_return
1323  (0, "number of '%s' threads is already configured", name);
1324 
1325  tr->count = count;
1326  }
1327  else
1328  break;
1329  }
1330 
1331  if (tm->sched_priority != ~0)
1332  {
1333  if (tm->sched_policy == SCHED_FIFO || tm->sched_policy == SCHED_RR)
1334  {
1335  u32 prio_max = sched_get_priority_max (tm->sched_policy);
1336  u32 prio_min = sched_get_priority_min (tm->sched_policy);
1337  if (tm->sched_priority > prio_max)
1338  tm->sched_priority = prio_max;
1339  if (tm->sched_priority < prio_min)
1340  tm->sched_priority = prio_min;
1341  }
1342  else
1343  {
1344  return clib_error_return
1345  (0,
1346  "scheduling priority (%d) is not allowed for `normal` scheduling policy",
1347  tm->sched_priority);
1348  }
1349  }
1350  tr = tm->next;
1351 
1352  if (!tm->thread_prefix)
1353  tm->thread_prefix = format (0, "vpp");
1354 
1355  while (tr)
1356  {
1357  tm->n_thread_stacks += tr->count;
1358  tm->n_pthreads += tr->count * tr->use_pthreads;
1359  tm->n_threads += tr->count * (tr->use_pthreads == 0);
1360  tr = tr->next;
1361  }
1362 
1363  return 0;
1364 }
1365 
1367 
1368 void vnet_main_fixup (vlib_fork_fixup_t which) __attribute__ ((weak));
1369 void
1371 {
1372 }
1373 
1374 void
1376 {
1377  vlib_main_t *vm = vlib_get_main ();
1378 
1379  if (vlib_mains == 0)
1380  return;
1381 
1382  ASSERT (vlib_get_thread_index () == 0);
1384 
1385  switch (which)
1386  {
1389  break;
1390 
1391  default:
1392  ASSERT (0);
1393  }
1395 }
1396 
1397  /*
1398  * Enforce minimum open time to minimize packet loss due to Rx overflow,
1399  * based on a test based heuristic that barrier should be open for at least
1400  * 3 time as long as it is closed (with an upper bound of 1ms because by that
1401  * point it is probably too late to make a difference)
1402  */
1403 
1404 #ifndef BARRIER_MINIMUM_OPEN_LIMIT
1405 #define BARRIER_MINIMUM_OPEN_LIMIT 0.001
1406 #endif
1407 
1408 #ifndef BARRIER_MINIMUM_OPEN_FACTOR
1409 #define BARRIER_MINIMUM_OPEN_FACTOR 3
1410 #endif
1411 
1412 void
1414 {
1415  f64 deadline;
1416  f64 now = vlib_time_now (vm);
1417  u32 count = vec_len (vlib_mains) - 1;
1418 
1419  /* No worker threads? */
1420  if (count == 0)
1421  return;
1422 
1423  deadline = now + BARRIER_SYNC_TIMEOUT;
1424  *vlib_worker_threads->wait_at_barrier = 1;
1425  while (*vlib_worker_threads->workers_at_barrier != count)
1426  {
1427  if ((now = vlib_time_now (vm)) > deadline)
1428  {
1429  fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__);
1430  os_panic ();
1431  }
1432  CLIB_PAUSE ();
1433  }
1434  *vlib_worker_threads->wait_at_barrier = 0;
1435 }
1436 
1437 /**
1438  * Return true if the wroker thread barrier is held
1439  */
1440 u8
1442 {
1443  if (vec_len (vlib_mains) < 2)
1444  return (1);
1445 
1446  return (*vlib_worker_threads->wait_at_barrier == 1);
1447 }
1448 
1449 void
1451 {
1452  f64 deadline;
1453  f64 now;
1454  f64 t_entry;
1455  f64 t_open;
1456  f64 t_closed;
1457  f64 max_vector_rate;
1458  u32 count;
1459  int i;
1460 
1461  if (vec_len (vlib_mains) < 2)
1462  return;
1463 
1464  ASSERT (vlib_get_thread_index () == 0);
1465 
1466  vlib_worker_threads[0].barrier_caller = func_name;
1467  count = vec_len (vlib_mains) - 1;
1468 
1469  /* Record entry relative to last close */
1470  now = vlib_time_now (vm);
1471  t_entry = now - vm->barrier_epoch;
1472 
1473  /* Tolerate recursive calls */
1474  if (++vlib_worker_threads[0].recursion_level > 1)
1475  {
1476  barrier_trace_sync_rec (t_entry);
1477  return;
1478  }
1479 
1480  /*
1481  * Need data to decide if we're working hard enough to honor
1482  * the barrier hold-down timer.
1483  */
1484  max_vector_rate = 0.0;
1485  for (i = 1; i < vec_len (vlib_mains); i++)
1486  max_vector_rate =
1487  clib_max (max_vector_rate,
1489 
1490  vlib_worker_threads[0].barrier_sync_count++;
1491 
1492  /* Enforce minimum barrier open time to minimize packet loss */
1494 
1495  /*
1496  * If any worker thread seems busy, which we define
1497  * as a vector rate above 10, we enforce the barrier hold-down timer
1498  */
1499  if (max_vector_rate > 10.0)
1500  {
1501  while (1)
1502  {
1503  now = vlib_time_now (vm);
1504  /* Barrier hold-down timer expired? */
1505  if (now >= vm->barrier_no_close_before)
1506  break;
1507  if ((vm->barrier_no_close_before - now)
1508  > (2.0 * BARRIER_MINIMUM_OPEN_LIMIT))
1509  {
1510  clib_warning
1511  ("clock change: would have waited for %.4f seconds",
1512  (vm->barrier_no_close_before - now));
1513  break;
1514  }
1515  }
1516  }
1517  /* Record time of closure */
1518  t_open = now - vm->barrier_epoch;
1519  vm->barrier_epoch = now;
1520 
1521  deadline = now + BARRIER_SYNC_TIMEOUT;
1522 
1523  *vlib_worker_threads->wait_at_barrier = 1;
1524  while (*vlib_worker_threads->workers_at_barrier != count)
1525  {
1526  if ((now = vlib_time_now (vm)) > deadline)
1527  {
1528  fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__);
1529  os_panic ();
1530  }
1531  }
1532 
1533  t_closed = now - vm->barrier_epoch;
1534 
1535  barrier_trace_sync (t_entry, t_open, t_closed);
1536 
1537 }
1538 
1539 void
1541 {
1542  f64 deadline;
1543  f64 now;
1544  f64 minimum_open;
1545  f64 t_entry;
1546  f64 t_closed_total;
1547  f64 t_update_main = 0.0;
1548  int refork_needed = 0;
1549 
1550  if (vec_len (vlib_mains) < 2)
1551  return;
1552 
1553  ASSERT (vlib_get_thread_index () == 0);
1554 
1555 
1556  now = vlib_time_now (vm);
1557  t_entry = now - vm->barrier_epoch;
1558 
1559  if (--vlib_worker_threads[0].recursion_level > 0)
1560  {
1561  barrier_trace_release_rec (t_entry);
1562  return;
1563  }
1564 
1565  /* Update (all) node runtimes before releasing the barrier, if needed */
1567  {
1568  /*
1569  * Lock stat segment here, so we's safe when
1570  * rebuilding the stat segment node clones from the
1571  * stat thread...
1572  */
1574 
1575  /* Do stats elements on main thread */
1578 
1579  /* Do per thread rebuilds in parallel */
1580  refork_needed = 1;
1581  clib_atomic_fetch_add (vlib_worker_threads->node_reforks_required,
1582  (vec_len (vlib_mains) - 1));
1583  now = vlib_time_now (vm);
1584  t_update_main = now - vm->barrier_epoch;
1585  }
1586 
1587  deadline = now + BARRIER_SYNC_TIMEOUT;
1588 
1589  /*
1590  * Note when we let go of the barrier.
1591  * Workers can use this to derive a reasonably accurate
1592  * time offset. See vlib_time_now(...)
1593  */
1596 
1597  *vlib_worker_threads->wait_at_barrier = 0;
1598 
1599  while (*vlib_worker_threads->workers_at_barrier > 0)
1600  {
1601  if ((now = vlib_time_now (vm)) > deadline)
1602  {
1603  fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__);
1604  os_panic ();
1605  }
1606  }
1607 
1608  /* Wait for reforks before continuing */
1609  if (refork_needed)
1610  {
1611  now = vlib_time_now (vm);
1612 
1613  deadline = now + BARRIER_SYNC_TIMEOUT;
1614 
1615  while (*vlib_worker_threads->node_reforks_required > 0)
1616  {
1617  if ((now = vlib_time_now (vm)) > deadline)
1618  {
1619  fformat (stderr, "%s: worker thread refork deadlock\n",
1620  __FUNCTION__);
1621  os_panic ();
1622  }
1623  }
1625  }
1626 
1627  t_closed_total = now - vm->barrier_epoch;
1628 
1629  minimum_open = t_closed_total * BARRIER_MINIMUM_OPEN_FACTOR;
1630 
1631  if (minimum_open > BARRIER_MINIMUM_OPEN_LIMIT)
1632  {
1633  minimum_open = BARRIER_MINIMUM_OPEN_LIMIT;
1634  }
1635 
1636  vm->barrier_no_close_before = now + minimum_open;
1637 
1638  /* Record barrier epoch (used to enforce minimum open time) */
1639  vm->barrier_epoch = now;
1640 
1641  barrier_trace_release (t_entry, t_closed_total, t_update_main);
1642 
1643 }
1644 
1645 /**
1646  * Wait until each of the workers has been once around the track
1647  */
1648 void
1650 {
1651  ASSERT (vlib_get_thread_index () == 0);
1652 
1653  if (vec_len (vlib_mains) < 2)
1654  return;
1655 
1657  return;
1658 
1659  u32 *counts = 0;
1660  u32 ii;
1661 
1662  vec_validate (counts, vec_len (vlib_mains) - 1);
1663 
1664  /* record the current loop counts */
1666  counts[ii] = vlib_mains[ii]->main_loop_count;
1667 
1668  /* spin until each changes, apart from the main thread, or we'd be
1669  * a while */
1670  for (ii = 1; ii < vec_len (counts); ii++)
1671  {
1672  while (counts[ii] == vlib_mains[ii]->main_loop_count)
1673  CLIB_PAUSE ();
1674  }
1675 
1676  vec_free (counts);
1677  return;
1678 }
1679 
1680 /*
1681  * Check the frame queue to see if any frames are available.
1682  * If so, pull the packets off the frames and put them to
1683  * the handoff node.
1684  */
1685 int
1687 {
1688  u32 thread_id = vm->thread_index;
1689  vlib_frame_queue_t *fq = fqm->vlib_frame_queues[thread_id];
1691  u32 *from, *to;
1692  vlib_frame_t *f;
1693  int msg_type;
1694  int processed = 0;
1695  u32 n_left_to_node;
1696  u32 vectors = 0;
1697 
1698  ASSERT (fq);
1699  ASSERT (vm == vlib_mains[thread_id]);
1700 
1701  if (PREDICT_FALSE (fqm->node_index == ~0))
1702  return 0;
1703  /*
1704  * Gather trace data for frame queues
1705  */
1706  if (PREDICT_FALSE (fq->trace))
1707  {
1708  frame_queue_trace_t *fqt;
1710  u32 elix;
1711 
1712  fqt = &fqm->frame_queue_traces[thread_id];
1713 
1714  fqt->nelts = fq->nelts;
1715  fqt->head = fq->head;
1716  fqt->head_hint = fq->head_hint;
1717  fqt->tail = fq->tail;
1718  fqt->threshold = fq->vector_threshold;
1719  fqt->n_in_use = fqt->tail - fqt->head;
1720  if (fqt->n_in_use >= fqt->nelts)
1721  {
1722  // if beyond max then use max
1723  fqt->n_in_use = fqt->nelts - 1;
1724  }
1725 
1726  /* Record the number of elements in use in the histogram */
1727  fqh = &fqm->frame_queue_histogram[thread_id];
1728  fqh->count[fqt->n_in_use]++;
1729 
1730  /* Record a snapshot of the elements in use */
1731  for (elix = 0; elix < fqt->nelts; elix++)
1732  {
1733  elt = fq->elts + ((fq->head + 1 + elix) & (fq->nelts - 1));
1734  if (1 || elt->valid)
1735  {
1736  fqt->n_vectors[elix] = elt->n_vectors;
1737  }
1738  }
1739  fqt->written = 1;
1740  }
1741 
1742  while (1)
1743  {
1744  vlib_buffer_t *b;
1745  if (fq->head == fq->tail)
1746  {
1747  fq->head_hint = fq->head;
1748  return processed;
1749  }
1750 
1751  elt = fq->elts + ((fq->head + 1) & (fq->nelts - 1));
1752 
1753  if (!elt->valid)
1754  {
1755  fq->head_hint = fq->head;
1756  return processed;
1757  }
1758 
1759  from = elt->buffer_index;
1760  msg_type = elt->msg_type;
1761 
1763  ASSERT (elt->n_vectors <= VLIB_FRAME_SIZE);
1764 
1765  f = vlib_get_frame_to_node (vm, fqm->node_index);
1766 
1767  /* If the first vector is traced, set the frame trace flag */
1768  b = vlib_get_buffer (vm, from[0]);
1769  if (b->flags & VLIB_BUFFER_IS_TRACED)
1771 
1772  to = vlib_frame_vector_args (f);
1773 
1774  n_left_to_node = elt->n_vectors;
1775 
1776  while (n_left_to_node >= 4)
1777  {
1778  to[0] = from[0];
1779  to[1] = from[1];
1780  to[2] = from[2];
1781  to[3] = from[3];
1782  to += 4;
1783  from += 4;
1784  n_left_to_node -= 4;
1785  }
1786 
1787  while (n_left_to_node > 0)
1788  {
1789  to[0] = from[0];
1790  to++;
1791  from++;
1792  n_left_to_node--;
1793  }
1794 
1795  vectors += elt->n_vectors;
1796  f->n_vectors = elt->n_vectors;
1797  vlib_put_frame_to_node (vm, fqm->node_index, f);
1798 
1799  elt->valid = 0;
1800  elt->n_vectors = 0;
1801  elt->msg_type = 0xfefefefe;
1803  fq->head++;
1804  processed++;
1805 
1806  /*
1807  * Limit the number of packets pushed into the graph
1808  */
1809  if (vectors >= fq->vector_threshold)
1810  {
1811  fq->head_hint = fq->head;
1812  return processed;
1813  }
1814  }
1815  ASSERT (0);
1816  return processed;
1817 }
1818 
1819 void
1821 {
1824  vlib_main_t *vm = vlib_get_main ();
1825  clib_error_t *e;
1826 
1828 
1830  clib_time_init (&vm->clib_time);
1832 
1834  (vm, &vm->worker_init_function_registrations, 1 /* call_once */ );
1835  if (e)
1836  clib_error_report (e);
1837 
1838  /* Wait until the dpdk init sequence is complete */
1839  while (tm->extern_thread_mgmt && tm->worker_thread_release == 0)
1841 
1842  vlib_worker_loop (vm);
1843 }
1844 
1845 /* *INDENT-OFF* */
1846 VLIB_REGISTER_THREAD (worker_thread_reg, static) = {
1847  .name = "workers",
1848  .short_name = "wk",
1849  .function = vlib_worker_thread_fn,
1850 };
1851 /* *INDENT-ON* */
1852 
1853 u32
1854 vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts)
1855 {
1858  vlib_frame_queue_t *fq;
1859  int i;
1860  u32 num_threads;
1861 
1862  if (frame_queue_nelts == 0)
1863  frame_queue_nelts = FRAME_QUEUE_MAX_NELTS;
1864 
1865  num_threads = 1 /* main thread */ + tm->n_threads;
1866  ASSERT (frame_queue_nelts >= 8 + num_threads);
1867 
1868  vec_add2 (tm->frame_queue_mains, fqm, 1);
1869 
1870  fqm->node_index = node_index;
1871  fqm->frame_queue_nelts = frame_queue_nelts;
1872  fqm->queue_hi_thresh = frame_queue_nelts - num_threads;
1873 
1875  vec_validate (fqm->per_thread_data, tm->n_vlib_mains - 1);
1876  _vec_len (fqm->vlib_frame_queues) = 0;
1877  for (i = 0; i < tm->n_vlib_mains; i++)
1878  {
1880  fq = vlib_frame_queue_alloc (frame_queue_nelts);
1881  vec_add1 (fqm->vlib_frame_queues, fq);
1882 
1883  ptd = vec_elt_at_index (fqm->per_thread_data, i);
1885  tm->n_vlib_mains - 1);
1887  tm->n_vlib_mains - 1,
1888  (vlib_frame_queue_t *) (~0));
1889  }
1890 
1891  return (fqm - tm->frame_queue_mains);
1892 }
1893 
1894 int
1896 {
1898 
1899  if (tm->extern_thread_mgmt)
1900  return -1;
1901 
1903  tm->extern_thread_mgmt = 1;
1904  return 0;
1905 }
1906 
1907 void
1909  args)
1910 {
1911  ASSERT (vlib_get_thread_index () == 0);
1913  args->type_opaque, args->data);
1914 }
1915 
1917 
1918 void
1919 vlib_rpc_call_main_thread (void *callback, u8 * args, u32 arg_size)
1920 {
1922  {
1923  void (*fp) (void *, u8 *, u32) = rpc_call_main_thread_cb_fn;
1924  (*fp) (callback, args, arg_size);
1925  }
1926  else
1927  clib_warning ("BUG: rpc_call_main_thread_cb_fn NULL!");
1928 }
1929 
1930 clib_error_t *
1932 {
1933  return 0;
1934 }
1935 
1937 
1938 
1939 static clib_error_t *
1941  unformat_input_t * input, vlib_cli_command_t * cmd)
1942 {
1943  int i;
1944  int verbose = 0;
1945  clib_timebase_t _tb, *tb = &_tb;
1946 
1947  (void) unformat (input, "verbose %=", &verbose, 1);
1948 
1950  &vm->clib_time);
1951 
1952  vlib_cli_output (vm, "%U, %U GMT", format_clib_time, &vm->clib_time,
1953  verbose, format_clib_timebase_time,
1954  clib_timebase_now (tb));
1955 
1956  if (vec_len (vlib_mains) == 1)
1957  return 0;
1958 
1959  vlib_cli_output (vm, "Time last barrier release %.9f",
1961 
1962  for (i = 1; i < vec_len (vlib_mains); i++)
1963  {
1964  if (vlib_mains[i] == 0)
1965  continue;
1966 
1967  vlib_cli_output (vm, "%d: %U", i, format_clib_time,
1968  &vlib_mains[i]->clib_time, verbose);
1969 
1970  vlib_cli_output (vm, "Thread %d offset %.9f error %.9f", i,
1971  vlib_mains[i]->time_offset,
1974  }
1975  return 0;
1976 }
1977 
1978 /* *INDENT-OFF* */
1979 VLIB_CLI_COMMAND (f_command, static) =
1980 {
1981  .path = "show clock",
1982  .short_help = "show clock",
1983  .function = show_clock_command_fn,
1984 };
1985 /* *INDENT-ON* */
1986 
1987 /*
1988  * fd.io coding-style-patch-verification: ON
1989  *
1990  * Local Variables:
1991  * eval: (c-set-style "gnu")
1992  * End:
1993  */
_vlib_init_function_list_elt_t * worker_init_function_registrations
Definition: main.h:224
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:507
volatile u32 main_loop_count
Definition: main.h:99
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
u8 count
Definition: dhcp.api:208
static void barrier_trace_release_rec(f64 t_entry)
Definition: threads.c:96
static void barrier_trace_sync(f64 t_entry, f64 t_open, f64 t_closed)
Definition: threads.c:45
#define CLIB_PAUSE()
Definition: lock.h:23
u32 vl(void *p)
Definition: threads.c:29
uword * pending_rpc_requests
Definition: main.h:275
vlib_main_t vlib_global_main
Definition: main.c:1999
#define vec_foreach_index(var, v)
Iterate over vector indices.
void vlib_worker_wait_one_loop(void)
Wait until each of the workers has been once around the track.
Definition: threads.c:1649
#define clib_min(x, y)
Definition: clib.h:319
void vlib_stats_pop_heap2(u64 *error_vector, u32 thread_index, void *oldheap, int lock)
Definition: stat_segment.c:339
clib_spinlock_t pending_rpc_lock
Definition: main.h:277
vlib_process_t ** processes
Definition: node.h:742
#define VLIB_PENDING_FRAME_NO_NEXT_FRAME
Definition: node.h:463
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:937
u8 runtime_data[0]
Function dependent node-runtime data.
Definition: node.h:525
int vlib_frame_queue_enqueue(vlib_main_t *vm, u32 node_runtime_index, u32 frame_queue_index, vlib_frame_t *frame, vlib_frame_queue_msg_type_t type)
#define clib_atomic_add_fetch(a, b)
Definition: atomics.h:30
#define VLIB_MAIN_LOOP_ENTER_FUNCTION(x)
Definition: init.h:176
word elog_track_register(elog_main_t *em, elog_track_t *t)
register an event track
Definition: elog.c:198
unsigned long u64
Definition: types.h:89
#define CLIB_MEMORY_STORE_BARRIER()
Definition: clib.h:133
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
u32 index
Definition: node.h:282
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:291
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:640
clib_error_t * threads_init(vlib_main_t *vm)
Definition: threads.c:1931
format_function_t format_clib_time
Definition: time.h:76
void os_panic(void)
Definition: unix-misc.c:175
uword numa_heap_size
Definition: threads.h:348
u32 vlib_frame_queue_main_init(u32 node_index, u32 frame_queue_nelts)
Definition: threads.c:1854
u32 thread_index
Definition: main.h:218
void * thread_function_arg
Definition: threads.h:98
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:590
static int sort_registrations_by_no_clone(void *a0, void *a1)
Definition: threads.c:167
static u64 clib_cpu_time_now(void)
Definition: time.h:81
frame_queue_trace_t * frame_queue_traces
Definition: threads.h:165
void vlib_process_signal_event_mt_helper(vlib_process_signal_event_mt_args_t *args)
Definition: threads.c:1908
elog_track_t elog_track
Definition: threads.h:100
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:628
void vnet_main_fixup(vlib_fork_fixup_t which)
Definition: threads.c:1370
void * vlib_stats_push_heap(void *old)
Definition: stat_segment.c:53
static uword * clib_bitmap_set(uword *ai, uword i, uword value)
Sets the ith bit of a bitmap to new_value Removes trailing zeros from the bitmap. ...
Definition: bitmap.h:167
#define hash_set_mem(h, key, value)
Definition: hash.h:275
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
clib_time_t clib_time
Definition: main.h:87
void vlib_worker_thread_fn(void *arg)
Definition: threads.c:1820
u32 unformat_sched_policy(unformat_input_t *input, va_list *args)
Definition: threads.c:1229
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:518
struct vlib_thread_registration_ * next
Definition: threads.h:31
u32 buffer_index[VLIB_FRAME_SIZE]
Definition: threads.h:82
void * runtime_data
Definition: node.h:288
volatile u32 valid
Definition: threads.h:76
void vlib_worker_thread_barrier_sync_int(vlib_main_t *vm, const char *func_name)
Definition: threads.c:1450
static u32 vlib_last_vectors_per_main_loop(vlib_main_t *vm)
Definition: main.h:393
vlib_main_t ** vlib_mains
Definition: buffer.c:332
unsigned char u8
Definition: types.h:56
#define clib_bitmap_dup(v)
Duplicate a bitmap.
Definition: bitmap.h:87
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
double f64
Definition: types.h:142
u8 state
Definition: node.h:311
u16 thread_index
thread this node runs on
Definition: node.h:523
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:204
#define clib_memcpy(d, s, n)
Definition: string.h:180
u64 * counters_last_clear
Definition: error.h:51
static void vlib_worker_thread_barrier_check(void)
Definition: threads.h:400
vlib_thread_registration_t * next
Definition: threads.h:289
#define vec_add1_aligned(V, E, A)
Add 1 element to end of vector (alignment specified).
Definition: vec.h:599
#define VLIB_NODE_RUNTIME_DATA_SIZE
Definition: node.h:536
vlib_node_stats_t stats_last_clear
Definition: node.h:276
void vlib_worker_thread_node_runtime_update(void)
Definition: threads.c:1219
u64 count[FRAME_QUEUE_MAX_NELTS]
Definition: node.h:805
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:173
#define VLIB_INVALID_NODE_INDEX
Definition: node.h:377
#define vec_new(T, N)
Create new vector of given type and length (unspecified alignment, no header).
Definition: vec.h:350
void * heap_aligned_base
Definition: main.h:146
static f64 clib_timebase_now(clib_timebase_t *tb)
Definition: time_range.h:88
vlib_frame_queue_msg_type_t
Definition: threads.h:68
vlib_node_t ** nodes
Definition: node.h:701
vlib_frame_queue_elt_t ** handoff_queue_elt_by_thread_index
Definition: threads.h:151
clib_error_t * vlib_call_init_exit_functions_no_sort(vlib_main_t *vm, _vlib_init_function_list_elt_t **headp, int call_once)
Definition: init.c:369
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:182
#define clib_error_return(e, args...)
Definition: error.h:99
#define VLIB_FRAME_ALIGN
Definition: node.h:381
uword * lock
SMP lock, non-zero means locking required.
Definition: elog.h:175
uword * cpu_core_bitmap
Definition: threads.h:326
#define BARRIER_MINIMUM_OPEN_FACTOR
Definition: threads.c:1409
vlib_frame_queue_elt_t * elts
Definition: threads.h:144
pthread_t thread[MAX_CONNS]
Definition: main.c:142
f64 time_last_barrier_release
Definition: main.h:90
void * clib_per_numa_mheaps[CLIB_MAX_NUMAS]
Definition: mem_dlmalloc.c:26
unsigned int u32
Definition: types.h:88
vlib_node_runtime_t * nodes_by_type[VLIB_N_NODE_TYPE]
Definition: node.h:711
#define VLIB_FRAME_SIZE
Definition: node.h:380
void vlib_set_thread_name(char *name)
Definition: threads.c:152
void vl_msg_api_handler_no_free(void *)
Definition: threads.c:410
u32 cpu_id
Definition: vpe.api:221
format_function_t format_clib_timebase_time
Definition: time_range.h:70
#define hash_create_string(elts, value_bytes)
Definition: hash.h:690
void unformat_init_string(unformat_input_t *input, char *string, int string_len)
Definition: unformat.c:1029
vlib_fork_fixup_t
Definition: threads.h:233
#define BARRIER_SYNC_TIMEOUT
Definition: threads.h:199
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:63
void * rpc_call_main_thread_cb_fn
Definition: threads.c:1916
vl_api_fib_path_type_t type
Definition: fib_types.api:123
VLIB_REGISTER_THREAD(worker_thread_reg, static)
int extern_thread_mgmt
Definition: threads.h:345
vlib_worker_thread_t * vlib_worker_threads
Definition: threads.c:34
#define clib_bitmap_foreach(i, ai, body)
Macro to iterate across set bits in a bitmap.
Definition: bitmap.h:361
void * thread_mheap
Definition: threads.h:95
u32 next_frame_index
Definition: node.h:460
vlib_node_stats_t stats_total
Definition: node.h:272
volatile u64 head
Definition: threads.h:131
u16 state
Input node state.
Definition: node.h:513
static void vlib_process_signal_event(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
Definition: node_funcs.h:934
static uword clib_bitmap_first_set(uword *ai)
Return the lowest numbered set bit in a bitmap.
Definition: bitmap.h:385
u8 * vlib_thread_stack_init(uword thread_index)
Definition: main.c:665
static void vlib_next_frame_init(vlib_next_frame_t *nf)
Definition: node.h:444
vlib_error_main_t error_main
Definition: main.h:179
u16 frame_flags
Definition: node.h:387
vlib_thread_callbacks_t cb
Definition: threads.h:344
#define VLIB_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:417
int vlib_thread_cb_register(struct vlib_main_t *vm, vlib_thread_callbacks_t *cb)
Definition: threads.c:1895
void vlib_stat_segment_unlock(void)
Definition: stat_segment.c:41
u8 vlib_worker_thread_barrier_held(void)
Return true if the wroker thread barrier is held.
Definition: threads.c:1441
struct _unformat_input_t unformat_input_t
const char * barrier_context
Definition: threads.h:107
char * name
Track name vector.
Definition: elog.h:116
#define clib_error_return_unix(e, args...)
Definition: error.h:102
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:216
static void * clib_mem_get_per_cpu_heap(void)
Definition: mem.h:65
u32 * node_by_error
Definition: node.h:769
void vlib_frame_free(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_frame_t *f)
Definition: main.c:238
#define ELOG_DATA(em, f)
Definition: elog.h:484
#define PREDICT_FALSE(x)
Definition: clib.h:118
static clib_error_t * vlib_launch_thread_int(void *fp, vlib_worker_thread_t *w, unsigned cpu_id)
Definition: threads.c:626
void vlib_worker_thread_node_refork(void)
Definition: threads.c:1033
clib_error_t *(* vlib_thread_set_lcore_cb)(u32 thread, u16 cpu)
Definition: threads.h:283
u32 node_index
Node index.
Definition: node.h:498
uword * init_functions_called
Definition: main.h:215
void clib_time_init(clib_time_t *c)
Definition: time.c:207
uword * frame_size_hash
Definition: node.h:757
vlib_thread_main_t vlib_thread_main
Definition: threads.c:35
clib_error_t * clib_sysfs_read(char *file_name, char *fmt,...)
Definition: sysfs.c:50
vlib_main_t * vm
Definition: in2out_ed.c:1599
word fformat(FILE *f, char *fmt,...)
Definition: format.c:462
void(* thread_function)(void *)
Definition: threads.h:97
static clib_error_t * cpu_config(vlib_main_t *vm, unformat_input_t *input)
Definition: threads.c:1243
i32 n_vectors[FRAME_QUEUE_MAX_NELTS]
Definition: node.h:800
u64 * counters
Definition: error.h:48
u32 owner_node_index
Definition: node.h:357
vlib_frame_queue_t * vlib_frame_queue_alloc(int nelts)
Definition: threads.c:376
volatile u64 tail
Definition: threads.h:123
#define clib_mem_alloc_no_fail(size)
Definition: mem.h:201
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
Definition: init.h:226
#define UNFORMAT_END_OF_INPUT
Definition: format.h:145
svmdb_client_t * c
u16 n_vectors
Definition: node.h:399
vlib_frame_queue_t ** vlib_frame_queues
Definition: threads.h:161
DLMALLOC_EXPORT mspace create_mspace(size_t capacity, int locked)
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:218
u32 node_runtime_index
Definition: node.h:454
vlib_pending_frame_t * pending_frames
Definition: node.h:727
vlib_thread_function_t * function
Definition: threads.h:36
int vlib_frame_queue_dequeue(vlib_main_t *vm, vlib_frame_queue_main_t *fqm)
Definition: threads.c:1686
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:317
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:380
void * heap_base
Definition: main.h:143
static void * clib_mem_set_heap(void *heap)
Definition: mem.h:268
#define clib_warning(format, args...)
Definition: error.h:59
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
Definition: node_funcs.h:89
void vlib_stat_segment_lock(void)
Definition: stat_segment.c:33
elog_main_t elog_main
Definition: main.h:193
frame_queue_nelt_counter_t * frame_queue_histogram
Definition: threads.h:166
#define VLIB_FRAME_PENDING
Definition: node.h:431
static uword clib_bitmap_get(uword *ai, uword i)
Gets the ith bit value from a bitmap.
Definition: bitmap.h:197
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:442
string name[64]
Definition: ip.api:44
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1599
void vlib_worker_thread_init(vlib_worker_thread_t *w)
Definition: threads.c:537
uword os_get_nthreads(void)
Definition: threads.c:146
static void * clib_mem_get_heap(void)
Definition: mem.h:262
volatile u32 * wait_at_barrier
Definition: threads.h:90
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:152
vlib_frame_queue_per_thread_data_t * per_thread_data
Definition: threads.h:162
vlib_frame_queue_t ** congested_handoff_queue_by_thread_index
Definition: threads.h:152
#define never_inline
Definition: clib.h:102
signed int i32
Definition: types.h:77
#define hash_create(elts, value_bytes)
Definition: hash.h:696
#define ASSERT(truth)
never_inline void vlib_node_runtime_sync_stats(vlib_main_t *vm, vlib_node_runtime_t *r, uword n_calls, uword n_vectors, uword n_clocks, uword n_ticks0, uword n_ticks1)
Definition: main.c:569
static clib_error_t * show_clock_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: threads.c:1940
static void barrier_trace_sync_rec(f64 t_entry)
Definition: threads.c:72
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:689
vlib_frame_queue_main_t * frame_queue_mains
Definition: threads.h:332
void clib_timebase_init(clib_timebase_t *tb, i32 timezone_offset_in_hours, clib_timebase_daylight_time_t daylight_type, clib_time_t *clib_time)
Definition: time_range.c:19
u16 flags
Definition: node.h:390
static void clib_mem_free(void *p)
Definition: mem.h:215
void vlib_worker_thread_initial_barrier_sync_and_release(vlib_main_t *vm)
Definition: threads.c:1413
#define clib_error_report(e)
Definition: error.h:113
#define clib_bitmap_free(v)
Free a bitmap.
Definition: bitmap.h:92
int need_vlib_worker_thread_node_runtime_update
Definition: main.h:252
uword * thread_registrations_by_name
Definition: threads.h:294
#define BARRIER_MINIMUM_OPEN_LIMIT
Definition: threads.c:1405
clib_error_t *(* vlib_launch_thread_cb)(void *fp, vlib_worker_thread_t *w, unsigned cpu_id)
Definition: threads.h:281
volatile u32 * node_reforks_required
Definition: threads.h:108
const char * barrier_caller
Definition: threads.h:106
#define clib_max(x, y)
Definition: clib.h:312
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
#define clib_atomic_fetch_add(a, b)
Definition: atomics.h:23
void vlib_node_sync_stats(vlib_main_t *vm, vlib_node_t *n)
Definition: main.c:611
void vlib_get_thread_core_numa(vlib_worker_thread_t *w, unsigned cpu_id)
Definition: threads.c:592
static uword clib_bitmap_count_set_bits(uword *ai)
Return the number of set bits in a bitmap.
Definition: bitmap.h:462
static void barrier_trace_release(f64 t_entry, f64 t_closed_total, f64 t_update_main)
Definition: threads.c:118
void vlib_worker_loop(vlib_main_t *vm)
Definition: main.c:1994
#define vec_dup_aligned(V, A)
Return copy of vector (no header, alignment specified).
Definition: vec.h:436
u32 elog_string(elog_main_t *em, char *fmt,...)
add a string to the event-log string table
Definition: elog.c:562
f64 barrier_no_close_before
Definition: main.h:269
static clib_error_t * start_workers(vlib_main_t *vm)
Definition: threads.c:672
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
void vlib_rpc_call_main_thread(void *callback, u8 *args, u32 arg_size)
Definition: threads.c:1919
vlib_node_main_t node_main
Definition: main.h:158
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
vlib_next_frame_t * next_frames
Definition: node.h:724
#define vec_sort_with_function(vec, f)
Sort a vector using the supplied element comparison function.
Definition: vec.h:1053
static void unformat_free(unformat_input_t *i)
Definition: format.h:163
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
void * clib_mem_init_thread_safe_numa(void *memory, uword memory_size, u8 numa)
Definition: mem_dlmalloc.c:242
volatile u64 head_hint
Definition: threads.h:140
#define VLIB_THREAD_STACK_SIZE
Definition: threads.h:66
f64 barrier_epoch
Definition: main.h:266
vlib_frame_size_t * frame_sizes
Definition: node.h:760
#define hash_get_mem(h, key)
Definition: hash.h:269
unformat_function_t unformat_memory_size
Definition: format.h:296
static void worker_thread_node_runtime_update_internal(void)
Definition: threads.c:981
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:165
volatile u32 * workers_at_barrier
Definition: threads.h:91
uword clib_calljmp(uword(*func)(uword func_arg), uword func_arg, void *stack)
static uword * clib_sysfs_list_to_bitmap(char *filename)
Definition: threads.c:177
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1540
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
static f64 vlib_time_now_ticks(vlib_main_t *vm, u64 n)
Definition: main.h:304
uword clib_bitmap_t
Definition: bitmap.h:50
static vlib_node_t * vlib_get_node(vlib_main_t *vm, u32 i)
Get vlib node by index.
Definition: node_funcs.h:59
#define vec_foreach(var, vec)
Vector iterator.
void * vlib_worker_thread_bootstrap_fn(void *arg)
Definition: threads.c:574
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
Definition: in2out_ed.c:1600
#define SYS_gettid
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:130
u32 node_runtime_index
Definition: node.h:411
uword * cpu_socket_bitmap
Definition: threads.h:329
#define foreach_sched_policy
Definition: threads.h:264
vlib_thread_registration_t ** registrations
Definition: threads.h:292
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:304
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
Definition: vec.h:554
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u8 ** vlib_thread_stacks
Definition: main.c:648
pthread_t thread_id
Definition: threads.h:114
vlib_thread_registration_t * registration
Definition: threads.h:102
volatile u32 worker_thread_release
Definition: threads.h:335
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
void vlib_worker_thread_fork_fixup(vlib_fork_fixup_t which)
Definition: threads.c:1375
clib_random_buffer_t random_buffer
Definition: main.h:212
u8 runtime_data_bytes
Definition: node.h:314
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:978
#define FRAME_QUEUE_MAX_NELTS
Definition: node.h:789
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:171
#define VLIB_FRAME_FREE_AFTER_DISPATCH
Definition: node.h:434
clib_error_t * vlib_thread_init(vlib_main_t *vm)
Definition: threads.c:207