FD.io VPP  v16.06
Vector Packet Processing
threads.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #define _GNU_SOURCE
16 #include <sched.h>
17 
18 #include <signal.h>
19 #include <math.h>
20 #include <vppinfra/format.h>
21 #include <vlib/vlib.h>
22 
23 #include <vlib/threads.h>
24 #include <vlib/unix/cj.h>
25 
26 
27 #if DPDK==1
28 #include <rte_config.h>
29 #include <rte_common.h>
30 #include <rte_eal.h>
31 #include <rte_launch.h>
32 #include <rte_lcore.h>
33 #endif
35 
36 #define FRAME_QUEUE_NELTS 32
37 
38 
39 #if DPDK==1
40 /*
41  * Weak definitions of DPDK symbols used in this file.
42  * Needed for linking test programs without DPDK libs.
43  */
44 unsigned __thread __attribute__((weak)) RTE_PER_LCORE(_lcore_id);
45 struct lcore_config __attribute__((weak)) lcore_config[];
46 unsigned __attribute__((weak)) rte_socket_id();
47 int __attribute__((weak)) rte_eal_remote_launch();
48 #endif
49 u32 vl(void *p)
50 {
51  return vec_len (p);
52 }
53 
54 void debug_hex_bytes (u8 *s, u32 n)
55 {
56  fformat (stderr, "%U\n", format_hex_bytes, s, n);
57 }
58 
60 
61 uword
63 {
64  void * sp;
65  uword n;
66  u32 len;
67 
69  if (len == 0)
70  return 0;
71 
72  /* Get any old stack address. */
73  sp = &sp;
74 
75  n = ((uword)sp - (uword)vlib_thread_stacks[0])
77 
78  /* "processes" have their own stacks, and they always run in thread 0 */
79  n = n >= len ? 0 : n;
80 
81  return n;
82 }
83 
84 void
86 {
87  int pthread_setname_np (pthread_t __target_thread, const char *__name);
88  pthread_t thread = pthread_self();
89 
90  if (thread)
91  pthread_setname_np(thread, name);
92 }
93 
94 static int sort_registrations_by_no_clone (void *a0, void * a1)
95 {
96  vlib_thread_registration_t ** tr0 = a0;
97  vlib_thread_registration_t ** tr1 = a1;
98 
99  return ((i32)((*tr0)->no_data_structure_clone)
100  - ((i32)((*tr1)->no_data_structure_clone)));
101 }
102 
103 static uword *
105 {
106  FILE *fp;
107  uword *r = 0;
108 
109  fp = fopen (filename, "r");
110 
111  if (fp != NULL)
112  {
113  u8 * buffer = 0;
114  vec_validate (buffer, 256-1);
115  if (fgets ((char *)buffer, 256, fp))
116  {
117  unformat_input_t in;
118  unformat_init_string (&in, (char *) buffer, strlen ((char *) buffer));
119  unformat(&in, "%U", unformat_bitmap_list, &r);
120  unformat_free (&in);
121  }
122  vec_free(buffer);
123  fclose(fp);
124  }
125  return r;
126 }
127 
128 
129 /* Called early in the init sequence */
130 
131 clib_error_t *
133 {
137  u32 n_vlib_mains = 1;
138  u32 first_index = 1;
139  u32 i;
140  uword * avail_cpu;
141 
142  /* get bitmaps of active cpu cores and sockets */
143  tm->cpu_core_bitmap =
144  vlib_sysfs_list_to_bitmap("/sys/devices/system/cpu/online");
145  tm->cpu_socket_bitmap =
146  vlib_sysfs_list_to_bitmap("/sys/devices/system/node/online");
147 
148  avail_cpu = clib_bitmap_dup(tm->cpu_core_bitmap);
149 
150  /* skip cores */
151  for (i=0; i < tm->skip_cores; i++)
152  {
153  uword c = clib_bitmap_first_set(avail_cpu);
154  if (c == ~0)
155  return clib_error_return (0, "no available cpus to skip");
156 
157  avail_cpu = clib_bitmap_set(avail_cpu, c, 0);
158  }
159 
160  /* grab cpu for main thread */
161  if (!tm->main_lcore)
162  {
163  tm->main_lcore = clib_bitmap_first_set(avail_cpu);
164  if (tm->main_lcore == (u8) ~0)
165  return clib_error_return (0, "no available cpus to be used for the"
166  " main thread");
167  }
168  else
169  {
170  if (clib_bitmap_get(avail_cpu, tm->main_lcore) == 0)
171  return clib_error_return (0, "cpu %u is not available to be used"
172  " for the main thread", tm->main_lcore);
173  }
174  avail_cpu = clib_bitmap_set(avail_cpu, tm->main_lcore, 0);
175 
176  /* assume that there is socket 0 only if there is no data from sysfs */
177  if (!tm->cpu_socket_bitmap)
178  tm->cpu_socket_bitmap = clib_bitmap_set(0, 0, 1);
179 
180  /* pin main thread to main_lcore */
181 #if DPDK==0
182  {
183  cpu_set_t cpuset;
184  CPU_ZERO(&cpuset);
185  CPU_SET(tm->main_lcore, &cpuset);
186  pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
187  }
188 #endif
189 
190  /* as many threads as stacks... */
193 
194  /* Preallocate thread 0 */
195  _vec_len(vlib_worker_threads) = 1;
199  w->dpdk_lcore_id = -1;
200  w->lwp = syscall(SYS_gettid);
201  tm->n_vlib_mains = 1;
202 
203  /* assign threads to cores and set n_vlib_mains */
204  tr = tm->next;
205 
206  while (tr)
207  {
208  vec_add1 (tm->registrations, tr);
209  tr = tr->next;
210  }
211 
214 
215  for (i = 0; i < vec_len (tm->registrations); i++)
216  {
217  int j;
218  tr = tm->registrations[i];
219  tr->first_index = first_index;
220  first_index += tr->count;
221  n_vlib_mains += (tr->no_data_structure_clone == 0) ? tr->count : 0;
222 
223  /* construct coremask */
224  if (tr->use_pthreads || !tr->count)
225  continue;
226 
227  if (tr->coremask)
228  {
229  uword c;
230  clib_bitmap_foreach (c, tr->coremask, ({
231  if (clib_bitmap_get(avail_cpu, c) == 0)
232  return clib_error_return (0, "cpu %u is not available to be used"
233  " for the '%s' thread",c, tr->name);
234 
235  avail_cpu = clib_bitmap_set(avail_cpu, c, 0);
236  }));
237 
238  }
239  else
240  {
241  for (j=0; j < tr->count; j++)
242  {
243  uword c = clib_bitmap_first_set(avail_cpu);
244  if (c == ~0)
245  return clib_error_return (0, "no available cpus to be used for"
246  " the '%s' thread", tr->name);
247 
248  avail_cpu = clib_bitmap_set(avail_cpu, c, 0);
249  tr->coremask = clib_bitmap_set(tr->coremask, c, 1);
250  }
251  }
252  }
253 
254  clib_bitmap_free(avail_cpu);
255 
256  tm->n_vlib_mains = n_vlib_mains;
257 
260 
261 
264  FRAME_QUEUE_NELTS)/100);
265  return 0;
266 }
267 
270 {
272 
274  {
275  clib_warning ("out of worker threads... Quitting...");
276  exit(1);
277  }
280  return w;
281 }
282 
284 {
285  vlib_frame_queue_t * fq;
286 
287  fq = clib_mem_alloc_aligned(sizeof (*fq), CLIB_CACHE_LINE_BYTES);
288  memset (fq, 0, sizeof (*fq));
289  fq->nelts = nelts;
290  fq->vector_threshold = 128; // packets
292 
293  if (1)
294  {
295  if (((uword)&fq->tail) & (CLIB_CACHE_LINE_BYTES - 1))
296  fformat(stderr, "WARNING: fq->tail unaligned\n");
297  if (((uword)&fq->head) & (CLIB_CACHE_LINE_BYTES - 1))
298  fformat(stderr, "WARNING: fq->head unaligned\n");
299  if (((uword)fq->elts) & (CLIB_CACHE_LINE_BYTES - 1))
300  fformat(stderr, "WARNING: fq->elts unaligned\n");
301 
302  if (sizeof (fq->elts[0]) % CLIB_CACHE_LINE_BYTES)
303  fformat(stderr, "WARNING: fq->elts[0] size %d\n",
304  sizeof (fq->elts[0]));
305  if (nelts & (nelts -1))
306  {
307  fformat (stderr, "FATAL: nelts MUST be a power of 2\n");
308  abort();
309  }
310  }
311 
312  return (fq);
313 }
314 
315 void vl_msg_api_handler_no_free (void *) __attribute__ ((weak));
316 void vl_msg_api_handler_no_free (void *v) { }
317 
318 /* Turned off, save as reference material... */
319 #if 0
320 static inline int vlib_frame_queue_dequeue_internal (int thread_id,
321  vlib_main_t *vm,
322  vlib_node_main_t *nm)
323 {
324  vlib_frame_queue_t *fq = vlib_frame_queues[thread_id];
326  vlib_frame_t *f;
329  u32 node_runtime_index;
330  int msg_type;
331  u64 before;
332  int processed = 0;
333 
334  ASSERT(vm == vlib_mains[thread_id]);
335 
336  while (1)
337  {
338  if (fq->head == fq->tail)
339  return processed;
340 
341  elt = fq->elts + ((fq->head+1) & (fq->nelts-1));
342 
343  if (!elt->valid)
344  return processed;
345 
346  before = clib_cpu_time_now();
347 
348  f = elt->frame;
349  node_runtime_index = elt->node_runtime_index;
350  msg_type = elt->msg_type;
351 
352  switch (msg_type)
353  {
354  case VLIB_FRAME_QUEUE_ELT_FREE_BUFFERS:
356  /* note fallthrough... */
357  case VLIB_FRAME_QUEUE_ELT_FREE_FRAME:
359  node_runtime_index);
360  vlib_frame_free (vm, r, f);
361  break;
363  vec_add2 (vm->node_main.pending_frames, p, 1);
365  p->node_runtime_index = elt->node_runtime_index;
366  p->frame_index = vlib_frame_index (vm, f);
368  fq->dequeue_vectors += (u64) f->n_vectors;
369  break;
370  case VLIB_FRAME_QUEUE_ELT_API_MSG:
372  break;
373  default:
374  clib_warning ("bogus frame queue message, type %d", msg_type);
375  break;
376  }
377  elt->valid = 0;
378  fq->dequeues++;
379  fq->dequeue_ticks += clib_cpu_time_now() - before;
381  fq->head++;
382  processed++;
383  }
384  ASSERT(0);
385  return processed;
386 }
387 
388 int vlib_frame_queue_dequeue (int thread_id,
389  vlib_main_t *vm,
390  vlib_node_main_t *nm)
391 {
392  return vlib_frame_queue_dequeue_internal (thread_id, vm, nm);
393 }
394 
395 int vlib_frame_queue_enqueue (vlib_main_t *vm, u32 node_runtime_index,
396  u32 frame_queue_index, vlib_frame_t *frame,
398 {
399  vlib_frame_queue_t *fq = vlib_frame_queues[frame_queue_index];
401  u32 save_count;
402  u64 new_tail;
403  u64 before = clib_cpu_time_now();
404 
405  ASSERT (fq);
406 
407  new_tail = __sync_add_and_fetch (&fq->tail, 1);
408 
409  /* Wait until a ring slot is available */
410  while (new_tail >= fq->head + fq->nelts)
411  {
412  f64 b4 = vlib_time_now_ticks (vm, before);
414  /* Bad idea. Dequeue -> enqueue -> dequeue -> trouble */
415  // vlib_frame_queue_dequeue (vm->cpu_index, vm, nm);
416  }
417 
418  elt = fq->elts + (new_tail & (fq->nelts-1));
419 
420  /* this would be very bad... */
421  while (elt->valid)
422  {
423  }
424 
425  /* Once we enqueue the frame, frame->n_vectors is owned elsewhere... */
426  save_count = frame->n_vectors;
427 
428  elt->frame = frame;
429  elt->node_runtime_index = node_runtime_index;
430  elt->msg_type = type;
432  elt->valid = 1;
433 
434  return save_count;
435 }
436 #endif /* 0 */
437 
438 /* To be called by vlib worker threads upon startup */
440 {
442 
443  /* worker threads wants no signals. */
444  {
445  sigset_t s;
446  sigfillset (&s);
447  pthread_sigmask (SIG_SETMASK, &s, 0);
448  }
449 
451 
453  {
454  w->name = format(0, "%v_%s_%d%c", tm->thread_prefix,
456  w->instance_id,
457  '\0');
458  vlib_set_thread_name((char *)w->name);
459  }
460 
461  if (!w->registration->use_pthreads)
462  {
463 
464  /* Initial barrier sync, for both worker and i/o threads */
466 
468  ;
469 
471  }
472 }
473 
475 {
476  void *rv;
477  vlib_worker_thread_t *w = arg;
478 
479  w->lwp = syscall(SYS_gettid);
480  w->dpdk_lcore_id = -1;
481 #if DPDK==1
482  if (w->registration && !w->registration->use_pthreads &&
483  rte_socket_id) /* do we really have dpdk linked */
484  {
485  unsigned lcore = rte_lcore_id();
486  lcore = lcore < RTE_MAX_LCORE ? lcore : -1;
487  w->dpdk_lcore_id = lcore;
488  }
489 #endif
490 
491  rv = (void *) clib_calljmp
492  ((uword (*)(uword)) w->thread_function,
494  /* NOTREACHED, we hope */
495  return rv;
496 }
497 
498 static int
499 vlib_launch_thread (void *fp, vlib_worker_thread_t *w, unsigned lcore_id)
500 {
501  void *(*fp_arg)(void *) = fp;
502 
503 #if DPDK==1
504  if (!w->registration->use_pthreads)
505  if (rte_eal_remote_launch) /* do we have dpdk linked */
506  return rte_eal_remote_launch (fp, (void *)w, lcore_id);
507  else
508  return -1;
509  else
510 #endif
511  {
512  int ret;
513  pthread_t worker;
514  cpu_set_t cpuset;
515  CPU_ZERO(&cpuset);
516  CPU_SET(lcore_id, &cpuset);
517 
518  ret = pthread_create (&worker, NULL /* attr */, fp_arg, (void *)w);
519  if(ret == 0)
520  return pthread_setaffinity_np(worker, sizeof(cpu_set_t), &cpuset);
521  else
522  return ret;
523  }
524 }
525 
527 {
528  int i, j;
530  vlib_main_t *vm_clone;
531  void *oldheap;
532  vlib_frame_queue_t *fq;
535  vlib_node_runtime_t * rt;
536  u32 n_vlib_mains = tm->n_vlib_mains;
537  u32 worker_thread_index;
538  u8 * main_heap = clib_mem_get_per_cpu_heap();
539  mheap_t * main_heap_header = mheap_header (main_heap);
540 
542 
543  /* Set up the main thread */
545  w->elog_track.name = "main thread";
547 
548  if (vec_len(tm->thread_prefix))
549  {
550  w->name = format(0, "%v_main%c", tm->thread_prefix, '\0');
551  vlib_set_thread_name((char *)w->name);
552  }
553 
554 #if DPDK==1
555  w->dpdk_lcore_id = -1;
556  if (rte_socket_id) /* do we really have dpdk linked */
557  {
558  unsigned lcore = rte_lcore_id();
559  w->dpdk_lcore_id = lcore < RTE_MAX_LCORE ? lcore : -1;;
560  }
561 #endif
562 
563  /*
564  * Truth of the matter: we always use at least two
565  * threads. So, make the main heap thread-safe
566  * and make the event log thread-safe.
567  */
568  main_heap_header->flags |= MHEAP_FLAG_THREAD_SAFE;
569  vm->elog_main.lock =
572  vm->elog_main.lock[0] = 0;
573 
574  if (n_vlib_mains > 1)
575  {
577  _vec_len (vlib_mains) = 0;
578  vec_add1 (vlib_mains, vm);
579 
581  _vec_len (vlib_frame_queues) = 0;
584 
589 
590  /* Ask for an initial barrier sync */
593 
594  worker_thread_index = 1;
595 
596  for (i = 0; i < vec_len(tm->registrations); i++)
597  {
598  vlib_node_main_t *nm, *nm_clone;
599  vlib_buffer_main_t *bm_clone;
600  vlib_buffer_free_list_t *fl_clone, *fl_orig;
601  vlib_buffer_free_list_t *orig_freelist_pool;
602  int k;
603 
604  tr = tm->registrations[i];
605 
606  if (tr->count == 0)
607  continue;
608 
609  for (k = 0; k < tr->count; k++)
610  {
612  if (tr->mheap_size)
613  w->thread_mheap = mheap_alloc (0 /* use VM */, tr->mheap_size);
614  else
615  w->thread_mheap = main_heap;
617  w->thread_function = tr->function;
618  w->thread_function_arg = w;
619  w->instance_id = k;
620  w->registration = tr;
621 
622  w->elog_track.name =
623  (char *) format (0, "%s %d", tr->name, k+1);
624  vec_add1 (w->elog_track.name, 0);
626 
627  if (tr->no_data_structure_clone)
628  continue;
629 
630  /* Allocate "to-worker-N" frame queue */
631  if (tr->frame_queue_nelts) {
633  } else {
635  }
636 
637  vec_validate (vlib_frame_queues, worker_thread_index);
638  vlib_frame_queues[worker_thread_index] = fq;
639 
640  /* Fork vlib_global_main et al. Look for bugs here */
641  oldheap = clib_mem_set_heap (w->thread_mheap);
642 
643  vm_clone = clib_mem_alloc (sizeof (*vm_clone));
644  clib_memcpy (vm_clone, vlib_mains[0], sizeof (*vm_clone));
645 
646  vm_clone->cpu_index = worker_thread_index;
647  vm_clone->heap_base = w->thread_mheap;
648  vm_clone->mbuf_alloc_list = 0;
649  memset (&vm_clone->random_buffer, 0, sizeof (vm_clone->random_buffer));
650 
651  nm = &vlib_mains[0]->node_main;
652  nm_clone = &vm_clone->node_main;
653  /* fork next frames array, preserving node runtime indices */
654  nm_clone->next_frames = vec_dup (nm->next_frames);
655  for (j = 0; j < vec_len (nm_clone->next_frames); j++)
656  {
657  vlib_next_frame_t *nf = &nm_clone->next_frames[j];
658  u32 save_node_runtime_index;
659  u32 save_flags;
660 
661  save_node_runtime_index = nf->node_runtime_index;
662  save_flags = nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH;
664  nf->node_runtime_index = save_node_runtime_index;
665  nf->flags = save_flags;
666  }
667 
668  /* fork the frame dispatch queue */
669  nm_clone->pending_frames = 0;
670  vec_validate (nm_clone->pending_frames, 10); /* $$$$$?????? */
671  _vec_len (nm_clone->pending_frames) = 0;
672 
673  /* fork nodes */
674  nm_clone->nodes = 0;
675  for (j = 0; j < vec_len (nm->nodes); j++)
676  {
677  vlib_node_t *n;
678  n = clib_mem_alloc_no_fail (sizeof(*n));
679  clib_memcpy (n, nm->nodes[j], sizeof (*n));
680  /* none of the copied nodes have enqueue rights given out */
682  memset (&n->stats_total, 0, sizeof (n->stats_total));
683  memset (&n->stats_last_clear, 0, sizeof (n->stats_last_clear));
684  vec_add1 (nm_clone->nodes, n);
685  }
688 
692  rt->cpu_index = vm_clone->cpu_index;
693 
694  nm_clone->processes = vec_dup (nm->processes);
695 
696  /* zap the (per worker) frame freelists, etc */
697  nm_clone->frame_sizes = 0;
698  nm_clone->frame_size_hash = 0;
699 
700  /* Packet trace buffers are guaranteed to be empty, nothing to do here */
701 
702  clib_mem_set_heap (oldheap);
703  vec_add1 (vlib_mains, vm_clone);
704 
705  vm_clone->error_main.counters =
706  vec_dup(vlib_mains[0]->error_main.counters);
707  vm_clone->error_main.counters_last_clear =
708  vec_dup(vlib_mains[0]->error_main.counters_last_clear);
709 
710  /* Fork the vlib_buffer_main_t free lists, etc. */
711  bm_clone = vec_dup (vm_clone->buffer_main);
712  vm_clone->buffer_main = bm_clone;
713 
714  orig_freelist_pool = bm_clone->buffer_free_list_pool;
715  bm_clone->buffer_free_list_pool = 0;
716 
717  pool_foreach (fl_orig, orig_freelist_pool,
718  ({
720  fl_clone, CLIB_CACHE_LINE_BYTES);
721  ASSERT (fl_orig - orig_freelist_pool
722  == fl_clone - bm_clone->buffer_free_list_pool);
723 
724  fl_clone[0] = fl_orig[0];
725  fl_clone->aligned_buffers = 0;
726  fl_clone->unaligned_buffers = 0;
727  fl_clone->n_alloc = 0;
728  }));
729 
730  worker_thread_index++;
731  }
732  }
733  }
734  else
735  {
736  /* only have non-data-structure copy threads to create... */
737  for (i = 0; i < vec_len(tm->registrations); i++)
738  {
739  tr = tm->registrations[i];
740 
741  for (j = 0; j < tr->count; j++)
742  {
744  if (tr->mheap_size)
745  w->thread_mheap = mheap_alloc (0 /* use VM */, tr->mheap_size);
746  else
747  w->thread_mheap = main_heap;
749  w->thread_function = tr->function;
750  w->thread_function_arg = w;
751  w->instance_id = j;
752  w->elog_track.name =
753  (char *) format (0, "%s %d", tr->name, j+1);
754  w->registration = tr;
755  vec_add1 (w->elog_track.name, 0);
757  }
758  }
759  }
760 
761  worker_thread_index = 1;
762 
763  for (i = 0; i < vec_len (tm->registrations); i++)
764  {
765  int j;
766 
767  tr = tm->registrations[i];
768 
769  if (tr->use_pthreads || tm->use_pthreads)
770  {
771  for (j = 0; j < tr->count; j++)
772  {
773  w = vlib_worker_threads + worker_thread_index++;
775  clib_warning ("Couldn't start '%s' pthread ", tr->name);
776  }
777  }
778  else
779  {
780  uword c;
781  clib_bitmap_foreach (c, tr->coremask, ({
782  w = vlib_worker_threads + worker_thread_index++;
783  if (vlib_launch_thread (vlib_worker_thread_bootstrap_fn, w, c) < 0)
784  clib_warning ("Couldn't start DPDK lcore %d", c);
785 
786  }));
787  }
788  }
791  return 0;
792 }
793 
795 
797 {
798  int i, j;
800  vlib_main_t *vm;
801  vlib_node_main_t *nm, *nm_clone;
802  vlib_node_t ** old_nodes_clone;
803  vlib_main_t *vm_clone;
804  vlib_node_runtime_t * rt, * old_rt;
805  void *oldheap;
806  never_inline void
809  uword n_calls,
810  uword n_vectors,
811  uword n_clocks);
812 
813  ASSERT (os_get_cpu_number() == 0);
814 
815  if (vec_len (vlib_mains) == 0)
816  return;
817 
818  vm = vlib_mains[0];
819  nm = &vm->node_main;
820 
821  ASSERT (os_get_cpu_number() == 0);
823 
824  /*
825  * Scrape all runtime stats, so we don't lose node runtime(s) with
826  * pending counts, or throw away worker / io thread counts.
827  */
828  for (j = 0; j < vec_len (nm->nodes); j++)
829  {
830  vlib_node_t * n;
831  n = nm->nodes[j];
832  vlib_node_sync_stats (vm, n);
833  }
834 
835  for (i = 1; i < vec_len (vlib_mains); i++)
836  {
837  vlib_node_t * n;
838 
839  vm_clone = vlib_mains[i];
840  nm_clone = &vm_clone->node_main;
841 
842  for (j = 0; j < vec_len (nm_clone->nodes); j++)
843  {
844  n = nm_clone->nodes[j];
845 
846  rt = vlib_node_get_runtime (vm_clone, n->index);
847  vlib_node_runtime_sync_stats (vm_clone, rt, 0, 0, 0);
848  }
849  }
850 
851  for (i = 1; i < vec_len (vlib_mains); i++)
852  {
853  vlib_node_runtime_t * rt;
854  w = vlib_worker_threads + i;
855  oldheap = clib_mem_set_heap (w->thread_mheap);
856 
857  vm_clone = vlib_mains[i];
858 
859  /* Re-clone error heap */
860  u64 * old_counters = vm_clone->error_main.counters;
861  u64 * old_counters_all_clear = vm_clone->error_main.counters_last_clear;
862  clib_memcpy (&vm_clone->error_main, &vm->error_main, sizeof (vm->error_main));
863  j = vec_len(vm->error_main.counters) - 1;
865  vec_validate_aligned(old_counters_all_clear, j, CLIB_CACHE_LINE_BYTES);
866  vm_clone->error_main.counters = old_counters;
867  vm_clone->error_main.counters_last_clear = old_counters_all_clear;
868 
869  nm_clone = &vm_clone->node_main;
870  vec_free (nm_clone->next_frames);
871  nm_clone->next_frames = vec_dup (nm->next_frames);
872 
873  for (j = 0; j < vec_len (nm_clone->next_frames); j++)
874  {
875  vlib_next_frame_t *nf = &nm_clone->next_frames[j];
876  u32 save_node_runtime_index;
877  u32 save_flags;
878 
879  save_node_runtime_index = nf->node_runtime_index;
880  save_flags = nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH;
882  nf->node_runtime_index = save_node_runtime_index;
883  nf->flags = save_flags;
884  }
885 
886  old_nodes_clone = nm_clone->nodes;
887  nm_clone->nodes = 0;
888 
889  /* re-fork nodes */
890  for (j = 0; j < vec_len (nm->nodes); j++) {
891  vlib_node_t *old_n_clone;
892  vlib_node_t *new_n, *new_n_clone;
893 
894  new_n = nm->nodes[j];
895  old_n_clone = old_nodes_clone[j];
896 
897  new_n_clone = clib_mem_alloc_no_fail (sizeof(*new_n_clone));
898  clib_memcpy (new_n_clone, new_n, sizeof (*new_n));
899  /* none of the copied nodes have enqueue rights given out */
901 
902  if (j >= vec_len (old_nodes_clone))
903  {
904  /* new node, set to zero */
905  memset (&new_n_clone->stats_total, 0,
906  sizeof (new_n_clone->stats_total));
907  memset (&new_n_clone->stats_last_clear, 0,
908  sizeof (new_n_clone->stats_last_clear));
909  }
910  else
911  {
912  /* Copy stats if the old data is valid */
913  clib_memcpy (&new_n_clone->stats_total,
914  &old_n_clone->stats_total,
915  sizeof (new_n_clone->stats_total));
916  clib_memcpy (&new_n_clone->stats_last_clear,
917  &old_n_clone->stats_last_clear,
918  sizeof (new_n_clone->stats_last_clear));
919 
920  /* keep previous node state */
921  new_n_clone->state = old_n_clone->state;
922  }
923  vec_add1 (nm_clone->nodes, new_n_clone);
924  }
925  /* Free the old node clone */
926  for (j = 0; j < vec_len(old_nodes_clone); j++)
927  clib_mem_free (old_nodes_clone[j]);
928  vec_free (old_nodes_clone);
929 
931 
934 
935  /* clone input node runtime */
936  old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT];
937 
940 
942  {
943  rt->cpu_index = vm_clone->cpu_index;
944  }
945 
946  for (j=0; j < vec_len(old_rt); j++)
947  {
948  rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
949  rt->state = old_rt[j].state;
950  }
951 
952  vec_free(old_rt);
953 
954  nm_clone->processes = vec_dup (nm->processes);
955 
956  clib_mem_set_heap (oldheap);
957 
958  // vnet_main_fork_fixup (i);
959  }
960 }
961 
962 static clib_error_t *
964 {
966  uword * p;
968  u8 * name;
969  u64 coremask;
970  uword * bitmap;
971  u32 count;
972 
974  tm->n_thread_stacks = 1; /* account for main thread */
975 
976  tr = tm->next;
977 
978  while (tr)
979  {
981  tr = tr->next;
982  }
983 
985  {
986  if (unformat (input, "main-thread-io"))
987  tm->main_thread_is_io_node = 1;
988  else if (unformat (input, "use-pthreads"))
989  tm->use_pthreads = 1;
990  else if (unformat (input, "thread-prefix %v", &tm->thread_prefix))
991  ;
992  else if (unformat (input, "main-core %u", &tm->main_lcore))
993  ;
994  else if (unformat (input, "skip-cores %u", &tm->skip_cores))
995  ;
996  else if (unformat (input, "coremask-%s %llx", &name, &coremask))
997  {
999  if (p == 0)
1000  return clib_error_return (0, "no such thread type '%s'", name);
1001 
1002  tr = (vlib_thread_registration_t *)p[0];
1003 
1004  if (tr->use_pthreads)
1005  return clib_error_return (0, "coremask cannot be set for '%s' threads",
1006  name);
1007 
1009  (tr->coremask, 0, coremask, BITS(coremask));
1011  }
1012  else if (unformat (input, "corelist-%s %U", &name, unformat_bitmap_list,
1013  &bitmap))
1014  {
1016  if (p == 0)
1017  return clib_error_return (0, "no such thread type '%s'", name);
1018 
1019  tr = (vlib_thread_registration_t *)p[0];
1020 
1021  if (tr->use_pthreads)
1022  return clib_error_return (0, "corelist cannot be set for '%s' threads",
1023  name);
1024 
1025  tr->coremask = bitmap;
1027  }
1028  else if (unformat (input, "%s %u", &name, &count))
1029  {
1031  if (p == 0)
1032  return clib_error_return (0, "no such thread type '%s'", name);
1033 
1034  tr = (vlib_thread_registration_t *)p[0];
1035  if (tr->fixed_count)
1036  return clib_error_return
1037  (0, "number of %s threads not configurable", tr->name);
1038  tr->count = count;
1039  }
1040  else
1041  break;
1042  }
1043 
1044  tr = tm->next;
1045 
1046  if (!tm->thread_prefix)
1047  tm->thread_prefix = format(0, "vpp");
1048 
1049  while (tr)
1050  {
1051  tm->n_thread_stacks += tr->count;
1052  tm->n_pthreads += tr->count * tr->use_pthreads;
1053  tm->n_eal_threads += tr->count * (tr->use_pthreads == 0);
1054  tr = tr->next;
1055  }
1056 
1057  return 0;
1058 }
1059 
1061 
1062 #if !defined (__x86_64__) && !defined (__aarch64__) && !defined (__powerpc64__) && !defined(__arm__)
1063 void __sync_fetch_and_add_8 (void)
1064 {
1065  fformat(stderr, "%s called\n", __FUNCTION__);
1066  abort();
1067 }
1068 void __sync_add_and_fetch_8 (void)
1069 {
1070  fformat(stderr, "%s called\n", __FUNCTION__);
1071  abort();
1072 }
1073 #endif
1074 
1075 void vnet_main_fixup (vlib_fork_fixup_t which) __attribute__ ((weak));
1077 
1079 {
1080  vlib_main_t * vm = vlib_get_main();
1081 
1082  if (vlib_mains == 0)
1083  return;
1084 
1085  ASSERT(os_get_cpu_number() == 0);
1087 
1088  switch (which)
1089  {
1092  break;
1093 
1094  default:
1095  ASSERT(0);
1096  }
1098 }
1099 
1101 {
1102  f64 deadline;
1103  u32 count;
1104 
1105  if (!vlib_mains)
1106  return;
1107 
1108  count = vec_len (vlib_mains) - 1;
1109 
1110  /* Tolerate recursive calls */
1111  if (++vlib_worker_threads[0].recursion_level > 1)
1112  return;
1113 
1115 
1116  ASSERT (os_get_cpu_number() == 0);
1117 
1118  deadline = vlib_time_now (vm) + BARRIER_SYNC_TIMEOUT;
1119 
1121  while (*vlib_worker_threads->workers_at_barrier != count)
1122  {
1123  if (vlib_time_now(vm) > deadline)
1124  {
1125  fformat(stderr, "%s: worker thread deadlock\n", __FUNCTION__);
1126  os_panic();
1127  }
1128  }
1129 }
1130 
1132 {
1133  f64 deadline;
1134 
1135  if (!vlib_mains)
1136  return;
1137 
1138  if (--vlib_worker_threads[0].recursion_level > 0)
1139  return;
1140 
1141  deadline = vlib_time_now (vm) + BARRIER_SYNC_TIMEOUT;
1142 
1144 
1146  {
1147  if (vlib_time_now(vm) > deadline)
1148  {
1149  fformat(stderr, "%s: worker thread deadlock\n", __FUNCTION__);
1150  os_panic();
1151  }
1152  }
1153 }
1154 
1155 static clib_error_t *
1157  unformat_input_t * input,
1158  vlib_cli_command_t * cmd)
1159 {
1161  int i;
1162 
1163  vlib_cli_output (vm, "%-7s%-20s%-12s%-8s%-7s%-7s%-7s%-10s",
1164  "ID", "Name", "Type", "LWP",
1165  "lcore", "Core", "Socket", "State");
1166 
1167 #if !defined(__powerpc64__)
1168  for (i = 0; i < vec_len(vlib_worker_threads); i++)
1169  {
1170  w = vlib_worker_threads + i;
1171  u8 * line = NULL;
1172 
1173  line = format(line, "%-7d%-20s%-12s%-8d",
1174  i,
1175  w->name ? w->name : (u8 *) "",
1176  w->registration ? w->registration->name : "",
1177  w->lwp);
1178 
1179 #if DPDK==1
1180  int lcore = w->dpdk_lcore_id;
1181  if (lcore > -1)
1182  {
1183  line = format(line, "%-7u%-7u%-7u",
1184  lcore,
1185  lcore_config[lcore].core_id,
1186  lcore_config[lcore].socket_id);
1187 
1188  switch(lcore_config[lcore].state)
1189  {
1190  case WAIT:
1191  line = format(line, "wait");
1192  break;
1193  case RUNNING:
1194  line = format(line, "running");
1195  break;
1196  case FINISHED:
1197  line = format(line, "finished");
1198  break;
1199  default:
1200  line = format(line, "unknown");
1201  }
1202  }
1203 #endif
1204  vlib_cli_output(vm, "%v", line);
1205  vec_free(line);
1206  }
1207 #endif
1208 
1209  return 0;
1210 }
1211 
1212 
1213 VLIB_CLI_COMMAND (show_threads_command, static) = {
1214  .path = "show threads",
1215  .short_help = "Show threads",
1216  .function = show_threads_fn,
1217 };
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:394
#define VLIB_THREAD_STACK_SIZE
Definition: threads.h:65
void * vlib_worker_thread_bootstrap_fn(void *arg)
Definition: threads.c:474
always_inline vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:267
uword unformat(unformat_input_t *i, char *fmt,...)
Definition: unformat.c:942
vlib_process_t ** processes
Definition: node.h:610
always_inline uword clib_bitmap_count_set_bits(uword *ai)
Definition: bitmap.h:357
u32 vl(void *p)
Definition: threads.c:49
bad routing header type(not 4)") sr_error (NO_MORE_SEGMENTS
#define VLIB_MAIN_LOOP_ENTER_FUNCTION(x)
Definition: init.h:111
word elog_track_register(elog_main_t *em, elog_track_t *t)
Definition: elog.c:179
static void vlib_worker_thread_barrier_check(void)
Definition: threads.h:201
#define VLIB_LOG2_THREAD_STACK_SIZE
Definition: threads.h:64
always_inline void clib_mem_free(void *p)
Definition: mem.h:149
void * mheap_alloc(void *memory, uword size)
Definition: mheap.c:908
always_inline void unformat_free(unformat_input_t *i)
Definition: format.h:160
#define UNFORMAT_END_OF_INPUT
Definition: format.h:142
static int sort_registrations_by_no_clone(void *a0, void *a1)
Definition: threads.c:94
#define NULL
Definition: clib.h:55
u32 index
Definition: node.h:203
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:531
void os_panic(void)
Definition: unix-misc.c:165
void * thread_function_arg
Definition: threads.h:94
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:480
elog_track_t elog_track
Definition: threads.h:96
#define VLIB_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:327
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:519
vlib_buffer_main_t * buffer_main
Definition: main.h:103
#define hash_set_mem(h, key, value)
Definition: hash.h:257
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:405
struct vlib_thread_registration_ * next
Definition: threads.h:30
#define MHEAP_FLAG_THREAD_SAFE
volatile u32 valid
Definition: threads.h:72
int vlib_frame_queue_enqueue(vlib_main_t *vm, u32 node_runtime_index, u32 frame_queue_index, vlib_frame_t *frame, vlib_frame_queue_msg_type_t type)
always_inline vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
#define clib_bitmap_dup(v)
Definition: bitmap.h:73
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
u8 state
Definition: node.h:231
uword os_get_cpu_number(void)
Definition: threads.c:62
always_inline uword unformat_check_input(unformat_input_t *i)
Definition: format.h:168
u64 * counters_last_clear
Definition: error.h:76
#define VLIB_EFD_DEF_WORKER_HI_THRESH_PCT
Definition: threads.h:244
vlib_thread_registration_t * next
Definition: threads.h:258
vlib_node_stats_t stats_last_clear
Definition: node.h:197
#define clib_smp_atomic_add(addr, increment)
Definition: smp.h:110
#define pool_foreach(VAR, POOL, BODY)
Definition: pool.h:328
always_inline void * clib_mem_get_per_cpu_heap(void)
Definition: mem.h:54
static int vlib_launch_thread(void *fp, vlib_worker_thread_t *w, unsigned lcore_id)
Definition: threads.c:499
#define FRAME_QUEUE_NELTS
Definition: threads.c:36
u16 queue_hi_thresh
Definition: threads.h:249
never_inline void vlib_node_runtime_sync_stats(vlib_main_t *vm, vlib_node_runtime_t *r, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:544
vlib_node_t ** nodes
Definition: node.h:570
int i32
Definition: types.h:81
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
always_inline uword clib_bitmap_first_set(uword *ai)
Definition: bitmap.h:329
u8 * format_hex_bytes(u8 *s, va_list *va)
Definition: std-formats.c:79
uword * lock
Definition: elog.h:156
uword * cpu_core_bitmap
Definition: threads.h:298
u32 cpu_index
Definition: main.h:159
#define clib_warning(format, args...)
Definition: error.h:59
unsigned long u64
Definition: types.h:89
vlib_frame_queue_elt_t * elts
Definition: threads.h:135
void unformat_init_string(unformat_input_t *input, char *string, int string_len)
Definition: unformat.c:991
vlib_node_runtime_t * nodes_by_type[VLIB_N_NODE_TYPE]
Definition: node.h:580
always_inline mheap_t * mheap_header(u8 *v)
always_inline void * vlib_frame_vector_args(vlib_frame_t *f)
Definition: node_funcs.h:202
#define hash_create_string(elts, value_bytes)
Definition: hash.h:609
always_inline void vlib_next_frame_init(vlib_next_frame_t *nf)
Definition: node.h:351
always_inline void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:113
#define clib_bitmap_foreach(i, ai, body)
Definition: bitmap.h:308
void * thread_mheap
Definition: threads.h:91
u32 next_frame_index
Definition: node.h:367
vlib_node_stats_t stats_total
Definition: node.h:193
volatile u64 head
Definition: threads.h:121
void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer.c:1060
#define BARRIER_SYNC_TIMEOUT
Definition: threads.h:177
vlib_error_main_t error_main
Definition: main.h:124
void vlib_frame_free(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_frame_t *f)
Definition: main.c:211
char * name
Definition: elog.h:102
DECLARE_CJ_GLOBAL_LOG
Definition: threads.c:34
#define vec_dup(V)
Return copy of vector (no header, no alignment)
Definition: vec.h:332
vlib_frame_queue_t * vlib_frame_queue_alloc(int nelts)
Definition: threads.c:283
#define VLIB_FRAME_PENDING
Definition: node.h:338
always_inline void * clib_mem_set_heap(void *heap)
Definition: mem.h:190
always_inline uword * clib_bitmap_set(uword *ai, uword i, uword value)
Definition: bitmap.h:132
vlib_frame_queue_t ** vlib_frame_queues
Definition: threads.h:139
u32 main_thread_is_io_node
Definition: threads.h:268
void vlib_worker_thread_fork_fixup(vlib_fork_fixup_t which)
Definition: threads.c:1078
#define VLIB_FRAME_FREE_AFTER_DISPATCH
Definition: node.h:341
always_inline void * clib_mem_get_heap(void)
Definition: mem.h:187
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:538
uword * frame_size_hash
Definition: node.h:625
vlib_fork_fixup_t
Definition: threads.h:194
static clib_error_t * show_threads_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: threads.c:1156
void debug_hex_bytes(u8 *s, u32 n)
Definition: threads.c:54
void(* thread_function)(void *)
Definition: threads.h:93
#define pool_get_aligned(P, E, A)
Definition: pool.h:155
always_inline uword clib_bitmap_get(uword *ai, uword i)
Definition: bitmap.h:158
u64 * counters
Definition: error.h:73
u32 owner_node_index
Definition: node.h:274
void vlib_worker_thread_node_runtime_update(void)
Definition: threads.c:796
volatile u64 tail
Definition: threads.h:110
#define clib_mem_alloc_no_fail(size)
Definition: mem.h:128
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
Definition: init.h:137
always_inline void * clib_mem_alloc(uword size)
Definition: mem.h:109
u16 n_vectors
Definition: node.h:307
u32 node_runtime_index
Definition: node.h:361
vlib_pending_frame_t * pending_frames
Definition: node.h:595
vlib_thread_function_t * function
Definition: threads.h:35
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:298
void * heap_base
Definition: main.h:100
#define clib_memcpy(a, b, c)
Definition: string.h:63
elog_main_t elog_main
Definition: main.h:141
void vnet_main_fixup(vlib_fork_fixup_t which)
Definition: threads.c:1076
always_inline u32 vlib_frame_index(vlib_main_t *vm, vlib_frame_t *f)
Definition: node_funcs.h:184
volatile u32 * wait_at_barrier
Definition: threads.h:86
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:150
always_inline f64 vlib_time_now_ticks(vlib_main_t *vm, u64 n)
Definition: main.h:186
vlib_worker_thread_t * vlib_worker_threads
Definition: threads.h:106
#define never_inline
Definition: clib.h:81
always_inline uword * clib_bitmap_set_multiple(uword *bitmap, uword i, uword value, uword n_bits)
Definition: bitmap.h:226
#define ASSERT(truth)
void vlib_worker_thread_barrier_sync(vlib_main_t *vm)
Definition: threads.c:1100
unsigned int u32
Definition: types.h:88
u8 * format(u8 *s, char *fmt,...)
Definition: format.c:405
u16 flags
Definition: node.h:298
vhost_vring_state_t state
Definition: vhost-user.h:77
#define clib_bitmap_free(v)
Definition: bitmap.h:76
uword * thread_registrations_by_name
Definition: threads.h:263
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1131
static clib_error_t * start_workers(vlib_main_t *vm)
Definition: threads.c:526
clib_error_t * vlib_thread_init(vlib_main_t *vm)
Definition: threads.c:132
u64 uword
Definition: types.h:112
vlib_worker_thread_t * vlib_alloc_thread(vlib_main_t *vm)
Definition: threads.c:269
vlib_frame_queue_msg_type_t
Definition: threads.h:67
void ** mbuf_alloc_list
Definition: main.h:161
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
double f64
Definition: types.h:140
unsigned char u8
Definition: types.h:56
#define VLIB_PENDING_FRAME_NO_NEXT_FRAME
Definition: node.h:370
#define VLIB_EFD_DISABLED
Definition: threads.h:240
vlib_node_main_t node_main
Definition: main.h:115
static uword unformat_bitmap_list(unformat_input_t *input, va_list *va)
Definition: bitmap.h:519
vlib_buffer_free_list_t * buffer_free_list_pool
Definition: buffer.h:295
vlib_next_frame_t * next_frames
Definition: node.h:592
#define vec_sort_with_function(vec, f)
Sort a vector using the supplied element comparison function.
Definition: vec.h:898
static uword * vlib_sysfs_list_to_bitmap(char *filename)
Definition: threads.c:104
vlib_thread_main_t vlib_thread_main
Definition: threads.c:59
word fformat(FILE *f, char *fmt,...)
Definition: format.c:437
static int vlib_frame_queue_dequeue_internal(vlib_main_t *vm)
Definition: threads.c:44
vlib_frame_size_t * frame_sizes
Definition: node.h:628
void vlib_node_sync_stats(vlib_main_t *vm, vlib_node_t *n)
Definition: main.c:577
#define hash_get_mem(h, key)
Definition: hash.h:251
volatile u32 * workers_at_barrier
Definition: threads.h:87
uword clib_calljmp(uword(*func)(uword func_arg), uword func_arg, void *stack)
#define VLIB_INVALID_NODE_INDEX
Definition: node.h:289
always_inline vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Definition: node_funcs.h:61
static clib_error_t * cpu_config(vlib_main_t *vm, unformat_input_t *input)
Definition: threads.c:963
#define vec_foreach(var, vec)
Vector iterator.
void vlib_worker_thread_init(vlib_worker_thread_t *w)
Definition: threads.c:439
always_inline f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:182
int vlib_frame_queue_dequeue(int thread_id, vlib_main_t *vm, vlib_node_main_t *nm)
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:101
u32 node_runtime_index
Definition: node.h:321
void vl_msg_api_handler_no_free(void *)
Definition: threads.c:316
uword * cpu_socket_bitmap
Definition: threads.h:301
#define clib_error_return(e, args...)
Definition: error.h:112
struct _unformat_input_t unformat_input_t
vlib_thread_registration_t ** registrations
Definition: threads.h:261
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
void vlib_set_thread_name(char *name)
Definition: threads.c:85
u8 ** vlib_thread_stacks
Definition: main.h:312
always_inline u64 clib_cpu_time_now(void)
Definition: time.h:71
vlib_thread_registration_t * registration
Definition: threads.h:98
#define BITS(x)
Definition: clib.h:58
clib_random_buffer_t random_buffer
Definition: main.h:153
u16 enabled
Definition: threads.h:248
vlib_efd_t efd
Definition: threads.h:303
vlib_main_t ** vlib_mains
Definition: buffer.c:244