FD.io VPP  v17.07.01-10-g3be13f0
Vector Packet Processing
main.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * main.c: main vector processing loop
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #include <math.h>
41 #include <vppinfra/format.h>
42 #include <vlib/vlib.h>
43 #include <vlib/threads.h>
44 
45 #include <vlib/unix/cj.h>
46 
48 
49 /* Actually allocate a few extra slots of vector data to support
50  speculative vector enqueues which overflow vector data in next frame. */
51 #define VLIB_FRAME_SIZE_ALLOC (VLIB_FRAME_SIZE + 4)
52 
54 
56 vlib_frame_bytes (u32 n_scalar_bytes, u32 n_vector_bytes)
57 {
58  u32 n_bytes;
59 
60  /* Make room for vlib_frame_t plus scalar arguments. */
61  n_bytes = vlib_frame_vector_byte_offset (n_scalar_bytes);
62 
63  /* Make room for vector arguments.
64  Allocate a few extra slots of vector data to support
65  speculative vector enqueues which overflow vector data in next frame. */
66 #define VLIB_FRAME_SIZE_EXTRA 4
67  n_bytes += (VLIB_FRAME_SIZE + VLIB_FRAME_SIZE_EXTRA) * n_vector_bytes;
68 
69  /* Magic number is first 32bit number after vector data.
70  Used to make sure that vector data is never overrun. */
71 #define VLIB_FRAME_MAGIC (0xabadc0ed)
72  n_bytes += sizeof (u32);
73 
74  /* Pad to cache line. */
75  n_bytes = round_pow2 (n_bytes, CLIB_CACHE_LINE_BYTES);
76 
77  return n_bytes;
78 }
79 
82 {
83  void *p = f;
84 
86 
88 
89  return p;
90 }
91 
92 static vlib_frame_size_t *
94  u32 n_scalar_bytes, u32 n_vector_bytes)
95 {
96  uword key = (n_scalar_bytes << 16) | n_vector_bytes;
97  uword *p, i;
98 
99  p = hash_get (nm->frame_size_hash, key);
100  if (p)
101  i = p[0];
102  else
103  {
104  i = vec_len (nm->frame_sizes);
105  vec_validate (nm->frame_sizes, i);
106  hash_set (nm->frame_size_hash, key, i);
107  }
108 
109  return vec_elt_at_index (nm->frame_sizes, i);
110 }
111 
112 static u32
114  u32 frame_flags)
115 {
116  vlib_node_main_t *nm = &vm->node_main;
117  vlib_frame_size_t *fs;
118  vlib_node_t *to_node;
119  vlib_frame_t *f;
120  u32 fi, l, n, scalar_size, vector_size;
121 
122  to_node = vlib_get_node (vm, to_node_index);
123 
124  scalar_size = to_node->scalar_size;
125  vector_size = to_node->vector_size;
126 
127  fs = get_frame_size_info (nm, scalar_size, vector_size);
128  n = vlib_frame_bytes (scalar_size, vector_size);
129  if ((l = vec_len (fs->free_frame_indices)) > 0)
130  {
131  /* Allocate from end of free list. */
132  fi = fs->free_frame_indices[l - 1];
133  f = vlib_get_frame_no_check (vm, fi);
134  _vec_len (fs->free_frame_indices) = l - 1;
135  }
136  else
137  {
139  f->thread_index = vm->thread_index;
140  fi = vlib_frame_index_no_check (vm, f);
141  }
142 
143  /* Poison frame when debugging. */
144  if (CLIB_DEBUG > 0)
145  {
146  u32 save_thread_index = f->thread_index;
147 
148  memset (f, 0xfe, n);
149 
150  f->thread_index = save_thread_index;
151  }
152 
153  /* Insert magic number. */
154  {
155  u32 *magic;
156 
157  magic = vlib_frame_find_magic (f, to_node);
158  *magic = VLIB_FRAME_MAGIC;
159  }
160 
161  f->flags = VLIB_FRAME_IS_ALLOCATED | frame_flags;
162  f->n_vectors = 0;
163  f->scalar_size = scalar_size;
164  f->vector_size = vector_size;
165 
166  fs->n_alloc_frames += 1;
167 
168  return fi;
169 }
170 
171 /* Allocate a frame for from FROM_NODE to TO_NODE via TO_NEXT_INDEX.
172  Returns frame index. */
173 static u32
175  u32 to_next_index)
176 {
177  vlib_node_t *from_node;
178 
179  from_node = vlib_get_node (vm, from_node_runtime->node_index);
180  ASSERT (to_next_index < vec_len (from_node->next_nodes));
181 
182  return vlib_frame_alloc_to_node (vm, from_node->next_nodes[to_next_index],
183  /* frame_flags */ 0);
184 }
185 
186 vlib_frame_t *
188 {
189  u32 fi = vlib_frame_alloc_to_node (vm, to_node_index,
190  /* frame_flags */
192  return vlib_get_frame (vm, fi);
193 }
194 
195 void
197 {
199  vlib_node_t *to_node;
200 
201  if (f->n_vectors == 0)
202  return;
203 
204  to_node = vlib_get_node (vm, to_node_index);
205 
206  vec_add2 (vm->node_main.pending_frames, p, 1);
207 
209  p->frame_index = vlib_frame_index (vm, f);
210  p->node_runtime_index = to_node->runtime_index;
212 }
213 
214 /* Free given frame. */
215 void
217 {
218  vlib_node_main_t *nm = &vm->node_main;
219  vlib_node_t *node;
220  vlib_frame_size_t *fs;
221  u32 frame_index;
222 
224 
225  node = vlib_get_node (vm, r->node_index);
226  fs = get_frame_size_info (nm, node->scalar_size, node->vector_size);
227 
228  frame_index = vlib_frame_index (vm, f);
229 
231 
232  /* No next frames may point to freed frame. */
233  if (CLIB_DEBUG > 0)
234  {
235  vlib_next_frame_t *nf;
237  ASSERT (nf->frame_index != frame_index);
238  }
239 
241 
242  vec_add1 (fs->free_frame_indices, frame_index);
243  ASSERT (fs->n_alloc_frames > 0);
244  fs->n_alloc_frames -= 1;
245 }
246 
247 static clib_error_t *
250 {
251  vlib_node_main_t *nm = &vm->node_main;
252  vlib_frame_size_t *fs;
253 
254  vlib_cli_output (vm, "%=6s%=12s%=12s", "Size", "# Alloc", "# Free");
255  vec_foreach (fs, nm->frame_sizes)
256  {
257  u32 n_alloc = fs->n_alloc_frames;
258  u32 n_free = vec_len (fs->free_frame_indices);
259 
260  if (n_alloc + n_free > 0)
261  vlib_cli_output (vm, "%=6d%=12d%=12d",
262  fs - nm->frame_sizes, n_alloc, n_free);
263  }
264 
265  return 0;
266 }
267 
268 /* *INDENT-OFF* */
269 VLIB_CLI_COMMAND (show_frame_stats_cli, static) = {
270  .path = "show vlib frame-allocation",
271  .short_help = "Show node dispatch frame statistics",
272  .function = show_frame_stats,
273 };
274 /* *INDENT-ON* */
275 
276 /* Change ownership of enqueue rights to given next node. */
277 static void
279  vlib_node_runtime_t * node_runtime,
280  u32 next_index)
281 {
282  vlib_node_main_t *nm = &vm->node_main;
283  vlib_next_frame_t *next_frame;
284  vlib_node_t *node, *next_node;
285 
286  node = vec_elt (nm->nodes, node_runtime->node_index);
287 
288  /* Only internal & input nodes are allowed to call other nodes. */
290  || node->type == VLIB_NODE_TYPE_INPUT
291  || node->type == VLIB_NODE_TYPE_PROCESS);
292 
293  ASSERT (vec_len (node->next_nodes) == node_runtime->n_next_nodes);
294 
295  next_frame =
296  vlib_node_runtime_get_next_frame (vm, node_runtime, next_index);
297  next_node = vec_elt (nm->nodes, node->next_nodes[next_index]);
298 
299  if (next_node->owner_node_index != VLIB_INVALID_NODE_INDEX)
300  {
301  /* Get frame from previous owner. */
302  vlib_next_frame_t *owner_next_frame;
303  vlib_next_frame_t tmp;
304 
305  owner_next_frame =
307  next_node->owner_node_index,
308  next_node->owner_next_index);
309 
310  /* Swap target next frame with owner's. */
311  tmp = owner_next_frame[0];
312  owner_next_frame[0] = next_frame[0];
313  next_frame[0] = tmp;
314 
315  /*
316  * If next_frame is already pending, we have to track down
317  * all pending frames and fix their next_frame_index fields.
318  */
319  if (next_frame->flags & VLIB_FRAME_PENDING)
320  {
322  if (next_frame->frame_index != ~0)
323  {
324  vec_foreach (p, nm->pending_frames)
325  {
326  if (p->frame_index == next_frame->frame_index)
327  {
328  p->next_frame_index =
329  next_frame - vm->node_main.next_frames;
330  }
331  }
332  }
333  }
334  }
335  else
336  {
337  /* No previous owner. Take ownership. */
338  next_frame->flags |= VLIB_FRAME_OWNER;
339  }
340 
341  /* Record new owner. */
342  next_node->owner_node_index = node->index;
343  next_node->owner_next_index = next_index;
344 
345  /* Now we should be owner. */
346  ASSERT (next_frame->flags & VLIB_FRAME_OWNER);
347 }
348 
349 /* Make sure that magic number is still there.
350  Otherwise, it is likely that caller has overrun frame arguments. */
351 always_inline void
353  vlib_frame_t * f, vlib_node_t * n, uword next_index)
354 {
355  vlib_node_t *next_node = vlib_get_node (vm, n->next_nodes[next_index]);
356  u32 *magic = vlib_frame_find_magic (f, next_node);
357  ASSERT (VLIB_FRAME_MAGIC == magic[0]);
358 }
359 
360 vlib_frame_t *
362  vlib_node_runtime_t * node,
363  u32 next_index, u32 allocate_new_next_frame)
364 {
365  vlib_frame_t *f;
366  vlib_next_frame_t *nf;
367  u32 n_used;
368 
369  nf = vlib_node_runtime_get_next_frame (vm, node, next_index);
370 
371  /* Make sure this next frame owns right to enqueue to destination frame. */
372  if (PREDICT_FALSE (!(nf->flags & VLIB_FRAME_OWNER)))
373  vlib_next_frame_change_ownership (vm, node, next_index);
374 
375  /* ??? Don't need valid flag: can use frame_index == ~0 */
377  {
378  nf->frame_index = vlib_frame_alloc (vm, node, next_index);
380  }
381 
382  f = vlib_get_frame (vm, nf->frame_index);
383 
384  /* Has frame been removed from pending vector (e.g. finished dispatching)?
385  If so we can reuse frame. */
386  if ((nf->flags & VLIB_FRAME_PENDING) && !(f->flags & VLIB_FRAME_PENDING))
387  {
388  nf->flags &= ~VLIB_FRAME_PENDING;
389  f->n_vectors = 0;
390  }
391 
392  /* Allocate new frame if current one is already full. */
393  n_used = f->n_vectors;
394  if (n_used >= VLIB_FRAME_SIZE || (allocate_new_next_frame && n_used > 0))
395  {
396  /* Old frame may need to be freed after dispatch, since we'll have
397  two redundant frames from node -> next node. */
399  {
400  vlib_frame_t *f_old = vlib_get_frame (vm, nf->frame_index);
402  }
403 
404  /* Allocate new frame to replace full one. */
405  nf->frame_index = vlib_frame_alloc (vm, node, next_index);
406  f = vlib_get_frame (vm, nf->frame_index);
407  n_used = f->n_vectors;
408  }
409 
410  /* Should have free vectors in frame now. */
411  ASSERT (n_used < VLIB_FRAME_SIZE);
412 
413  if (CLIB_DEBUG > 0)
414  {
415  validate_frame_magic (vm, f,
416  vlib_get_node (vm, node->node_index), next_index);
417  }
418 
419  return f;
420 }
421 
422 static void
424  vlib_node_runtime_t * rt,
425  u32 next_index, u32 n_vectors_left)
426 {
427  vlib_node_main_t *nm = &vm->node_main;
428  vlib_next_frame_t *nf;
429  vlib_frame_t *f;
430  vlib_node_runtime_t *next_rt;
431  vlib_node_t *next_node;
432  u32 n_before, n_after;
433 
434  nf = vlib_node_runtime_get_next_frame (vm, rt, next_index);
435  f = vlib_get_frame (vm, nf->frame_index);
436 
437  ASSERT (n_vectors_left <= VLIB_FRAME_SIZE);
438  n_after = VLIB_FRAME_SIZE - n_vectors_left;
439  n_before = f->n_vectors;
440 
441  ASSERT (n_after >= n_before);
442 
444  nf->node_runtime_index);
445  next_node = vlib_get_node (vm, next_rt->node_index);
446  if (n_after > 0 && next_node->validate_frame)
447  {
448  u8 *msg = next_node->validate_frame (vm, rt, f);
449  if (msg)
450  {
451  clib_warning ("%v", msg);
452  ASSERT (0);
453  }
454  vec_free (msg);
455  }
456 }
457 
458 void
461  u32 next_index, u32 n_vectors_left)
462 {
463  vlib_node_main_t *nm = &vm->node_main;
464  vlib_next_frame_t *nf;
465  vlib_frame_t *f;
466  u32 n_vectors_in_frame;
467 
468  if (vm->buffer_main->extern_buffer_mgmt == 0 && CLIB_DEBUG > 0)
469  vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left);
470 
471  nf = vlib_node_runtime_get_next_frame (vm, r, next_index);
472  f = vlib_get_frame (vm, nf->frame_index);
473 
474  /* Make sure that magic number is still there. Otherwise, caller
475  has overrun frame meta data. */
476  if (CLIB_DEBUG > 0)
477  {
478  vlib_node_t *node = vlib_get_node (vm, r->node_index);
479  validate_frame_magic (vm, f, node, next_index);
480  }
481 
482  /* Convert # of vectors left -> number of vectors there. */
483  ASSERT (n_vectors_left <= VLIB_FRAME_SIZE);
484  n_vectors_in_frame = VLIB_FRAME_SIZE - n_vectors_left;
485 
486  f->n_vectors = n_vectors_in_frame;
487 
488  /* If vectors were added to frame, add to pending vector. */
489  if (PREDICT_TRUE (n_vectors_in_frame > 0))
490  {
492  u32 v0, v1;
493 
494  r->cached_next_index = next_index;
495 
496  if (!(f->flags & VLIB_FRAME_PENDING))
497  {
498  __attribute__ ((unused)) vlib_node_t *node;
499  vlib_node_t *next_node;
500  vlib_node_runtime_t *next_runtime;
501 
502  node = vlib_get_node (vm, r->node_index);
503  next_node = vlib_get_next_node (vm, r->node_index, next_index);
504  next_runtime = vlib_node_get_runtime (vm, next_node->index);
505 
506  vec_add2 (nm->pending_frames, p, 1);
507 
508  p->frame_index = nf->frame_index;
510  p->next_frame_index = nf - nm->next_frames;
511  nf->flags |= VLIB_FRAME_PENDING;
513 
514  /*
515  * If we're going to dispatch this frame on another thread,
516  * force allocation of a new frame. Otherwise, we create
517  * a dangling frame reference. Each thread has its own copy of
518  * the next_frames vector.
519  */
520  if (0 && r->thread_index != next_runtime->thread_index)
521  {
522  nf->frame_index = ~0;
524  }
525  }
526 
527  /* Copy trace flag from next_frame and from runtime. */
528  nf->flags |=
529  (nf->flags & VLIB_NODE_FLAG_TRACE) | (r->
531 
533  v1 = v0 + n_vectors_in_frame;
535  if (PREDICT_FALSE (v1 < v0))
536  {
537  vlib_node_t *node = vlib_get_node (vm, r->node_index);
538  vec_elt (node->n_vectors_by_next_node, next_index) += v0;
539  }
540  }
541 }
542 
543 /* Sync up runtime (32 bit counters) and main node stats (64 bit counters). */
544 never_inline void
547  uword n_calls, uword n_vectors, uword n_clocks)
548 {
549  vlib_node_t *n = vlib_get_node (vm, r->node_index);
550 
551  n->stats_total.calls += n_calls + r->calls_since_last_overflow;
552  n->stats_total.vectors += n_vectors + r->vectors_since_last_overflow;
553  n->stats_total.clocks += n_clocks + r->clocks_since_last_overflow;
556 
560 }
561 
562 always_inline void __attribute__ ((unused))
564  vlib_process_t * p,
565  uword n_calls, uword n_vectors, uword n_clocks)
566 {
567  vlib_node_runtime_t *rt = &p->node_runtime;
568  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
569  vlib_node_runtime_sync_stats (vm, rt, n_calls, n_vectors, n_clocks);
570  n->stats_total.suspends += p->n_suspends;
571  p->n_suspends = 0;
572 }
573 
574 void
576 {
578 
579  if (n->type == VLIB_NODE_TYPE_PROCESS)
580  {
581  /* Nothing to do for PROCESS nodes except in main thread */
582  if (vm != &vlib_global_main)
583  return;
584 
587  p->n_suspends = 0;
588  rt = &p->node_runtime;
589  }
590  else
591  rt =
593  n->runtime_index);
594 
595  vlib_node_runtime_sync_stats (vm, rt, 0, 0, 0);
596 
597  /* Sync up runtime next frame vector counters with main node structure. */
598  {
599  vlib_next_frame_t *nf;
600  uword i;
601  for (i = 0; i < rt->n_next_nodes; i++)
602  {
603  nf = vlib_node_runtime_get_next_frame (vm, rt, i);
607  }
608  }
609 }
610 
613  vlib_node_runtime_t * node,
614  uword n_calls,
615  uword n_vectors, uword n_clocks)
616 {
617  u32 ca0, ca1, v0, v1, cl0, cl1, r;
618 
619  cl0 = cl1 = node->clocks_since_last_overflow;
620  ca0 = ca1 = node->calls_since_last_overflow;
621  v0 = v1 = node->vectors_since_last_overflow;
622 
623  ca1 = ca0 + n_calls;
624  v1 = v0 + n_vectors;
625  cl1 = cl0 + n_clocks;
626 
627  node->calls_since_last_overflow = ca1;
628  node->clocks_since_last_overflow = cl1;
629  node->vectors_since_last_overflow = v1;
630  node->max_clock_n = node->max_clock > n_clocks ?
631  node->max_clock_n : n_vectors;
632  node->max_clock = node->max_clock > n_clocks ? node->max_clock : n_clocks;
633 
634  r = vlib_node_runtime_update_main_loop_vector_stats (vm, node, n_vectors);
635 
636  if (PREDICT_FALSE (ca1 < ca0 || v1 < v0 || cl1 < cl0))
637  {
638  node->calls_since_last_overflow = ca0;
639  node->clocks_since_last_overflow = cl0;
640  node->vectors_since_last_overflow = v0;
641  vlib_node_runtime_sync_stats (vm, node, n_calls, n_vectors, n_clocks);
642  }
643 
644  return r;
645 }
646 
647 always_inline void
649  vlib_process_t * p,
650  uword n_calls, uword n_vectors, uword n_clocks)
651 {
653  n_calls, n_vectors, n_clocks);
654 }
655 
656 static clib_error_t *
659 {
661  return 0;
662 }
663 
664 /* *INDENT-OFF* */
665 VLIB_CLI_COMMAND (elog_clear_cli, static) = {
666  .path = "event-logger clear",
667  .short_help = "Clear the event log",
668  .function = vlib_cli_elog_clear,
669 };
670 /* *INDENT-ON* */
671 
672 #ifdef CLIB_UNIX
673 static clib_error_t *
676 {
677  elog_main_t *em = &vm->elog_main;
678  char *file, *chroot_file;
679  clib_error_t *error = 0;
680 
681  if (!unformat (input, "%s", &file))
682  {
683  vlib_cli_output (vm, "expected file name, got `%U'",
684  format_unformat_error, input);
685  return 0;
686  }
687 
688  /* It's fairly hard to get "../oopsie" through unformat; just in case */
689  if (strstr (file, "..") || index (file, '/'))
690  {
691  vlib_cli_output (vm, "illegal characters in filename '%s'", file);
692  return 0;
693  }
694 
695  chroot_file = (char *) format (0, "/tmp/%s%c", file, 0);
696 
697  vec_free (file);
698 
699  vlib_cli_output (vm, "Saving %wd of %wd events to %s",
701  elog_buffer_capacity (em), chroot_file);
702 
704  error = elog_write_file (em, chroot_file, 1 /* flush ring */ );
706  vec_free (chroot_file);
707  return error;
708 }
709 
710 void
712 {
714  elog_main_t *em = &vm->elog_main;
715  u8 *filename;
716  clib_error_t *error;
717 
718  if (!vm->elog_post_mortem_dump)
719  return;
720 
721  filename = format (0, "/tmp/elog_post_mortem.%d%c", getpid (), 0);
722  error = elog_write_file (em, (char *) filename, 1 /* flush ring */ );
723  if (error)
724  clib_error_report (error);
725  vec_free (filename);
726 }
727 
728 /* *INDENT-OFF* */
729 VLIB_CLI_COMMAND (elog_save_cli, static) = {
730  .path = "event-logger save",
731  .short_help = "event-logger save <filename> (saves log in /tmp/<filename>)",
732  .function = elog_save_buffer,
733 };
734 /* *INDENT-ON* */
735 
736 static clib_error_t *
739 {
740  elog_main_t *em = &vm->elog_main;
741 
743 
744  vlib_cli_output (vm, "Stopped the event logger...");
745  return 0;
746 }
747 
748 /* *INDENT-OFF* */
749 VLIB_CLI_COMMAND (elog_stop_cli, static) = {
750  .path = "event-logger stop",
751  .short_help = "Stop the event-logger",
752  .function = elog_stop,
753 };
754 /* *INDENT-ON* */
755 
756 static clib_error_t *
759 {
760  elog_main_t *em = &vm->elog_main;
761 
763 
764  vlib_cli_output (vm, "Restarted the event logger...");
765  return 0;
766 }
767 
768 /* *INDENT-OFF* */
769 VLIB_CLI_COMMAND (elog_restart_cli, static) = {
770  .path = "event-logger restart",
771  .short_help = "Restart the event-logger",
772  .function = elog_restart,
773 };
774 /* *INDENT-ON* */
775 
776 static clib_error_t *
779 {
780  elog_main_t *em = &vm->elog_main;
781  u32 tmp;
782 
783  /* Stop the parade */
785 
786  if (unformat (input, "%d", &tmp))
787  {
788  elog_alloc (em, tmp);
790  }
791  else
792  return clib_error_return (0, "Must specify how many events in the ring");
793 
794  vlib_cli_output (vm, "Resized ring and restarted the event logger...");
795  return 0;
796 }
797 
798 /* *INDENT-OFF* */
799 VLIB_CLI_COMMAND (elog_resize_cli, static) = {
800  .path = "event-logger resize",
801  .short_help = "event-logger resize <nnn>",
802  .function = elog_resize,
803 };
804 /* *INDENT-ON* */
805 
806 #endif /* CLIB_UNIX */
807 
808 static void
809 elog_show_buffer_internal (vlib_main_t * vm, u32 n_events_to_show)
810 {
811  elog_main_t *em = &vm->elog_main;
812  elog_event_t *e, *es;
813  f64 dt;
814 
815  /* Show events in VLIB time since log clock starts after VLIB clock. */
816  dt = (em->init_time.cpu - vm->clib_time.init_cpu_time)
818 
819  es = elog_peek_events (em);
820  vlib_cli_output (vm, "%d of %d events in buffer, logger %s", vec_len (es),
821  em->event_ring_size,
823  "running" : "stopped");
824  vec_foreach (e, es)
825  {
826  vlib_cli_output (vm, "%18.9f: %U",
827  e->time + dt, format_elog_event, em, e);
828  n_events_to_show--;
829  if (n_events_to_show == 0)
830  break;
831  }
832  vec_free (es);
833 
834 }
835 
836 static clib_error_t *
839 {
840  u32 n_events_to_show;
841  clib_error_t *error = 0;
842 
843  n_events_to_show = 250;
845  {
846  if (unformat (input, "%d", &n_events_to_show))
847  ;
848  else if (unformat (input, "all"))
849  n_events_to_show = ~0;
850  else
851  return unformat_parse_error (input);
852  }
853  elog_show_buffer_internal (vm, n_events_to_show);
854  return error;
855 }
856 
857 /* *INDENT-OFF* */
858 VLIB_CLI_COMMAND (elog_show_cli, static) = {
859  .path = "show event-logger",
860  .short_help = "Show event logger info",
861  .function = elog_show_buffer,
862 };
863 /* *INDENT-ON* */
864 
865 void
867 {
869 }
870 
871 static inline void
873  u32 node_index,
874  u64 time, u32 n_vectors, u32 is_return)
875 {
877  elog_main_t *em = &evm->elog_main;
878 
879  if (VLIB_ELOG_MAIN_LOOP && n_vectors)
880  elog_track (em,
881  /* event type */
882  vec_elt_at_index (is_return
885  node_index),
886  /* track */
888  elog_track : &em->default_track),
889  /* data to log */ n_vectors);
890 }
891 
892 void
894 {
895  vlib_node_main_t *vnm = &vm->node_main;
896  vlib_buffer_t *b;
897  u8 i, n;
898 
900  {
901  b = vlib_get_buffer (vm, bi);
902  n = b->pre_data[0];
903 
904  fformat (stderr, "Context trace for bi %d b 0x%llx, visited %d\n",
905  bi, b, n);
906 
907  if (n == 0 || n > 20)
908  {
909  fformat (stderr, "n is unreasonable\n");
910  return;
911  }
912 
913 
914  for (i = 0; i < n; i++)
915  {
916  u32 node_index;
917 
918  node_index = b->pre_data[i + 1];
919 
920  if (node_index > vec_len (vnm->nodes))
921  {
922  fformat (stderr, "Skip bogus node index %d\n", node_index);
923  continue;
924  }
925 
926  fformat (stderr, "%v (%d)\n", vnm->nodes[node_index]->name,
927  node_index);
928  }
929  }
930  else
931  {
932  fformat (stderr,
933  "in vlib/buffers.h, #define VLIB_BUFFER_TRACE_TRAJECTORY 1\n");
934  }
935 }
936 
937 
940  vlib_node_runtime_t * node,
941  vlib_node_type_t type,
942  vlib_node_state_t dispatch_state,
943  vlib_frame_t * frame, u64 last_time_stamp)
944 {
945  uword n, v;
946  u64 t;
947  vlib_node_main_t *nm = &vm->node_main;
948  vlib_next_frame_t *nf;
949 
950  if (CLIB_DEBUG > 0)
951  {
952  vlib_node_t *n = vlib_get_node (vm, node->node_index);
953  ASSERT (n->type == type);
954  }
955 
956  /* Only non-internal nodes may be disabled. */
957  if (type != VLIB_NODE_TYPE_INTERNAL && node->state != dispatch_state)
958  {
960  return last_time_stamp;
961  }
962 
963  if ((type == VLIB_NODE_TYPE_PRE_INPUT || type == VLIB_NODE_TYPE_INPUT)
964  && dispatch_state != VLIB_NODE_STATE_INTERRUPT)
965  {
967  /* Only call node when count reaches zero. */
968  if (c)
969  {
970  node->input_main_loops_per_call = c - 1;
971  return last_time_stamp;
972  }
973  }
974 
975  /* Speculatively prefetch next frames. */
976  if (node->n_next_nodes > 0)
977  {
979  CLIB_PREFETCH (nf, 4 * sizeof (nf[0]), WRITE);
980  }
981 
982  vm->cpu_time_last_node_dispatch = last_time_stamp;
983 
984  if (1 /* || vm->thread_index == node->thread_index */ )
985  {
986  vlib_main_t *stat_vm;
987 
988  stat_vm = /* vlib_mains ? vlib_mains[0] : */ vm;
989 
991  last_time_stamp,
992  frame ? frame->n_vectors : 0,
993  /* is_after */ 0);
994 
995  /*
996  * Turn this on if you run into
997  * "bad monkey" contexts, and you want to know exactly
998  * which nodes they've visited... See ixge.c...
999  */
1000  if (VLIB_BUFFER_TRACE_TRAJECTORY && frame)
1001  {
1002  int i;
1003  int log_index;
1004  u32 *from;
1005  from = vlib_frame_vector_args (frame);
1006  for (i = 0; i < frame->n_vectors; i++)
1007  {
1008  vlib_buffer_t *b = vlib_get_buffer (vm, from[i]);
1009  ASSERT (b->pre_data[0] < 32);
1010  log_index = b->pre_data[0]++ + 1;
1011  b->pre_data[log_index] = node->node_index;
1012  }
1013  n = node->function (vm, node, frame);
1014  }
1015  else
1016  n = node->function (vm, node, frame);
1017 
1018  t = clib_cpu_time_now ();
1019 
1020  vlib_elog_main_loop_event (vm, node->node_index, t, n, /* is_after */
1021  1);
1022 
1023  vm->main_loop_vectors_processed += n;
1024  vm->main_loop_nodes_processed += n > 0;
1025 
1026  v = vlib_node_runtime_update_stats (stat_vm, node,
1027  /* n_calls */ 1,
1028  /* n_vectors */ n,
1029  /* n_clocks */ t - last_time_stamp);
1030 
1031  /* When in interrupt mode and vector rate crosses threshold switch to
1032  polling mode. */
1033  if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT)
1034  || (dispatch_state == VLIB_NODE_STATE_POLLING
1035  && (node->flags
1037  {
1038 #ifdef DISPATCH_NODE_ELOG_REQUIRED
1039  ELOG_TYPE_DECLARE (e) =
1040  {
1041  .function = (char *) __FUNCTION__,.format =
1042  "%s vector length %d, switching to %s",.format_args =
1043  "T4i4t4",.n_enum_strings = 2,.enum_strings =
1044  {
1045  "interrupt", "polling",},};
1046  struct
1047  {
1048  u32 node_name, vector_length, is_polling;
1049  } *ed;
1051 #endif
1052 
1053  if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT
1054  && v >= nm->polling_threshold_vector_length) &&
1055  !(node->flags &
1057  {
1058  vlib_node_t *n = vlib_get_node (vm, node->node_index);
1059  n->state = VLIB_NODE_STATE_POLLING;
1060  node->state = VLIB_NODE_STATE_POLLING;
1061  node->flags &=
1063  node->flags |=
1065  nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] -= 1;
1066  nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] += 1;
1067 
1068 #ifdef DISPATCH_NODE_ELOG_REQUIRED
1069  ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
1070  w->elog_track);
1071  ed->node_name = n->name_elog_string;
1072  ed->vector_length = v;
1073  ed->is_polling = 1;
1074 #endif
1075  }
1076  else if (dispatch_state == VLIB_NODE_STATE_POLLING
1077  && v <= nm->interrupt_threshold_vector_length)
1078  {
1079  vlib_node_t *n = vlib_get_node (vm, node->node_index);
1080  if (node->flags &
1082  {
1083  /* Switch to interrupt mode after dispatch in polling one more time.
1084  This allows driver to re-enable interrupts. */
1085  n->state = VLIB_NODE_STATE_INTERRUPT;
1086  node->state = VLIB_NODE_STATE_INTERRUPT;
1087  node->flags &=
1089  nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] -=
1090  1;
1091  nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] +=
1092  1;
1093 
1094  }
1095  else
1096  {
1097  node->flags |=
1099 #ifdef DISPATCH_NODE_ELOG_REQUIRED
1100  ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
1101  w->elog_track);
1102  ed->node_name = n->name_elog_string;
1103  ed->vector_length = v;
1104  ed->is_polling = 0;
1105 #endif
1106  }
1107  }
1108  }
1109  }
1110 
1111  return t;
1112 }
1113 
1114 static u64
1115 dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index,
1116  u64 last_time_stamp)
1117 {
1118  vlib_node_main_t *nm = &vm->node_main;
1119  vlib_frame_t *f;
1120  vlib_next_frame_t *nf, nf_dummy;
1122  u32 restore_frame_index;
1124 
1125  /* See comment below about dangling references to nm->pending_frames */
1126  p = nm->pending_frames + pending_frame_index;
1127 
1129  p->node_runtime_index);
1130 
1131  f = vlib_get_frame (vm, p->frame_index);
1133  {
1134  /* No next frame: so use dummy on stack. */
1135  nf = &nf_dummy;
1136  nf->flags = f->flags & VLIB_NODE_FLAG_TRACE;
1137  nf->frame_index = ~p->frame_index;
1138  }
1139  else
1141 
1143 
1144  /* Force allocation of new frame while current frame is being
1145  dispatched. */
1146  restore_frame_index = ~0;
1147  if (nf->frame_index == p->frame_index)
1148  {
1149  nf->frame_index = ~0;
1152  restore_frame_index = p->frame_index;
1153  }
1154 
1155  /* Frame must be pending. */
1157  ASSERT (f->n_vectors > 0);
1158 
1159  /* Copy trace flag from next frame to node.
1160  Trace flag indicates that at least one vector in the dispatched
1161  frame is traced. */
1162  n->flags &= ~VLIB_NODE_FLAG_TRACE;
1163  n->flags |= (nf->flags & VLIB_FRAME_TRACE) ? VLIB_NODE_FLAG_TRACE : 0;
1164  nf->flags &= ~VLIB_FRAME_TRACE;
1165 
1166  last_time_stamp = dispatch_node (vm, n,
1168  VLIB_NODE_STATE_POLLING,
1169  f, last_time_stamp);
1170 
1171  f->flags &= ~VLIB_FRAME_PENDING;
1172 
1173  /* Frame is ready to be used again, so restore it. */
1174  if (restore_frame_index != ~0)
1175  {
1176  /*
1177  * We musn't restore a frame that is flagged to be freed. This
1178  * shouldn't happen since frames to be freed post dispatch are
1179  * those used when the to-node frame becomes full i.e. they form a
1180  * sort of queue of frames to a single node. If we get here then
1181  * the to-node frame and the pending frame *were* the same, and so
1182  * we removed the to-node frame. Therefore this frame is no
1183  * longer part of the queue for that node and hence it cannot be
1184  * it's overspill.
1185  */
1187 
1188  /*
1189  * NB: dispatching node n can result in the creation and scheduling
1190  * of new frames, and hence in the reallocation of nm->pending_frames.
1191  * Recompute p, or no supper. This was broken for more than 10 years.
1192  */
1193  p = nm->pending_frames + pending_frame_index;
1194 
1195  /*
1196  * p->next_frame_index can change during node dispatch if node
1197  * function decides to change graph hook up.
1198  */
1201 
1202  if (~0 == nf->frame_index)
1203  {
1204  /* no new frame has been assigned to this node, use the saved one */
1205  nf->frame_index = restore_frame_index;
1206  f->n_vectors = 0;
1207  }
1208  else
1209  {
1210  /* The node has gained a frame, implying packets from the current frame
1211  were re-queued to this same node. we don't need the saved one
1212  anymore */
1213  vlib_frame_free (vm, n, f);
1214  }
1215  }
1216  else
1217  {
1219  {
1221  vlib_frame_free (vm, n, f);
1222  }
1223  }
1224 
1225  return last_time_stamp;
1226 }
1227 
1230 {
1231  return p->stack[0] == VLIB_PROCESS_STACK_MAGIC;
1232 }
1233 
1234 typedef struct
1235 {
1240 
1241 /* Called in process stack. */
1242 static uword
1244 {
1246  vlib_main_t *vm;
1247  vlib_node_runtime_t *node;
1248  vlib_frame_t *f;
1249  vlib_process_t *p;
1250  uword n;
1251 
1253 
1254  vm = a->vm;
1255  p = a->process;
1256  f = a->frame;
1257  node = &p->node_runtime;
1258 
1259  n = node->function (vm, node, f);
1260 
1262 
1263  clib_longjmp (&p->return_longjmp, n);
1264 
1265  return n;
1266 }
1267 
1268 /* Called in main stack. */
1271 {
1273  uword r;
1274 
1275  a.vm = vm;
1276  a.process = p;
1277  a.frame = f;
1278 
1282  (void *) p->stack + (1 << p->log2_n_stack_bytes));
1283 
1284  return r;
1285 }
1286 
1289 {
1290  uword r;
1297  return r;
1298 }
1299 
1300 static u64
1302  vlib_process_t * p, vlib_frame_t * f, u64 last_time_stamp)
1303 {
1304  vlib_node_main_t *nm = &vm->node_main;
1305  vlib_node_runtime_t *node_runtime = &p->node_runtime;
1306  vlib_node_t *node = vlib_get_node (vm, node_runtime->node_index);
1307  u64 t;
1308  uword n_vectors, is_suspend;
1309 
1310  if (node->state != VLIB_NODE_STATE_POLLING
1313  return last_time_stamp;
1314 
1316 
1317  t = last_time_stamp;
1318  vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
1319  f ? f->n_vectors : 0, /* is_after */ 0);
1320 
1321  /* Save away current process for suspend. */
1323 
1324  n_vectors = vlib_process_startup (vm, p, f);
1325 
1326  nm->current_process_index = ~0;
1327 
1329  is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND;
1330  if (is_suspend)
1331  {
1333 
1334  n_vectors = 0;
1336  pf->node_runtime_index = node->runtime_index;
1337  pf->frame_index = f ? vlib_frame_index (vm, f) : ~0;
1338  pf->next_frame_index = ~0;
1339 
1340  p->n_suspends += 1;
1342 
1346  (node->runtime_index));
1347  }
1348  else
1350 
1351  t = clib_cpu_time_now ();
1352 
1353  vlib_elog_main_loop_event (vm, node_runtime->node_index, t, is_suspend,
1354  /* is_after */ 1);
1355 
1357  /* n_calls */ !is_suspend,
1358  /* n_vectors */ n_vectors,
1359  /* n_clocks */ t - last_time_stamp);
1360 
1361  return t;
1362 }
1363 
1364 void
1365 vlib_start_process (vlib_main_t * vm, uword process_index)
1366 {
1367  vlib_node_main_t *nm = &vm->node_main;
1368  vlib_process_t *p = vec_elt (nm->processes, process_index);
1369  dispatch_process (vm, p, /* frame */ 0, /* cpu_time_now */ 0);
1370 }
1371 
1372 static u64
1374  uword process_index, u64 last_time_stamp)
1375 {
1376  vlib_node_main_t *nm = &vm->node_main;
1377  vlib_node_runtime_t *node_runtime;
1378  vlib_node_t *node;
1379  vlib_frame_t *f;
1380  vlib_process_t *p;
1382  u64 t, n_vectors, is_suspend;
1383 
1384  t = last_time_stamp;
1385 
1386  p = vec_elt (nm->processes, process_index);
1388  return last_time_stamp;
1389 
1392 
1393  pf =
1396 
1397  node_runtime = &p->node_runtime;
1398  node = vlib_get_node (vm, node_runtime->node_index);
1399  f = pf->frame_index != ~0 ? vlib_get_frame (vm, pf->frame_index) : 0;
1400 
1401  vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
1402  f ? f->n_vectors : 0, /* is_after */ 0);
1403 
1404  /* Save away current process for suspend. */
1406 
1407  n_vectors = vlib_process_resume (p);
1408  t = clib_cpu_time_now ();
1409 
1410  nm->current_process_index = ~0;
1411 
1412  is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND;
1413  if (is_suspend)
1414  {
1415  /* Suspend it again. */
1416  n_vectors = 0;
1417  p->n_suspends += 1;
1421  (node->runtime_index));
1422  }
1423  else
1424  {
1428  }
1429 
1430  t = clib_cpu_time_now ();
1431  vlib_elog_main_loop_event (vm, node_runtime->node_index, t, !is_suspend,
1432  /* is_after */ 1);
1433 
1435  /* n_calls */ !is_suspend,
1436  /* n_vectors */ n_vectors,
1437  /* n_clocks */ t - last_time_stamp);
1438 
1439  return t;
1440 }
1441 
1444 {
1445  vlib_node_main_t *nm = &vm->node_main;
1447  uword i;
1448  u64 cpu_time_now;
1450  u32 *last_node_runtime_indices = 0;
1451 
1452  /* Initialize pending node vector. */
1453  if (is_main)
1454  {
1455  vec_resize (nm->pending_frames, 32);
1456  _vec_len (nm->pending_frames) = 0;
1457  }
1458 
1459  /* Mark time of main loop start. */
1460  if (is_main)
1461  {
1462  cpu_time_now = vm->clib_time.last_cpu_time;
1463  vm->cpu_time_main_loop_start = cpu_time_now;
1464  }
1465  else
1466  cpu_time_now = clib_cpu_time_now ();
1467 
1468  /* Arrange for first level of timing wheel to cover times we care
1469  most about. */
1470  if (is_main)
1471  {
1472  nm->timing_wheel.min_sched_time = 10e-6;
1473  nm->timing_wheel.max_sched_time = 10e-3;
1475  cpu_time_now, vm->clib_time.clocks_per_second);
1477  }
1478 
1479  /* Pre-allocate interupt runtime indices and lock. */
1481  vec_alloc (last_node_runtime_indices, 32);
1482  if (!is_main)
1484 
1485  /* Pre-allocate expired nodes. */
1490 
1491  /* Start all processes. */
1492  if (is_main)
1493  {
1494  uword i;
1495  nm->current_process_index = ~0;
1496  for (i = 0; i < vec_len (nm->processes); i++)
1497  cpu_time_now = dispatch_process (vm, nm->processes[i], /* frame */ 0,
1498  cpu_time_now);
1499  }
1500 
1501  while (1)
1502  {
1504 
1505  if (!is_main)
1506  {
1508  vec_foreach (fqm, tm->frame_queue_mains)
1509  vlib_frame_queue_dequeue (vm, fqm);
1510  }
1511 
1512  /* Process pre-input nodes. */
1513  if (is_main)
1515  cpu_time_now = dispatch_node (vm, n,
1517  VLIB_NODE_STATE_POLLING,
1518  /* frame */ 0,
1519  cpu_time_now);
1520 
1521  /* Next process input nodes. */
1523  cpu_time_now = dispatch_node (vm, n,
1525  VLIB_NODE_STATE_POLLING,
1526  /* frame */ 0,
1527  cpu_time_now);
1528 
1529  if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0))
1530  vm->queue_signal_callback (vm);
1531 
1532  /* Next handle interrupts. */
1533  {
1534  uword l = _vec_len (nm->pending_interrupt_node_runtime_indices);
1535  uword i;
1536  if (l > 0)
1537  {
1538  u32 *tmp;
1539  if (!is_main)
1543  last_node_runtime_indices;
1544  last_node_runtime_indices = tmp;
1545  _vec_len (last_node_runtime_indices) = 0;
1546  if (!is_main)
1548  for (i = 0; i < l; i++)
1549  {
1551  last_node_runtime_indices[i]);
1552  cpu_time_now =
1554  VLIB_NODE_STATE_INTERRUPT,
1555  /* frame */ 0,
1556  cpu_time_now);
1557  }
1558  }
1559  }
1560 
1561  if (is_main)
1562  {
1563  /* Check if process nodes have expired from timing wheel. */
1565  = timing_wheel_advance (&nm->timing_wheel, cpu_time_now,
1568 
1570  if (PREDICT_FALSE
1571  (_vec_len (nm->data_from_advancing_timing_wheel) > 0))
1572  {
1573  uword i;
1574 
1575  processes_timing_wheel_data:
1576  for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel);
1577  i++)
1578  {
1581 
1583  {
1586  di);
1587  vlib_node_t *n =
1589  vlib_process_t *p =
1590  vec_elt (nm->processes, n->runtime_index);
1591  void *data;
1592  data =
1594  te->event_type_index,
1595  te->n_data_elts,
1596  te->n_data_elt_bytes);
1597  if (te->n_data_bytes < sizeof (te->inline_event_data))
1598  clib_memcpy (data, te->inline_event_data,
1599  te->n_data_bytes);
1600  else
1601  {
1602  clib_memcpy (data, te->event_data_as_vector,
1603  te->n_data_bytes);
1605  }
1607  }
1608  else
1609  {
1610  cpu_time_now = clib_cpu_time_now ();
1611  cpu_time_now =
1612  dispatch_suspended_process (vm, di, cpu_time_now);
1613  }
1614  }
1615 
1616  /* Reset vector. */
1617  _vec_len (nm->data_from_advancing_timing_wheel) = 0;
1618  }
1619  }
1620 
1621  /* Input nodes may have added work to the pending vector.
1622  Process pending vector until there is nothing left.
1623  All pending vectors will be processed from input -> output. */
1624  for (i = 0; i < _vec_len (nm->pending_frames); i++)
1625  cpu_time_now = dispatch_pending_node (vm, i, cpu_time_now);
1626  /* Reset pending vector for next iteration. */
1627  _vec_len (nm->pending_frames) = 0;
1628 
1629  /* Pending internal nodes may resume processes. */
1630  if (is_main && _vec_len (nm->data_from_advancing_timing_wheel) > 0)
1631  goto processes_timing_wheel_data;
1632 
1634 
1635  /* Record time stamp in case there are no enabled nodes and above
1636  calls do not update time stamp. */
1637  cpu_time_now = clib_cpu_time_now ();
1638  }
1639 }
1640 
1641 static void
1643 {
1644  vlib_main_or_worker_loop (vm, /* is_main */ 1);
1645 }
1646 
1647 void
1649 {
1650  vlib_main_or_worker_loop (vm, /* is_main */ 0);
1651 }
1652 
1654 
1655 static clib_error_t *
1657 {
1658  int turn_on_mem_trace = 0;
1659 
1660  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1661  {
1662  if (unformat (input, "memory-trace"))
1663  turn_on_mem_trace = 1;
1664 
1665  else if (unformat (input, "elog-events %d",
1666  &vm->elog_main.event_ring_size))
1667  ;
1668  else if (unformat (input, "elog-post-mortem-dump"))
1669  vm->elog_post_mortem_dump = 1;
1670  else
1671  return unformat_parse_error (input);
1672  }
1673 
1674  unformat_free (input);
1675 
1676  /* Enable memory trace as early as possible. */
1677  if (turn_on_mem_trace)
1678  clib_mem_trace (1);
1679 
1680  return 0;
1681 }
1682 
1684 
1685 static void
1687 {
1688 }
1689 
1690 /* Main function. */
1691 int
1693 {
1694  clib_error_t *volatile error;
1695 
1697 
1698  clib_time_init (&vm->clib_time);
1699 
1700  /* Turn on event log. */
1701  if (!vm->elog_main.event_ring_size)
1702  vm->elog_main.event_ring_size = 128 << 10;
1704  elog_enable_disable (&vm->elog_main, 1);
1705 
1706  /* Default name. */
1707  if (!vm->name)
1708  vm->name = "VLIB";
1709 
1710  vec_validate (vm->buffer_main, 0);
1711  vlib_buffer_cb_init (vm);
1712 
1713  if ((error = vlib_thread_init (vm)))
1714  {
1715  clib_error_report (error);
1716  goto done;
1717  }
1718 
1719  /* Register static nodes so that init functions may use them. */
1721 
1722  /* Set seed for random number generator.
1723  Allow user to specify seed to make random sequence deterministic. */
1724  if (!unformat (input, "seed %wd", &vm->random_seed))
1725  vm->random_seed = clib_cpu_time_now ();
1727 
1728  /* Initialize node graph. */
1729  if ((error = vlib_node_main_init (vm)))
1730  {
1731  /* Arrange for graph hook up error to not be fatal when debugging. */
1732  if (CLIB_DEBUG > 0)
1733  clib_error_report (error);
1734  else
1735  goto done;
1736  }
1737 
1738  /* See unix/main.c; most likely already set up */
1739  if (vm->init_functions_called == 0)
1740  vm->init_functions_called = hash_create (0, /* value bytes */ 0);
1741  if ((error = vlib_call_all_init_functions (vm)))
1742  goto done;
1743 
1744  /* Create default buffer free list. */
1747  "default");
1748 
1750  {
1752  vm->main_loop_exit_set = 1;
1753  break;
1754 
1756  goto done;
1757 
1758  default:
1759  error = vm->main_loop_error;
1760  goto done;
1761  }
1762 
1763  if ((error = vlib_call_all_config_functions (vm, input, 0 /* is_early */ )))
1764  goto done;
1765 
1766  /* Call all main loop enter functions. */
1767  {
1768  clib_error_t *sub_error;
1769  sub_error = vlib_call_all_main_loop_enter_functions (vm);
1770  if (sub_error)
1771  clib_error_report (sub_error);
1772  }
1773 
1774  vlib_main_loop (vm);
1775 
1776 done:
1777  /* Call all exit functions. */
1778  {
1779  clib_error_t *sub_error;
1780  sub_error = vlib_call_all_main_loop_exit_functions (vm);
1781  if (sub_error)
1782  clib_error_report (sub_error);
1783  }
1784 
1785  if (error)
1786  clib_error_report (error);
1787 
1788  return 0;
1789 }
1790 
1791 /*
1792  * fd.io coding-style-patch-verification: ON
1793  *
1794  * Local Variables:
1795  * eval: (c-set-style "gnu")
1796  * End:
1797  */
static u32 * vlib_frame_find_magic(vlib_frame_t *f, vlib_node_t *node)
Definition: main.c:81
u32 * next_nodes
Definition: node.h:289
static void elog_enable_disable(elog_main_t *em, int is_enabled)
Enable or disable event logging.
Definition: elog.h:217
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:436
static void elog_show_buffer_internal(vlib_main_t *vm, u32 n_events_to_show)
Definition: main.c:809
clib_error_t * vlib_call_all_main_loop_exit_functions(vlib_main_t *vm)
Definition: init.c:87
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:187
f64 time
Absolute time as floating point number in seconds.
Definition: elog.h:66
u16 vector_size
Definition: node.h:275
u32 max_clock
Maximum clock cycle for an invocation.
Definition: node.h:427
u32 next_frame_index
Start of next frames for this node.
Definition: node.h:438
#define hash_set(h, key, value)
Definition: hash.h:254
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:337
clib_error_t * vlib_call_all_main_loop_enter_functions(vlib_main_t *vm)
Definition: init.c:80
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
Definition: lock.h:72
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
Definition: lock.h:50
static vlib_next_frame_t * vlib_node_get_next_frame(vlib_main_t *vm, u32 node_index, u32 next_index)
Get pointer to frame by (node_index, next_index).
Definition: node_funcs.h:327
u32 interrupt_threshold_vector_length
Definition: node.h:658
u32 vlib_buffer_get_or_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
vlib_process_t ** processes
Definition: node.h:679
vlib_node_runtime_t node_runtime
Definition: node.h:499
a
Definition: bitmap.h:516
u32 n_suspends
Definition: node.h:527
static void vlib_main_loop(vlib_main_t *vm)
Definition: main.c:1642
static void vlib_elog_main_loop_event(vlib_main_t *vm, u32 node_index, u64 time, u32 n_vectors, u32 is_return)
Definition: main.c:872
void vlib_gdb_show_event_log(void)
Definition: main.c:866
static void dummy_queue_signal_callback(vlib_main_t *vm)
Definition: main.c:1686
int elog_post_mortem_dump
Definition: main.h:182
#define VLIB_FRAME_OWNER
Definition: node.h:371
#define PREDICT_TRUE(x)
Definition: clib.h:98
CJ_GLOBAL_LOG_PROTOTYPE
Definition: main.c:47
uword random_seed
Definition: main.h:150
#define VLIB_FRAME_IS_ALLOCATED
Definition: node.h:374
void timing_wheel_init(timing_wheel_t *w, u64 current_cpu_time, f64 cpu_clocks_per_second)
Definition: timing_wheel.c:21
static void vlib_increment_main_loop_counter(vlib_main_t *vm)
Definition: main.h:294
elog_time_stamp_t init_time
Timestamps.
Definition: elog.h:169
void clib_random_buffer_init(clib_random_buffer_t *b, uword seed)
Definition: random_buffer.c:62
u32 index
Definition: node.h:238
f64 clocks_per_second
Definition: time.h:53
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:459
u32 current_process_index
Definition: node.h:682
u16 thread_index
Definition: node.h:348
u32 thread_index
Definition: main.h:159
u32 main_loop_exit_set
Definition: main.h:87
u64 cpu_time_next_process_ready
Definition: node.h:675
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:522
static u64 clib_cpu_time_now(void)
Definition: time.h:73
static u64 dispatch_process(vlib_main_t *vm, vlib_process_t *p, vlib_frame_t *f, u64 last_time_stamp)
Definition: main.c:1301
elog_track_t elog_track
Definition: threads.h:100
static_always_inline uword vlib_process_startup(vlib_main_t *vm, vlib_process_t *p, vlib_frame_t *f)
Definition: main.c:1270
#define VLIB_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:366
u32 clocks_since_last_overflow
Number of clock cycles.
Definition: node.h:425
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:561
vlib_buffer_main_t * buffer_main
Definition: main.h:104
#define VLIB_MAIN_LOOP_EXIT_CLI
Definition: main.h:92
static void vlib_process_update_stats(vlib_main_t *vm, vlib_process_t *p, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:648
void clib_longjmp(clib_longjmp_t *save, uword return_value)
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
u8 * format_elog_event(u8 *s, va_list *va)
Definition: elog.c:296
clib_time_t clib_time
Definition: main.h:62
vlib_frame_t * vlib_get_next_frame_internal(vlib_main_t *vm, vlib_node_runtime_t *node, u32 next_index, u32 allocate_new_next_frame)
Definition: main.c:361
static void validate_frame_magic(vlib_main_t *vm, vlib_frame_t *f, vlib_node_t *n, uword next_index)
Definition: main.c:352
static clib_error_t * elog_resize(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:777
clib_spinlock_t pending_interrupt_lock
Definition: node.h:652
static clib_error_t * elog_save_buffer(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:674
void timing_wheel_insert(timing_wheel_t *w, u64 insert_cpu_time, u32 user_data)
Definition: timing_wheel.c:294
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:200
uword clib_setjmp(clib_longjmp_t *save, uword return_value_not_taken)
#define vec_alloc(V, N)
Allocate space for N more elements (no header, unspecified alignment)
Definition: vec.h:279
#define VLIB_FRAME_SIZE_EXTRA
vlib_node_state_t
Definition: node.h:208
#define VLIB_PROCESS_RESUME_PENDING
Definition: node.h:516
static vlib_frame_t * vlib_get_frame_no_check(vlib_main_t *vm, uword frame_index)
Definition: node_funcs.h:215
void vlib_frame_free(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_frame_t *f)
Definition: main.c:216
u16 scalar_size
Definition: node.h:275
u8 state
Definition: node.h:266
u16 thread_index
thread this node runs on
Definition: node.h:466
unformat_input_t input
Definition: init.h:70
static void vlib_worker_thread_barrier_check(void)
Definition: threads.h:356
u32 * pending_interrupt_node_runtime_indices
Definition: node.h:651
u32 input_main_loops_per_call
For input nodes: decremented on each main loop interation until it reaches zero and function is calle...
Definition: node.h:443
#define static_always_inline
Definition: clib.h:85
static_always_inline u64 dispatch_node(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_node_type_t type, vlib_node_state_t dispatch_state, vlib_frame_t *frame, u64 last_time_stamp)
Definition: main.c:939
vlib_node_function_t * function
Node function to call.
Definition: node.h:417
void elog_init(elog_main_t *em, u32 n_events)
Definition: elog.c:466
#define always_inline
Definition: clib.h:84
never_inline void vlib_node_runtime_sync_stats(vlib_main_t *vm, vlib_node_runtime_t *r, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:545
static void vlib_put_next_frame_validate(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 next_index, u32 n_vectors_left)
Definition: main.c:423
u32 main_loop_vectors_processed
Definition: main.h:74
u16 log2_n_stack_bytes
Definition: node.h:522
static clib_error_t * elog_stop(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:737
vlib_node_t ** nodes
Definition: node.h:638
void vlib_dump_context_trace(vlib_main_t *vm, u32 bi)
Definition: main.c:893
u32 vectors_since_last_overflow
Number of vector elements processed by this node.
Definition: node.h:435
vlib_frame_t * frame
Definition: main.c:1238
void di(unformat_input_t *i)
Definition: unformat.c:163
static clib_error_t * vlib_cli_elog_clear(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:657
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
static uword vlib_process_stack_is_valid(vlib_process_t *p)
Definition: main.c:1229
#define VLIB_FRAME_ALIGN
Definition: node.h:330
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
Definition: buffer.h:144
#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES
Definition: buffer.h:403
unsigned long u64
Definition: types.h:89
u32 wraps
Definition: main.c:53
#define vec_resize(V, N)
Resize a vector (no header, unspecified alignment) Add N elements to end of given vector V...
Definition: vec.h:241
u64 cpu_time_main_loop_start
Definition: main.h:68
vlib_node_runtime_t * nodes_by_type[VLIB_N_NODE_TYPE]
Definition: node.h:648
u32 * free_frame_indices
Definition: node.h:487
#define VLIB_MAIN_LOOP_EXIT_NONE
Definition: main.h:89
#define VLIB_FRAME_TRACE
Definition: node.h:383
clib_error_t * vlib_node_main_init(vlib_main_t *vm)
Definition: node.c:547
static vlib_next_frame_t * vlib_node_runtime_get_next_frame(vlib_main_t *vm, vlib_node_runtime_t *n, u32 next_index)
Definition: node_funcs.h:292
static uword pointer_to_uword(const void *p)
Definition: types.h:131
u64 max_clock_n
Definition: node.h:196
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:33
u32 calls_since_last_overflow
Number of calls.
Definition: node.h:433
char * name
Definition: main.h:98
vlib_worker_thread_t * vlib_worker_threads
Definition: threads.c:35
#define hash_get(h, key)
Definition: hash.h:248
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:397
u32 next_frame_index
Definition: node.h:407
static void * vlib_process_signal_event_helper(vlib_node_main_t *nm, vlib_node_t *n, vlib_process_t *p, uword t, uword n_data_elts, uword n_data_elt_bytes)
Definition: node_funcs.h:770
vlib_node_stats_t stats_total
Definition: node.h:228
static clib_error_t * elog_restart(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:757
u16 state
Input node state.
Definition: node.h:456
u32 frame_index
Definition: node.h:357
void vlib_node_sync_stats(vlib_main_t *vm, vlib_node_t *n)
Definition: main.c:575
vlib_signal_timed_event_data_t * signal_timed_event_data_pool
Definition: node.h:669
vlib_main_t vlib_global_main
Definition: main.c:1653
vlib_node_type_t
Definition: node.h:59
static u32 vlib_frame_index(vlib_main_t *vm, vlib_frame_t *f)
Definition: node_funcs.h:247
static vlib_frame_size_t * get_frame_size_info(vlib_node_main_t *nm, u32 n_scalar_bytes, u32 n_vector_bytes)
Definition: main.c:93
#define v
Definition: acl.c:320
struct _unformat_input_t unformat_input_t
static clib_error_t * vlib_main_configure(vlib_main_t *vm, unformat_input_t *input)
Definition: main.c:1656
u32 n_alloc_frames
Definition: node.h:484
static clib_error_t * elog_write_file(elog_main_t *em, char *unix_file, int flush_ring)
Definition: elog.h:528
u8 inline_event_data[64-3 *sizeof(u32)-2 *sizeof(u16)]
Definition: node.h:603
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:241
#define VLIB_FRAME_MAGIC
elog_event_type_t * node_return_elog_event_types
Definition: main.h:145
u32 polling_threshold_vector_length
Definition: node.h:657
u64 * n_vectors_by_next_node
Definition: node.h:298
#define PREDICT_FALSE(x)
Definition: clib.h:97
#define VLIB_FRAME_PENDING
Definition: node.h:377
static u32 vlib_timing_wheel_data_set_suspended_process(u32 i)
Definition: node.h:618
static_always_inline uword vlib_process_resume(vlib_process_t *p)
Definition: main.c:1288
#define VLIB_FRAME_SIZE
Definition: node.h:329
f64 seconds_per_clock
Definition: time.h:57
static u64 dispatch_pending_node(vlib_main_t *vm, uword pending_frame_index, u64 last_time_stamp)
Definition: main.c:1115
static vlib_frame_t * vlib_get_frame(vlib_main_t *vm, uword frame_index)
Definition: node_funcs.h:239
u32 node_index
Node index.
Definition: node.h:441
uword * init_functions_called
Definition: main.h:156
#define VLIB_FRAME_FREE_AFTER_DISPATCH
Definition: node.h:380
#define VLIB_ELOG_MAIN_LOOP
Definition: main.h:56
void clib_time_init(clib_time_t *c)
Definition: time.c:175
uword * frame_size_hash
Definition: node.h:694
void elog_alloc(elog_main_t *em, u32 n_events)
Definition: elog.c:452
volatile u32 queue_signal_pending
Definition: main.h:173
static u64 dispatch_suspended_process(vlib_main_t *vm, uword process_index, u64 last_time_stamp)
Definition: main.c:1373
#define VLIB_PROCESS_RETURN_LONGJMP_SUSPEND
Definition: node.h:505
word fformat(FILE *f, char *fmt,...)
Definition: format.c:453
#define uword_to_pointer(u, type)
Definition: types.h:136
u8 * name
Definition: node.h:222
timing_wheel_t timing_wheel
Definition: node.h:667
static u32 vlib_frame_index_no_check(vlib_main_t *vm, vlib_frame_t *f)
Definition: node_funcs.h:226
u32 owner_node_index
Definition: node.h:309
static u32 vlib_frame_alloc_to_node(vlib_main_t *vm, u32 to_node_index, u32 frame_flags)
Definition: main.c:113
void vlib_register_all_static_nodes(vlib_main_t *vm)
Definition: node.c:518
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
Definition: init.h:140
static u32 vlib_node_runtime_update_main_loop_vector_stats(vlib_main_t *vm, vlib_node_runtime_t *node, uword n_vectors)
Definition: node_funcs.h:1018
#define UNFORMAT_END_OF_INPUT
Definition: format.h:143
svmdb_client_t * c
u16 n_vectors
Definition: node.h:345
u32 runtime_index
Definition: node.h:241
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:82
static uword elog_n_events_in_buffer(elog_main_t *em)
Return number of events in the event-log buffer.
Definition: elog.h:188
u32 node_runtime_index
Definition: node.h:401
vlib_pending_frame_t * pending_frames
Definition: node.h:664
int vlib_frame_queue_dequeue(vlib_main_t *vm, vlib_frame_queue_main_t *fqm)
Definition: threads.c:1228
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:340
void elog_post_mortem_dump(void)
Definition: main.c:711
#define clib_warning(format, args...)
Definition: error.h:59
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
Definition: node_funcs.h:88
#define clib_memcpy(a, b, c)
Definition: string.h:69
u32 * timing_wheel_advance(timing_wheel_t *w, u64 advance_cpu_time, u32 *expired_user_data, u64 *next_expiring_element_cpu_time)
Definition: timing_wheel.c:592
void vlib_worker_loop(vlib_main_t *vm)
Definition: main.c:1648
#define VLIB_BUFFER_TRACE_TRAJECTORY
Compile time buffer trajectory tracing option Turn this on if you run into "bad monkey" contexts...
Definition: buffer.h:491
elog_main_t elog_main
Definition: main.h:141
static clib_error_t * elog_show_buffer(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:837
#define VLIB_PROCESS_RETURN_LONGJMP_RETURN
Definition: node.h:504
static u32 vlib_frame_vector_byte_offset(u32 scalar_size)
Definition: node_funcs.h:258
void vlib_worker_thread_barrier_sync(vlib_main_t *vm)
Definition: threads.c:1166
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:439
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:278
u32 n_total_events_disable_limit
When count reaches limit logging is disabled.
Definition: elog.h:138
void vlib_start_process(vlib_main_t *vm, uword process_index)
Definition: main.c:1365
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:154
#define never_inline
Definition: clib.h:81
#define hash_create(elts, value_bytes)
Definition: hash.h:658
#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT
Definition: node.h:514
static uword vlib_timing_wheel_data_is_timed_event(u32 d)
Definition: node.h:612
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
Definition: node.h:460
int vlib_main(vlib_main_t *volatile vm, unformat_input_t *input)
Definition: main.c:1692
#define ASSERT(truth)
u64 last_cpu_time
Definition: time.h:50
unsigned int u32
Definition: types.h:88
static clib_error_t * show_frame_stats(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:248
static void elog_reset_buffer(elog_main_t *em)
Reset the event buffer.
Definition: elog.h:207
vlib_frame_queue_main_t * frame_queue_mains
Definition: threads.h:299
#define ELOG_TRACK_DATA(em, f, track)
Definition: elog.h:475
uword event_ring_size
Power of 2 number of elements in ring.
Definition: elog.h:144
#define VLIB_PROCESS_STACK_MAGIC
Definition: node.h:564
u16 flags
Definition: node.h:336
u32 main_loop_nodes_processed
Definition: main.h:75
u64 cpu
CPU cycle counter.
Definition: elog.h:125
#define VLIB_PROCESS_RESUME_LONGJMP_RESUME
Definition: node.h:510
#define clib_error_report(e)
Definition: error.h:113
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:260
static_always_inline void vlib_main_or_worker_loop(vlib_main_t *vm, int is_main)
Definition: main.c:1443
void(* queue_signal_callback)(struct vlib_main_t *)
Definition: main.h:175
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
static u32 vlib_node_runtime_update_stats(vlib_main_t *vm, vlib_node_runtime_t *node, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:612
u64 uword
Definition: types.h:112
#define vec_elt(v, i)
Get vector value at index i.
#define unformat_parse_error(input)
Definition: format.h:267
static u32 vlib_frame_bytes(u32 n_scalar_bytes, u32 n_vector_bytes)
Definition: main.c:56
static vlib_node_t * vlib_get_next_node(vlib_main_t *vm, u32 node_index, u32 next_index)
Get vlib node by graph arc (next) index.
Definition: node_funcs.h:71
#define clib_mem_alloc_aligned_no_fail(size, align)
Definition: mem.h:143
u32 * data_from_advancing_timing_wheel
Definition: node.h:672
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
double f64
Definition: types.h:142
u32 input_node_counts_by_state[VLIB_N_NODE_STATE]
Definition: node.h:691
unsigned char u8
Definition: types.h:56
#define VLIB_PENDING_FRAME_NO_NEXT_FRAME
Definition: node.h:410
vlib_pending_frame_t * suspended_process_frames
Definition: node.h:685
void vlib_buffer_cb_init(struct vlib_main_t *vm)
vlib_node_main_t node_main
Definition: main.h:115
u32 owner_next_index
Definition: node.h:309
u8 vector_size
Definition: node.h:342
vlib_next_frame_t * next_frames
Definition: node.h:661
static void unformat_free(unformat_input_t *i)
Definition: format.h:161
clib_error_t * vlib_call_all_config_functions(vlib_main_t *vm, unformat_input_t *input, int is_early)
Definition: init.c:94
#define VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE
Definition: node.h:263
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:269
int extern_buffer_mgmt
Definition: buffer.h:420
clib_error_t * vlib_call_all_init_functions(vlib_main_t *vm)
Definition: init.c:67
#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
Definition: node.h:513
static void vlib_next_frame_change_ownership(vlib_main_t *vm, vlib_node_runtime_t *node_runtime, u32 next_index)
Definition: main.c:278
static void elog_track(elog_main_t *em, elog_event_type_t *type, elog_track_t *track, u32 data)
Log a single-datum event to a specific track, non-inline version.
Definition: elog.h:395
vlib_frame_size_t * frame_sizes
Definition: node.h:697
u16 flags
Definition: node.h:512
elog_event_type_t * node_call_elog_event_types
Definition: main.h:144
vlib_node_type_t type
Definition: node.h:235
static vlib_process_t * vlib_get_process_from_node(vlib_main_t *vm, vlib_node_t *node)
Definition: node_funcs.h:206
uword clib_calljmp(uword(*func)(uword func_arg), uword func_arg, void *stack)
u8 * format_unformat_error(u8 *s, va_list *va)
Definition: unformat.c:91
u32 name_elog_string
Definition: node.h:225
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1198
static void vlib_process_sync_stats(vlib_main_t *vm, vlib_process_t *p, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:563
#define VLIB_INVALID_NODE_INDEX
Definition: node.h:326
u64 init_cpu_time
Definition: time.h:60
u32 suspended_process_frame_index
Definition: node.h:524
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
#define VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE
Definition: node.h:262
u64 resume_cpu_time
Definition: node.h:546
static vlib_node_t * vlib_get_node(vlib_main_t *vm, u32 i)
Get vlib node by index.
Definition: node_funcs.h:58
static uword vlib_timing_wheel_data_get_index(u32 d)
Definition: node.h:630
#define vec_foreach(var, vec)
Vector iterator.
elog_track_t default_track
Default track.
Definition: elog.h:163
clib_longjmp_t return_longjmp
Definition: node.h:502
u16 flags
Copy of main node flags.
Definition: node.h:454
u32 node_runtime_index
Definition: node.h:360
u32 n_total_events
Total number of events in buffer.
Definition: elog.h:134
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:196
u8 scalar_size
Definition: node.h:339
clib_longjmp_t main_loop_exit
Definition: main.h:88
clib_longjmp_t resume_longjmp
Definition: node.h:508
void clib_mem_trace(int enable)
Definition: mem_mheap.c:154
u32 flags
Definition: vhost-user.h:76
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
#define VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:251
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:680
static uword vlib_process_bootstrap(uword _a)
Definition: main.c:1243
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
clib_random_buffer_t random_buffer
Definition: main.h:153
static u32 vlib_frame_alloc(vlib_main_t *vm, vlib_node_runtime_t *from_node_runtime, u32 to_next_index)
Definition: main.c:174
u8 *(* validate_frame)(struct vlib_main_t *vm, struct vlib_node_runtime_t *, struct vlib_frame_t *f)
Definition: node.h:319
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:972
clib_error_t * main_loop_error
Definition: main.h:95
vlib_process_t * process
Definition: main.c:1237
elog_event_t * elog_peek_events(elog_main_t *em)
convert event ring events to events, and return them as a vector.
Definition: elog.c:509
static uword elog_buffer_capacity(elog_main_t *em)
Return number of events which can fit in the event buffer.
Definition: elog.h:198
u64 cpu_time_last_node_dispatch
Definition: main.h:65
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:169
clib_error_t * vlib_thread_init(vlib_main_t *vm)
Definition: threads.c:106
u32 vectors_since_last_overflow
Definition: node.h:386
#define VLIB_PROCESS_IS_RUNNING
Definition: node.h:519
u32 max_clock_n
Number of vectors in the recorded max_clock.
Definition: node.h:430