FD.io VPP  v18.07.1-19-g511ce25
Vector Packet Processing
main.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * main.c: main vector processing loop
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #include <math.h>
41 #include <vppinfra/format.h>
42 #include <vlib/vlib.h>
43 #include <vlib/threads.h>
45 
46 #include <vlib/unix/unix.h>
47 #include <vlib/unix/cj.h>
48 
50 
51 /* Actually allocate a few extra slots of vector data to support
52  speculative vector enqueues which overflow vector data in next frame. */
53 #define VLIB_FRAME_SIZE_ALLOC (VLIB_FRAME_SIZE + 4)
54 
56 
58 vlib_frame_bytes (u32 n_scalar_bytes, u32 n_vector_bytes)
59 {
60  u32 n_bytes;
61 
62  /* Make room for vlib_frame_t plus scalar arguments. */
63  n_bytes = vlib_frame_vector_byte_offset (n_scalar_bytes);
64 
65  /* Make room for vector arguments.
66  Allocate a few extra slots of vector data to support
67  speculative vector enqueues which overflow vector data in next frame. */
68 #define VLIB_FRAME_SIZE_EXTRA 4
69  n_bytes += (VLIB_FRAME_SIZE + VLIB_FRAME_SIZE_EXTRA) * n_vector_bytes;
70 
71  /* Magic number is first 32bit number after vector data.
72  Used to make sure that vector data is never overrun. */
73 #define VLIB_FRAME_MAGIC (0xabadc0ed)
74  n_bytes += sizeof (u32);
75 
76  /* Pad to cache line. */
77  n_bytes = round_pow2 (n_bytes, CLIB_CACHE_LINE_BYTES);
78 
79  return n_bytes;
80 }
81 
84 {
85  void *p = f;
86 
88 
90 
91  return p;
92 }
93 
94 static vlib_frame_size_t *
96  u32 n_scalar_bytes, u32 n_vector_bytes)
97 {
98  uword key = (n_scalar_bytes << 16) | n_vector_bytes;
99  uword *p, i;
100 
101  p = hash_get (nm->frame_size_hash, key);
102  if (p)
103  i = p[0];
104  else
105  {
106  i = vec_len (nm->frame_sizes);
107  vec_validate (nm->frame_sizes, i);
108  hash_set (nm->frame_size_hash, key, i);
109  }
110 
111  return vec_elt_at_index (nm->frame_sizes, i);
112 }
113 
114 static u32
116  u32 frame_flags)
117 {
118  vlib_node_main_t *nm = &vm->node_main;
119  vlib_frame_size_t *fs;
120  vlib_node_t *to_node;
121  vlib_frame_t *f;
122  u32 fi, l, n, scalar_size, vector_size;
123 
124  to_node = vlib_get_node (vm, to_node_index);
125 
126  scalar_size = to_node->scalar_size;
127  vector_size = to_node->vector_size;
128 
129  fs = get_frame_size_info (nm, scalar_size, vector_size);
130  n = vlib_frame_bytes (scalar_size, vector_size);
131  if ((l = vec_len (fs->free_frame_indices)) > 0)
132  {
133  /* Allocate from end of free list. */
134  fi = fs->free_frame_indices[l - 1];
135  f = vlib_get_frame_no_check (vm, fi);
136  _vec_len (fs->free_frame_indices) = l - 1;
137  }
138  else
139  {
141  fi = vlib_frame_index_no_check (vm, f);
142  }
143 
144  /* Poison frame when debugging. */
145  if (CLIB_DEBUG > 0)
146  memset (f, 0xfe, n);
147 
148  /* Insert magic number. */
149  {
150  u32 *magic;
151 
152  magic = vlib_frame_find_magic (f, to_node);
153  *magic = VLIB_FRAME_MAGIC;
154  }
155 
156  f->flags = VLIB_FRAME_IS_ALLOCATED | frame_flags;
157  f->n_vectors = 0;
158  f->scalar_size = scalar_size;
159  f->vector_size = vector_size;
160 
161  fs->n_alloc_frames += 1;
162 
163  return fi;
164 }
165 
166 /* Allocate a frame for from FROM_NODE to TO_NODE via TO_NEXT_INDEX.
167  Returns frame index. */
168 static u32
170  u32 to_next_index)
171 {
172  vlib_node_t *from_node;
173 
174  from_node = vlib_get_node (vm, from_node_runtime->node_index);
175  ASSERT (to_next_index < vec_len (from_node->next_nodes));
176 
177  return vlib_frame_alloc_to_node (vm, from_node->next_nodes[to_next_index],
178  /* frame_flags */ 0);
179 }
180 
181 vlib_frame_t *
183 {
184  u32 fi = vlib_frame_alloc_to_node (vm, to_node_index,
185  /* frame_flags */
187  return vlib_get_frame (vm, fi);
188 }
189 
190 void
192 {
194  vlib_node_t *to_node;
195 
196  if (f->n_vectors == 0)
197  return;
198 
199  to_node = vlib_get_node (vm, to_node_index);
200 
201  vec_add2 (vm->node_main.pending_frames, p, 1);
202 
204  p->frame_index = vlib_frame_index (vm, f);
205  p->node_runtime_index = to_node->runtime_index;
207 }
208 
209 /* Free given frame. */
210 void
212 {
213  vlib_node_main_t *nm = &vm->node_main;
214  vlib_node_t *node;
215  vlib_frame_size_t *fs;
216  u32 frame_index;
217 
219 
220  node = vlib_get_node (vm, r->node_index);
221  fs = get_frame_size_info (nm, node->scalar_size, node->vector_size);
222 
223  frame_index = vlib_frame_index (vm, f);
224 
226 
227  /* No next frames may point to freed frame. */
228  if (CLIB_DEBUG > 0)
229  {
230  vlib_next_frame_t *nf;
232  ASSERT (nf->frame_index != frame_index);
233  }
234 
236 
237  vec_add1 (fs->free_frame_indices, frame_index);
238  ASSERT (fs->n_alloc_frames > 0);
239  fs->n_alloc_frames -= 1;
240 }
241 
242 static clib_error_t *
244  unformat_input_t * input, vlib_cli_command_t * cmd)
245 {
246  vlib_node_main_t *nm = &vm->node_main;
247  vlib_frame_size_t *fs;
248 
249  vlib_cli_output (vm, "%=6s%=12s%=12s", "Size", "# Alloc", "# Free");
250  vec_foreach (fs, nm->frame_sizes)
251  {
252  u32 n_alloc = fs->n_alloc_frames;
253  u32 n_free = vec_len (fs->free_frame_indices);
254 
255  if (n_alloc + n_free > 0)
256  vlib_cli_output (vm, "%=6d%=12d%=12d",
257  fs - nm->frame_sizes, n_alloc, n_free);
258  }
259 
260  return 0;
261 }
262 
263 /* *INDENT-OFF* */
264 VLIB_CLI_COMMAND (show_frame_stats_cli, static) = {
265  .path = "show vlib frame-allocation",
266  .short_help = "Show node dispatch frame statistics",
267  .function = show_frame_stats,
268 };
269 /* *INDENT-ON* */
270 
271 /* Change ownership of enqueue rights to given next node. */
272 static void
274  vlib_node_runtime_t * node_runtime,
275  u32 next_index)
276 {
277  vlib_node_main_t *nm = &vm->node_main;
278  vlib_next_frame_t *next_frame;
279  vlib_node_t *node, *next_node;
280 
281  node = vec_elt (nm->nodes, node_runtime->node_index);
282 
283  /* Only internal & input nodes are allowed to call other nodes. */
285  || node->type == VLIB_NODE_TYPE_INPUT
286  || node->type == VLIB_NODE_TYPE_PROCESS);
287 
288  ASSERT (vec_len (node->next_nodes) == node_runtime->n_next_nodes);
289 
290  next_frame =
291  vlib_node_runtime_get_next_frame (vm, node_runtime, next_index);
292  next_node = vec_elt (nm->nodes, node->next_nodes[next_index]);
293 
294  if (next_node->owner_node_index != VLIB_INVALID_NODE_INDEX)
295  {
296  /* Get frame from previous owner. */
297  vlib_next_frame_t *owner_next_frame;
298  vlib_next_frame_t tmp;
299 
300  owner_next_frame =
302  next_node->owner_node_index,
303  next_node->owner_next_index);
304 
305  /* Swap target next frame with owner's. */
306  tmp = owner_next_frame[0];
307  owner_next_frame[0] = next_frame[0];
308  next_frame[0] = tmp;
309 
310  /*
311  * If next_frame is already pending, we have to track down
312  * all pending frames and fix their next_frame_index fields.
313  */
314  if (next_frame->flags & VLIB_FRAME_PENDING)
315  {
317  if (next_frame->frame_index != ~0)
318  {
319  vec_foreach (p, nm->pending_frames)
320  {
321  if (p->frame_index == next_frame->frame_index)
322  {
323  p->next_frame_index =
324  next_frame - vm->node_main.next_frames;
325  }
326  }
327  }
328  }
329  }
330  else
331  {
332  /* No previous owner. Take ownership. */
333  next_frame->flags |= VLIB_FRAME_OWNER;
334  }
335 
336  /* Record new owner. */
337  next_node->owner_node_index = node->index;
338  next_node->owner_next_index = next_index;
339 
340  /* Now we should be owner. */
341  ASSERT (next_frame->flags & VLIB_FRAME_OWNER);
342 }
343 
344 /* Make sure that magic number is still there.
345  Otherwise, it is likely that caller has overrun frame arguments. */
346 always_inline void
348  vlib_frame_t * f, vlib_node_t * n, uword next_index)
349 {
350  vlib_node_t *next_node = vlib_get_node (vm, n->next_nodes[next_index]);
351  u32 *magic = vlib_frame_find_magic (f, next_node);
352  ASSERT (VLIB_FRAME_MAGIC == magic[0]);
353 }
354 
355 vlib_frame_t *
357  vlib_node_runtime_t * node,
358  u32 next_index, u32 allocate_new_next_frame)
359 {
360  vlib_frame_t *f;
361  vlib_next_frame_t *nf;
362  u32 n_used;
363 
364  nf = vlib_node_runtime_get_next_frame (vm, node, next_index);
365 
366  /* Make sure this next frame owns right to enqueue to destination frame. */
367  if (PREDICT_FALSE (!(nf->flags & VLIB_FRAME_OWNER)))
368  vlib_next_frame_change_ownership (vm, node, next_index);
369 
370  /* ??? Don't need valid flag: can use frame_index == ~0 */
372  {
373  nf->frame_index = vlib_frame_alloc (vm, node, next_index);
375  }
376 
377  f = vlib_get_frame (vm, nf->frame_index);
378 
379  /* Has frame been removed from pending vector (e.g. finished dispatching)?
380  If so we can reuse frame. */
381  if ((nf->flags & VLIB_FRAME_PENDING) && !(f->flags & VLIB_FRAME_PENDING))
382  {
383  nf->flags &= ~VLIB_FRAME_PENDING;
384  f->n_vectors = 0;
385  }
386 
387  /* Allocate new frame if current one is already full. */
388  n_used = f->n_vectors;
389  if (n_used >= VLIB_FRAME_SIZE || (allocate_new_next_frame && n_used > 0))
390  {
391  /* Old frame may need to be freed after dispatch, since we'll have
392  two redundant frames from node -> next node. */
394  {
395  vlib_frame_t *f_old = vlib_get_frame (vm, nf->frame_index);
397  }
398 
399  /* Allocate new frame to replace full one. */
400  nf->frame_index = vlib_frame_alloc (vm, node, next_index);
401  f = vlib_get_frame (vm, nf->frame_index);
402  n_used = f->n_vectors;
403  }
404 
405  /* Should have free vectors in frame now. */
406  ASSERT (n_used < VLIB_FRAME_SIZE);
407 
408  if (CLIB_DEBUG > 0)
409  {
410  validate_frame_magic (vm, f,
411  vlib_get_node (vm, node->node_index), next_index);
412  }
413 
414  return f;
415 }
416 
417 static void
419  vlib_node_runtime_t * rt,
420  u32 next_index, u32 n_vectors_left)
421 {
422  vlib_node_main_t *nm = &vm->node_main;
423  vlib_next_frame_t *nf;
424  vlib_frame_t *f;
425  vlib_node_runtime_t *next_rt;
426  vlib_node_t *next_node;
427  u32 n_before, n_after;
428 
429  nf = vlib_node_runtime_get_next_frame (vm, rt, next_index);
430  f = vlib_get_frame (vm, nf->frame_index);
431 
432  ASSERT (n_vectors_left <= VLIB_FRAME_SIZE);
433  n_after = VLIB_FRAME_SIZE - n_vectors_left;
434  n_before = f->n_vectors;
435 
436  ASSERT (n_after >= n_before);
437 
439  nf->node_runtime_index);
440  next_node = vlib_get_node (vm, next_rt->node_index);
441  if (n_after > 0 && next_node->validate_frame)
442  {
443  u8 *msg = next_node->validate_frame (vm, rt, f);
444  if (msg)
445  {
446  clib_warning ("%v", msg);
447  ASSERT (0);
448  }
449  vec_free (msg);
450  }
451 }
452 
453 void
456  u32 next_index, u32 n_vectors_left)
457 {
458  vlib_node_main_t *nm = &vm->node_main;
459  vlib_next_frame_t *nf;
460  vlib_frame_t *f;
461  u32 n_vectors_in_frame;
462 
463  if (buffer_main.callbacks_registered == 0 && CLIB_DEBUG > 0)
464  vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left);
465 
466  nf = vlib_node_runtime_get_next_frame (vm, r, next_index);
467  f = vlib_get_frame (vm, nf->frame_index);
468 
469  /* Make sure that magic number is still there. Otherwise, caller
470  has overrun frame meta data. */
471  if (CLIB_DEBUG > 0)
472  {
473  vlib_node_t *node = vlib_get_node (vm, r->node_index);
474  validate_frame_magic (vm, f, node, next_index);
475  }
476 
477  /* Convert # of vectors left -> number of vectors there. */
478  ASSERT (n_vectors_left <= VLIB_FRAME_SIZE);
479  n_vectors_in_frame = VLIB_FRAME_SIZE - n_vectors_left;
480 
481  f->n_vectors = n_vectors_in_frame;
482 
483  /* If vectors were added to frame, add to pending vector. */
484  if (PREDICT_TRUE (n_vectors_in_frame > 0))
485  {
487  u32 v0, v1;
488 
489  r->cached_next_index = next_index;
490 
491  if (!(f->flags & VLIB_FRAME_PENDING))
492  {
493  __attribute__ ((unused)) vlib_node_t *node;
494  vlib_node_t *next_node;
495  vlib_node_runtime_t *next_runtime;
496 
497  node = vlib_get_node (vm, r->node_index);
498  next_node = vlib_get_next_node (vm, r->node_index, next_index);
499  next_runtime = vlib_node_get_runtime (vm, next_node->index);
500 
501  vec_add2 (nm->pending_frames, p, 1);
502 
503  p->frame_index = nf->frame_index;
505  p->next_frame_index = nf - nm->next_frames;
506  nf->flags |= VLIB_FRAME_PENDING;
508 
509  /*
510  * If we're going to dispatch this frame on another thread,
511  * force allocation of a new frame. Otherwise, we create
512  * a dangling frame reference. Each thread has its own copy of
513  * the next_frames vector.
514  */
515  if (0 && r->thread_index != next_runtime->thread_index)
516  {
517  nf->frame_index = ~0;
519  }
520  }
521 
522  /* Copy trace flag from next_frame and from runtime. */
523  nf->flags |=
524  (nf->flags & VLIB_NODE_FLAG_TRACE) | (r->
526 
528  v1 = v0 + n_vectors_in_frame;
530  if (PREDICT_FALSE (v1 < v0))
531  {
532  vlib_node_t *node = vlib_get_node (vm, r->node_index);
533  vec_elt (node->n_vectors_by_next_node, next_index) += v0;
534  }
535  }
536 }
537 
538 /* Sync up runtime (32 bit counters) and main node stats (64 bit counters). */
539 never_inline void
542  uword n_calls, uword n_vectors, uword n_clocks)
543 {
544  vlib_node_t *n = vlib_get_node (vm, r->node_index);
545 
546  n->stats_total.calls += n_calls + r->calls_since_last_overflow;
547  n->stats_total.vectors += n_vectors + r->vectors_since_last_overflow;
548  n->stats_total.clocks += n_clocks + r->clocks_since_last_overflow;
551 
555 }
556 
557 always_inline void __attribute__ ((unused))
559  vlib_process_t * p,
560  uword n_calls, uword n_vectors, uword n_clocks)
561 {
562  vlib_node_runtime_t *rt = &p->node_runtime;
564  vlib_node_runtime_sync_stats (vm, rt, n_calls, n_vectors, n_clocks);
565  n->stats_total.suspends += p->n_suspends;
566  p->n_suspends = 0;
567 }
568 
569 void
571 {
573 
574  if (n->type == VLIB_NODE_TYPE_PROCESS)
575  {
576  /* Nothing to do for PROCESS nodes except in main thread */
577  if (vm != &vlib_global_main)
578  return;
579 
582  p->n_suspends = 0;
583  rt = &p->node_runtime;
584  }
585  else
586  rt =
588  n->runtime_index);
589 
590  vlib_node_runtime_sync_stats (vm, rt, 0, 0, 0);
591 
592  /* Sync up runtime next frame vector counters with main node structure. */
593  {
594  vlib_next_frame_t *nf;
595  uword i;
596  for (i = 0; i < rt->n_next_nodes; i++)
597  {
598  nf = vlib_node_runtime_get_next_frame (vm, rt, i);
602  }
603  }
604 }
605 
608  vlib_node_runtime_t * node,
609  uword n_calls,
610  uword n_vectors, uword n_clocks)
611 {
612  u32 ca0, ca1, v0, v1, cl0, cl1, r;
613 
614  cl0 = cl1 = node->clocks_since_last_overflow;
615  ca0 = ca1 = node->calls_since_last_overflow;
616  v0 = v1 = node->vectors_since_last_overflow;
617 
618  ca1 = ca0 + n_calls;
619  v1 = v0 + n_vectors;
620  cl1 = cl0 + n_clocks;
621 
622  node->calls_since_last_overflow = ca1;
623  node->clocks_since_last_overflow = cl1;
624  node->vectors_since_last_overflow = v1;
625  node->max_clock_n = node->max_clock > n_clocks ?
626  node->max_clock_n : n_vectors;
627  node->max_clock = node->max_clock > n_clocks ? node->max_clock : n_clocks;
628 
629  r = vlib_node_runtime_update_main_loop_vector_stats (vm, node, n_vectors);
630 
631  if (PREDICT_FALSE (ca1 < ca0 || v1 < v0 || cl1 < cl0))
632  {
633  node->calls_since_last_overflow = ca0;
634  node->clocks_since_last_overflow = cl0;
635  node->vectors_since_last_overflow = v0;
636  vlib_node_runtime_sync_stats (vm, node, n_calls, n_vectors, n_clocks);
637  }
638 
639  return r;
640 }
641 
642 always_inline void
644  vlib_process_t * p,
645  uword n_calls, uword n_vectors, uword n_clocks)
646 {
648  n_calls, n_vectors, n_clocks);
649 }
650 
651 static clib_error_t *
653  unformat_input_t * input, vlib_cli_command_t * cmd)
654 {
656  return 0;
657 }
658 
659 /* *INDENT-OFF* */
660 VLIB_CLI_COMMAND (elog_clear_cli, static) = {
661  .path = "event-logger clear",
662  .short_help = "Clear the event log",
663  .function = vlib_cli_elog_clear,
664 };
665 /* *INDENT-ON* */
666 
667 #ifdef CLIB_UNIX
668 static clib_error_t *
670  unformat_input_t * input, vlib_cli_command_t * cmd)
671 {
672  elog_main_t *em = &vm->elog_main;
673  char *file, *chroot_file;
674  clib_error_t *error = 0;
675 
676  if (!unformat (input, "%s", &file))
677  {
678  vlib_cli_output (vm, "expected file name, got `%U'",
679  format_unformat_error, input);
680  return 0;
681  }
682 
683  /* It's fairly hard to get "../oopsie" through unformat; just in case */
684  if (strstr (file, "..") || index (file, '/'))
685  {
686  vlib_cli_output (vm, "illegal characters in filename '%s'", file);
687  return 0;
688  }
689 
690  chroot_file = (char *) format (0, "/tmp/%s%c", file, 0);
691 
692  vec_free (file);
693 
694  vlib_cli_output (vm, "Saving %wd of %wd events to %s",
696  elog_buffer_capacity (em), chroot_file);
697 
699  error = elog_write_file (em, chroot_file, 1 /* flush ring */ );
701  vec_free (chroot_file);
702  return error;
703 }
704 
705 void
707 {
709  elog_main_t *em = &vm->elog_main;
710  u8 *filename;
711  clib_error_t *error;
712 
713  if (!vm->elog_post_mortem_dump)
714  return;
715 
716  filename = format (0, "/tmp/elog_post_mortem.%d%c", getpid (), 0);
717  error = elog_write_file (em, (char *) filename, 1 /* flush ring */ );
718  if (error)
719  clib_error_report (error);
720  vec_free (filename);
721 }
722 
723 /* *INDENT-OFF* */
724 VLIB_CLI_COMMAND (elog_save_cli, static) = {
725  .path = "event-logger save",
726  .short_help = "event-logger save <filename> (saves log in /tmp/<filename>)",
727  .function = elog_save_buffer,
728 };
729 /* *INDENT-ON* */
730 
731 static clib_error_t *
733  unformat_input_t * input, vlib_cli_command_t * cmd)
734 {
735  elog_main_t *em = &vm->elog_main;
736 
738 
739  vlib_cli_output (vm, "Stopped the event logger...");
740  return 0;
741 }
742 
743 /* *INDENT-OFF* */
744 VLIB_CLI_COMMAND (elog_stop_cli, static) = {
745  .path = "event-logger stop",
746  .short_help = "Stop the event-logger",
747  .function = elog_stop,
748 };
749 /* *INDENT-ON* */
750 
751 static clib_error_t *
753  unformat_input_t * input, vlib_cli_command_t * cmd)
754 {
755  elog_main_t *em = &vm->elog_main;
756 
758 
759  vlib_cli_output (vm, "Restarted the event logger...");
760  return 0;
761 }
762 
763 /* *INDENT-OFF* */
764 VLIB_CLI_COMMAND (elog_restart_cli, static) = {
765  .path = "event-logger restart",
766  .short_help = "Restart the event-logger",
767  .function = elog_restart,
768 };
769 /* *INDENT-ON* */
770 
771 static clib_error_t *
773  unformat_input_t * input, vlib_cli_command_t * cmd)
774 {
775  elog_main_t *em = &vm->elog_main;
776  u32 tmp;
777 
778  /* Stop the parade */
780 
781  if (unformat (input, "%d", &tmp))
782  {
783  elog_alloc (em, tmp);
785  }
786  else
787  return clib_error_return (0, "Must specify how many events in the ring");
788 
789  vlib_cli_output (vm, "Resized ring and restarted the event logger...");
790  return 0;
791 }
792 
793 /* *INDENT-OFF* */
794 VLIB_CLI_COMMAND (elog_resize_cli, static) = {
795  .path = "event-logger resize",
796  .short_help = "event-logger resize <nnn>",
797  .function = elog_resize,
798 };
799 /* *INDENT-ON* */
800 
801 #endif /* CLIB_UNIX */
802 
803 static void
805 {
806  elog_main_t *em = &vm->elog_main;
807  elog_event_t *e, *es;
808  f64 dt;
809 
810  /* Show events in VLIB time since log clock starts after VLIB clock. */
811  dt = (em->init_time.cpu - vm->clib_time.init_cpu_time)
813 
814  es = elog_peek_events (em);
815  vlib_cli_output (vm, "%d of %d events in buffer, logger %s", vec_len (es),
816  em->event_ring_size,
818  "running" : "stopped");
819  vec_foreach (e, es)
820  {
821  vlib_cli_output (vm, "%18.9f: %U",
822  e->time + dt, format_elog_event, em, e);
823  n_events_to_show--;
824  if (n_events_to_show == 0)
825  break;
826  }
827  vec_free (es);
828 
829 }
830 
831 static clib_error_t *
833  unformat_input_t * input, vlib_cli_command_t * cmd)
834 {
835  u32 n_events_to_show;
836  clib_error_t *error = 0;
837 
838  n_events_to_show = 250;
840  {
841  if (unformat (input, "%d", &n_events_to_show))
842  ;
843  else if (unformat (input, "all"))
844  n_events_to_show = ~0;
845  else
846  return unformat_parse_error (input);
847  }
848  elog_show_buffer_internal (vm, n_events_to_show);
849  return error;
850 }
851 
852 /* *INDENT-OFF* */
853 VLIB_CLI_COMMAND (elog_show_cli, static) = {
854  .path = "show event-logger",
855  .short_help = "Show event logger info",
856  .function = elog_show_buffer,
857 };
858 /* *INDENT-ON* */
859 
860 void
862 {
864 }
865 
866 static inline void
868  u32 node_index,
869  u64 time, u32 n_vectors, u32 is_return)
870 {
872  elog_main_t *em = &evm->elog_main;
873 
874  if (VLIB_ELOG_MAIN_LOOP && n_vectors)
875  elog_track (em,
876  /* event type */
877  vec_elt_at_index (is_return
880  node_index),
881  /* track */
883  elog_track : &em->default_track),
884  /* data to log */ n_vectors);
885 }
886 
887 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
888 void (*vlib_buffer_trace_trajectory_cb) (vlib_buffer_t * b, u32 node_index);
889 void (*vlib_buffer_trace_trajectory_init_cb) (vlib_buffer_t * b);
890 
891 void
892 vlib_buffer_trace_trajectory_init (vlib_buffer_t * b)
893 {
894  if (PREDICT_TRUE (vlib_buffer_trace_trajectory_init_cb != 0))
895  {
896  (*vlib_buffer_trace_trajectory_init_cb) (b);
897  }
898 }
899 
900 #endif
901 
902 static inline void
904 {
905 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
906  if (PREDICT_TRUE (vlib_buffer_trace_trajectory_cb != 0))
907  {
908  (*vlib_buffer_trace_trajectory_cb) (b, node_index);
909  }
910 #endif
911 }
912 
915  vlib_node_runtime_t * node,
916  vlib_node_type_t type,
917  vlib_node_state_t dispatch_state,
918  vlib_frame_t * frame, u64 last_time_stamp)
919 {
920  uword n, v;
921  u64 t;
922  vlib_node_main_t *nm = &vm->node_main;
923  vlib_next_frame_t *nf;
924 
925  if (CLIB_DEBUG > 0)
926  {
927  vlib_node_t *n = vlib_get_node (vm, node->node_index);
928  ASSERT (n->type == type);
929  }
930 
931  /* Only non-internal nodes may be disabled. */
932  if (type != VLIB_NODE_TYPE_INTERNAL && node->state != dispatch_state)
933  {
935  return last_time_stamp;
936  }
937 
938  if ((type == VLIB_NODE_TYPE_PRE_INPUT || type == VLIB_NODE_TYPE_INPUT)
939  && dispatch_state != VLIB_NODE_STATE_INTERRUPT)
940  {
942  /* Only call node when count reaches zero. */
943  if (c)
944  {
945  node->input_main_loops_per_call = c - 1;
946  return last_time_stamp;
947  }
948  }
949 
950  /* Speculatively prefetch next frames. */
951  if (node->n_next_nodes > 0)
952  {
954  CLIB_PREFETCH (nf, 4 * sizeof (nf[0]), WRITE);
955  }
956 
957  vm->cpu_time_last_node_dispatch = last_time_stamp;
958 
959  if (1 /* || vm->thread_index == node->thread_index */ )
960  {
961  vlib_main_t *stat_vm;
962 
963  stat_vm = /* vlib_mains ? vlib_mains[0] : */ vm;
964 
966  last_time_stamp,
967  frame ? frame->n_vectors : 0,
968  /* is_after */ 0);
969 
970  /*
971  * Turn this on if you run into
972  * "bad monkey" contexts, and you want to know exactly
973  * which nodes they've visited... See ixge.c...
974  */
975  if (VLIB_BUFFER_TRACE_TRAJECTORY && frame)
976  {
977  int i;
978  u32 *from;
979  from = vlib_frame_vector_args (frame);
980  for (i = 0; i < frame->n_vectors; i++)
981  {
982  vlib_buffer_t *b = vlib_get_buffer (vm, from[i]);
983  add_trajectory_trace (b, node->node_index);
984  }
985  n = node->function (vm, node, frame);
986  }
987  else
988  n = node->function (vm, node, frame);
989 
990  t = clib_cpu_time_now ();
991 
992  vlib_elog_main_loop_event (vm, node->node_index, t, n, /* is_after */
993  1);
994 
996  vm->main_loop_nodes_processed += n > 0;
997 
998  v = vlib_node_runtime_update_stats (stat_vm, node,
999  /* n_calls */ 1,
1000  /* n_vectors */ n,
1001  /* n_clocks */ t - last_time_stamp);
1002 
1003  /* When in interrupt mode and vector rate crosses threshold switch to
1004  polling mode. */
1005  if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT)
1006  || (dispatch_state == VLIB_NODE_STATE_POLLING
1007  && (node->flags
1009  {
1010 #ifdef DISPATCH_NODE_ELOG_REQUIRED
1011  ELOG_TYPE_DECLARE (e) =
1012  {
1013  .function = (char *) __FUNCTION__,.format =
1014  "%s vector length %d, switching to %s",.format_args =
1015  "T4i4t4",.n_enum_strings = 2,.enum_strings =
1016  {
1017  "interrupt", "polling",},};
1018  struct
1019  {
1020  u32 node_name, vector_length, is_polling;
1021  } *ed;
1023 #endif
1024 
1025  if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT
1026  && v >= nm->polling_threshold_vector_length) &&
1027  !(node->flags &
1029  {
1030  vlib_node_t *n = vlib_get_node (vm, node->node_index);
1031  n->state = VLIB_NODE_STATE_POLLING;
1032  node->state = VLIB_NODE_STATE_POLLING;
1033  node->flags &=
1035  node->flags |=
1037  nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] -= 1;
1038  nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] += 1;
1039 
1040 #ifdef DISPATCH_NODE_ELOG_REQUIRED
1041  ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
1042  w->elog_track);
1043  ed->node_name = n->name_elog_string;
1044  ed->vector_length = v;
1045  ed->is_polling = 1;
1046 #endif
1047  }
1048  else if (dispatch_state == VLIB_NODE_STATE_POLLING
1049  && v <= nm->interrupt_threshold_vector_length)
1050  {
1051  vlib_node_t *n = vlib_get_node (vm, node->node_index);
1052  if (node->flags &
1054  {
1055  /* Switch to interrupt mode after dispatch in polling one more time.
1056  This allows driver to re-enable interrupts. */
1057  n->state = VLIB_NODE_STATE_INTERRUPT;
1058  node->state = VLIB_NODE_STATE_INTERRUPT;
1059  node->flags &=
1061  nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] -=
1062  1;
1063  nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] +=
1064  1;
1065 
1066  }
1067  else
1068  {
1069  node->flags |=
1071 #ifdef DISPATCH_NODE_ELOG_REQUIRED
1072  ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
1073  w->elog_track);
1074  ed->node_name = n->name_elog_string;
1075  ed->vector_length = v;
1076  ed->is_polling = 0;
1077 #endif
1078  }
1079  }
1080  }
1081  }
1082 
1083  return t;
1084 }
1085 
1086 static u64
1087 dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index,
1088  u64 last_time_stamp)
1089 {
1090  vlib_node_main_t *nm = &vm->node_main;
1091  vlib_frame_t *f;
1092  vlib_next_frame_t *nf, nf_dummy;
1094  u32 restore_frame_index;
1096 
1097  /* See comment below about dangling references to nm->pending_frames */
1098  p = nm->pending_frames + pending_frame_index;
1099 
1101  p->node_runtime_index);
1102 
1103  f = vlib_get_frame (vm, p->frame_index);
1105  {
1106  /* No next frame: so use dummy on stack. */
1107  nf = &nf_dummy;
1108  nf->flags = f->flags & VLIB_NODE_FLAG_TRACE;
1109  nf->frame_index = ~p->frame_index;
1110  }
1111  else
1113 
1115 
1116  /* Force allocation of new frame while current frame is being
1117  dispatched. */
1118  restore_frame_index = ~0;
1119  if (nf->frame_index == p->frame_index)
1120  {
1121  nf->frame_index = ~0;
1124  restore_frame_index = p->frame_index;
1125  }
1126 
1127  /* Frame must be pending. */
1129  ASSERT (f->n_vectors > 0);
1130 
1131  /* Copy trace flag from next frame to node.
1132  Trace flag indicates that at least one vector in the dispatched
1133  frame is traced. */
1134  n->flags &= ~VLIB_NODE_FLAG_TRACE;
1135  n->flags |= (nf->flags & VLIB_FRAME_TRACE) ? VLIB_NODE_FLAG_TRACE : 0;
1136  nf->flags &= ~VLIB_FRAME_TRACE;
1137 
1138  last_time_stamp = dispatch_node (vm, n,
1140  VLIB_NODE_STATE_POLLING,
1141  f, last_time_stamp);
1142 
1143  f->flags &= ~VLIB_FRAME_PENDING;
1144 
1145  /* Frame is ready to be used again, so restore it. */
1146  if (restore_frame_index != ~0)
1147  {
1148  /*
1149  * We musn't restore a frame that is flagged to be freed. This
1150  * shouldn't happen since frames to be freed post dispatch are
1151  * those used when the to-node frame becomes full i.e. they form a
1152  * sort of queue of frames to a single node. If we get here then
1153  * the to-node frame and the pending frame *were* the same, and so
1154  * we removed the to-node frame. Therefore this frame is no
1155  * longer part of the queue for that node and hence it cannot be
1156  * it's overspill.
1157  */
1159 
1160  /*
1161  * NB: dispatching node n can result in the creation and scheduling
1162  * of new frames, and hence in the reallocation of nm->pending_frames.
1163  * Recompute p, or no supper. This was broken for more than 10 years.
1164  */
1165  p = nm->pending_frames + pending_frame_index;
1166 
1167  /*
1168  * p->next_frame_index can change during node dispatch if node
1169  * function decides to change graph hook up.
1170  */
1173 
1174  if (~0 == nf->frame_index)
1175  {
1176  /* no new frame has been assigned to this node, use the saved one */
1177  nf->frame_index = restore_frame_index;
1178  f->n_vectors = 0;
1179  }
1180  else
1181  {
1182  /* The node has gained a frame, implying packets from the current frame
1183  were re-queued to this same node. we don't need the saved one
1184  anymore */
1185  vlib_frame_free (vm, n, f);
1186  }
1187  }
1188  else
1189  {
1191  {
1193  vlib_frame_free (vm, n, f);
1194  }
1195  }
1196 
1197  return last_time_stamp;
1198 }
1199 
1202 {
1203  return p->stack[0] == VLIB_PROCESS_STACK_MAGIC;
1204 }
1205 
1206 typedef struct
1207 {
1212 
1213 /* Called in process stack. */
1214 static uword
1216 {
1218  vlib_main_t *vm;
1219  vlib_node_runtime_t *node;
1220  vlib_frame_t *f;
1221  vlib_process_t *p;
1222  uword n;
1223 
1225 
1226  vm = a->vm;
1227  p = a->process;
1228  f = a->frame;
1229  node = &p->node_runtime;
1230 
1231  n = node->function (vm, node, f);
1232 
1234 
1235  clib_longjmp (&p->return_longjmp, n);
1236 
1237  return n;
1238 }
1239 
1240 /* Called in main stack. */
1243 {
1245  uword r;
1246 
1247  a.vm = vm;
1248  a.process = p;
1249  a.frame = f;
1250 
1254  (void *) p->stack + (1 << p->log2_n_stack_bytes));
1255 
1256  return r;
1257 }
1258 
1261 {
1262  uword r;
1269  return r;
1270 }
1271 
1272 static u64
1274  vlib_process_t * p, vlib_frame_t * f, u64 last_time_stamp)
1275 {
1276  vlib_node_main_t *nm = &vm->node_main;
1277  vlib_node_runtime_t *node_runtime = &p->node_runtime;
1278  vlib_node_t *node = vlib_get_node (vm, node_runtime->node_index);
1279  u32 old_process_index;
1280  u64 t;
1281  uword n_vectors, is_suspend;
1282 
1283  if (node->state != VLIB_NODE_STATE_POLLING
1286  return last_time_stamp;
1287 
1289 
1290  t = last_time_stamp;
1291  vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
1292  f ? f->n_vectors : 0, /* is_after */ 0);
1293 
1294  /* Save away current process for suspend. */
1295  old_process_index = nm->current_process_index;
1297 
1298  n_vectors = vlib_process_startup (vm, p, f);
1299 
1300  nm->current_process_index = old_process_index;
1301 
1303  is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND;
1304  if (is_suspend)
1305  {
1307 
1308  n_vectors = 0;
1310  pf->node_runtime_index = node->runtime_index;
1311  pf->frame_index = f ? vlib_frame_index (vm, f) : ~0;
1312  pf->next_frame_index = ~0;
1313 
1314  p->n_suspends += 1;
1316 
1318  {
1319  TWT (tw_timer_wheel) * tw =
1320  (TWT (tw_timer_wheel) *) nm->timing_wheel;
1321  p->stop_timer_handle =
1322  TW (tw_timer_start) (tw,
1324  (node->runtime_index) /* [sic] pool idex */ ,
1325  0 /* timer_id */ ,
1327  }
1328  }
1329  else
1331 
1332  t = clib_cpu_time_now ();
1333 
1334  vlib_elog_main_loop_event (vm, node_runtime->node_index, t, is_suspend,
1335  /* is_after */ 1);
1336 
1338  /* n_calls */ !is_suspend,
1339  /* n_vectors */ n_vectors,
1340  /* n_clocks */ t - last_time_stamp);
1341 
1342  return t;
1343 }
1344 
1345 void
1347 {
1348  vlib_node_main_t *nm = &vm->node_main;
1349  vlib_process_t *p = vec_elt (nm->processes, process_index);
1350  dispatch_process (vm, p, /* frame */ 0, /* cpu_time_now */ 0);
1351 }
1352 
1353 static u64
1355  uword process_index, u64 last_time_stamp)
1356 {
1357  vlib_node_main_t *nm = &vm->node_main;
1358  vlib_node_runtime_t *node_runtime;
1359  vlib_node_t *node;
1360  vlib_frame_t *f;
1361  vlib_process_t *p;
1363  u64 t, n_vectors, is_suspend;
1364 
1365  t = last_time_stamp;
1366 
1367  p = vec_elt (nm->processes, process_index);
1369  return last_time_stamp;
1370 
1373 
1374  pf =
1377 
1378  node_runtime = &p->node_runtime;
1379  node = vlib_get_node (vm, node_runtime->node_index);
1380  f = pf->frame_index != ~0 ? vlib_get_frame (vm, pf->frame_index) : 0;
1381 
1382  vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
1383  f ? f->n_vectors : 0, /* is_after */ 0);
1384 
1385  /* Save away current process for suspend. */
1387 
1388  n_vectors = vlib_process_resume (p);
1389  t = clib_cpu_time_now ();
1390 
1391  nm->current_process_index = ~0;
1392 
1393  is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND;
1394  if (is_suspend)
1395  {
1396  /* Suspend it again. */
1397  n_vectors = 0;
1398  p->n_suspends += 1;
1400  {
1401  p->stop_timer_handle =
1402  TW (tw_timer_start) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
1404  (node->runtime_index) /* [sic] pool idex */ ,
1405  0 /* timer_id */ ,
1407  }
1408  }
1409  else
1410  {
1414  }
1415 
1416  t = clib_cpu_time_now ();
1417  vlib_elog_main_loop_event (vm, node_runtime->node_index, t, !is_suspend,
1418  /* is_after */ 1);
1419 
1421  /* n_calls */ !is_suspend,
1422  /* n_vectors */ n_vectors,
1423  /* n_clocks */ t - last_time_stamp);
1424 
1425  return t;
1426 }
1427 
1428 void vl_api_send_pending_rpc_requests (vlib_main_t *) __attribute__ ((weak));
1429 void
1431 {
1432 }
1433 
1434 
1437 {
1438  vlib_node_main_t *nm = &vm->node_main;
1440  uword i;
1441  u64 cpu_time_now;
1443  u32 *last_node_runtime_indices = 0;
1444 
1445  /* Initialize pending node vector. */
1446  if (is_main)
1447  {
1448  vec_resize (nm->pending_frames, 32);
1449  _vec_len (nm->pending_frames) = 0;
1450  }
1451 
1452  /* Mark time of main loop start. */
1453  if (is_main)
1454  {
1455  cpu_time_now = vm->clib_time.last_cpu_time;
1456  vm->cpu_time_main_loop_start = cpu_time_now;
1457  }
1458  else
1459  cpu_time_now = clib_cpu_time_now ();
1460 
1461  /* Pre-allocate interupt runtime indices and lock. */
1463  vec_alloc (last_node_runtime_indices, 32);
1464  if (!is_main)
1466 
1467  /* Pre-allocate expired nodes. */
1472 
1473  /* Start all processes. */
1474  if (is_main)
1475  {
1476  uword i;
1477  nm->current_process_index = ~0;
1478  for (i = 0; i < vec_len (nm->processes); i++)
1479  cpu_time_now = dispatch_process (vm, nm->processes[i], /* frame */ 0,
1480  cpu_time_now);
1481  }
1482 
1483  while (1)
1484  {
1486 
1487  if (PREDICT_FALSE (_vec_len (vm->pending_rpc_requests) > 0))
1489 
1490  if (!is_main)
1491  {
1493  vec_foreach (fqm, tm->frame_queue_mains)
1494  vlib_frame_queue_dequeue (vm, fqm);
1495  }
1496 
1497  /* Process pre-input nodes. */
1499  cpu_time_now = dispatch_node (vm, n,
1501  VLIB_NODE_STATE_POLLING,
1502  /* frame */ 0,
1503  cpu_time_now);
1504 
1505  /* Next process input nodes. */
1507  cpu_time_now = dispatch_node (vm, n,
1509  VLIB_NODE_STATE_POLLING,
1510  /* frame */ 0,
1511  cpu_time_now);
1512 
1513  if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0))
1514  vm->queue_signal_callback (vm);
1515 
1516  /* Next handle interrupts. */
1517  {
1518  /* unlocked read, for performance */
1519  uword l = _vec_len (nm->pending_interrupt_node_runtime_indices);
1520  uword i;
1521  if (PREDICT_FALSE (l > 0))
1522  {
1523  u32 *tmp;
1524  if (!is_main)
1525  {
1527  /* Re-read w/ lock held, in case another thread added an item */
1528  l = _vec_len (nm->pending_interrupt_node_runtime_indices);
1529  }
1530 
1533  last_node_runtime_indices;
1534  last_node_runtime_indices = tmp;
1535  _vec_len (last_node_runtime_indices) = 0;
1536  if (!is_main)
1538  for (i = 0; i < l; i++)
1539  {
1541  last_node_runtime_indices[i]);
1542  cpu_time_now =
1544  VLIB_NODE_STATE_INTERRUPT,
1545  /* frame */ 0,
1546  cpu_time_now);
1547  }
1548  }
1549  }
1550 
1551  if (is_main)
1552  {
1553  /* Check if process nodes have expired from timing wheel. */
1555 
1558  ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm),
1560 
1562 
1563  if (PREDICT_FALSE
1564  (_vec_len (nm->data_from_advancing_timing_wheel) > 0))
1565  {
1566  uword i;
1567 
1568  processes_timing_wheel_data:
1569  for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel);
1570  i++)
1571  {
1574 
1576  {
1579  di);
1580  vlib_node_t *n =
1582  vlib_process_t *p =
1583  vec_elt (nm->processes, n->runtime_index);
1584  void *data;
1585  data =
1587  te->event_type_index,
1588  te->n_data_elts,
1589  te->n_data_elt_bytes);
1590  if (te->n_data_bytes < sizeof (te->inline_event_data))
1591  clib_memcpy (data, te->inline_event_data,
1592  te->n_data_bytes);
1593  else
1594  {
1595  clib_memcpy (data, te->event_data_as_vector,
1596  te->n_data_bytes);
1598  }
1600  }
1601  else
1602  {
1603  cpu_time_now = clib_cpu_time_now ();
1604  cpu_time_now =
1605  dispatch_suspended_process (vm, di, cpu_time_now);
1606  }
1607  }
1608  _vec_len (nm->data_from_advancing_timing_wheel) = 0;
1609  }
1610  }
1611 
1612  /* Input nodes may have added work to the pending vector.
1613  Process pending vector until there is nothing left.
1614  All pending vectors will be processed from input -> output. */
1615  for (i = 0; i < _vec_len (nm->pending_frames); i++)
1616  cpu_time_now = dispatch_pending_node (vm, i, cpu_time_now);
1617  /* Reset pending vector for next iteration. */
1618  _vec_len (nm->pending_frames) = 0;
1619 
1620  /* Pending internal nodes may resume processes. */
1621  if (is_main && _vec_len (nm->data_from_advancing_timing_wheel) > 0)
1622  goto processes_timing_wheel_data;
1623 
1625 
1626  /* Record time stamp in case there are no enabled nodes and above
1627  calls do not update time stamp. */
1628  cpu_time_now = clib_cpu_time_now ();
1629  }
1630 }
1631 
1632 static void
1634 {
1635  vlib_main_or_worker_loop (vm, /* is_main */ 1);
1636 }
1637 
1638 void
1640 {
1641  vlib_main_or_worker_loop (vm, /* is_main */ 0);
1642 }
1643 
1645 
1646 static clib_error_t *
1648 {
1649  int turn_on_mem_trace = 0;
1650 
1651  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1652  {
1653  if (unformat (input, "memory-trace"))
1654  turn_on_mem_trace = 1;
1655 
1656  else if (unformat (input, "elog-events %d",
1657  &vm->elog_main.event_ring_size))
1658  ;
1659  else if (unformat (input, "elog-post-mortem-dump"))
1660  vm->elog_post_mortem_dump = 1;
1661  else
1662  return unformat_parse_error (input);
1663  }
1664 
1665  unformat_free (input);
1666 
1667  /* Enable memory trace as early as possible. */
1668  if (turn_on_mem_trace)
1669  clib_mem_trace (1);
1670 
1671  return 0;
1672 }
1673 
1675 
1676 static void
1678 {
1679 }
1680 
1681 #define foreach_weak_reference_stub \
1682 _(vlib_map_stat_segment_init) \
1683 _(vpe_api_init) \
1684 _(vlibmemory_init) \
1685 _(map_api_segment_init)
1686 
1687 #define _(name) \
1688 clib_error_t *name (vlib_main_t *vm) __attribute__((weak)); \
1689 clib_error_t *name (vlib_main_t *vm) { return 0; }
1691 #undef _
1692 
1693 /* Main function. */
1694 int
1696 {
1697  clib_error_t *volatile error;
1698  vlib_node_main_t *nm = &vm->node_main;
1699 
1701 
1702  clib_time_init (&vm->clib_time);
1703 
1704  /* Turn on event log. */
1705  if (!vm->elog_main.event_ring_size)
1706  vm->elog_main.event_ring_size = 128 << 10;
1708  elog_enable_disable (&vm->elog_main, 1);
1709 
1710  /* Default name. */
1711  if (!vm->name)
1712  vm->name = "VLIB";
1713 
1714  if ((error = unix_physmem_init (vm)))
1715  {
1716  clib_error_report (error);
1717  goto done;
1718  }
1719 
1720  if ((error = vlib_buffer_main_init (vm)))
1721  {
1722  clib_error_report (error);
1723  goto done;
1724  }
1725 
1726  if ((error = vlib_thread_init (vm)))
1727  {
1728  clib_error_report (error);
1729  goto done;
1730  }
1731 
1732  if ((error = vlib_map_stat_segment_init (vm)))
1733  {
1734  clib_error_report (error);
1735  goto done;
1736  }
1737 
1738  /* Register static nodes so that init functions may use them. */
1740 
1741  /* Set seed for random number generator.
1742  Allow user to specify seed to make random sequence deterministic. */
1743  if (!unformat (input, "seed %wd", &vm->random_seed))
1744  vm->random_seed = clib_cpu_time_now ();
1746 
1747  /* Initialize node graph. */
1748  if ((error = vlib_node_main_init (vm)))
1749  {
1750  /* Arrange for graph hook up error to not be fatal when debugging. */
1751  if (CLIB_DEBUG > 0)
1752  clib_error_report (error);
1753  else
1754  goto done;
1755  }
1756 
1757  /* Direct call / weak reference, for vlib standalone use-cases */
1758  if ((error = vpe_api_init (vm)))
1759  {
1760  clib_error_report (error);
1761  goto done;
1762  }
1763 
1764  if ((error = vlibmemory_init (vm)))
1765  {
1766  clib_error_report (error);
1767  goto done;
1768  }
1769 
1770  if ((error = map_api_segment_init (vm)))
1771  {
1772  clib_error_report (error);
1773  goto done;
1774  }
1775 
1776  /* See unix/main.c; most likely already set up */
1777  if (vm->init_functions_called == 0)
1778  vm->init_functions_called = hash_create (0, /* value bytes */ 0);
1779  if ((error = vlib_call_all_init_functions (vm)))
1780  goto done;
1781 
1782  /* Create default buffer free list. */
1784  "default");
1785 
1786  nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)),
1788 
1790  _vec_len (nm->data_from_advancing_timing_wheel) = 0;
1791 
1792  /* Create the process timing wheel */
1793  TW (tw_timer_wheel_init) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
1794  0 /* no callback */ ,
1795  10e-6 /* timer period 10us */ ,
1796  ~0 /* max expirations per call */ );
1797 
1799  _vec_len (vm->pending_rpc_requests) = 0;
1800 
1802  {
1804  vm->main_loop_exit_set = 1;
1805  break;
1806 
1808  goto done;
1809 
1810  default:
1811  error = vm->main_loop_error;
1812  goto done;
1813  }
1814 
1815  if ((error = vlib_call_all_config_functions (vm, input, 0 /* is_early */ )))
1816  goto done;
1817 
1818  /* Call all main loop enter functions. */
1819  {
1820  clib_error_t *sub_error;
1821  sub_error = vlib_call_all_main_loop_enter_functions (vm);
1822  if (sub_error)
1823  clib_error_report (sub_error);
1824  }
1825 
1826  vlib_main_loop (vm);
1827 
1828 done:
1829  /* Call all exit functions. */
1830  {
1831  clib_error_t *sub_error;
1832  sub_error = vlib_call_all_main_loop_exit_functions (vm);
1833  if (sub_error)
1834  clib_error_report (sub_error);
1835  }
1836 
1837  if (error)
1838  clib_error_report (error);
1839 
1840  return 0;
1841 }
1842 
1843 /*
1844  * fd.io coding-style-patch-verification: ON
1845  *
1846  * Local Variables:
1847  * eval: (c-set-style "gnu")
1848  * End:
1849  */
u32 * next_nodes
Definition: node.h:324
static void elog_enable_disable(elog_main_t *em, int is_enabled)
Enable or disable event logging.
Definition: elog.h:217
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:437
u32 TW() tw_timer_start(TWT(tw_timer_wheel)*tw, u32 user_id, u32 timer_id, u64 interval)
Start a Tw Timer.
clib_error_t * vlib_call_all_main_loop_exit_functions(vlib_main_t *vm)
Definition: init.c:87
uword * pending_rpc_requests
Definition: main.h:218
vlib_main_t vlib_global_main
Definition: main.c:1644
f64 time
Absolute time as floating point number in seconds.
Definition: elog.h:66
void TW() tw_timer_wheel_init(TWT(tw_timer_wheel)*tw, void *expired_timer_callback, f64 timer_interval_in_seconds, u32 max_expirations)
Initialize a tw timer wheel template instance.
u16 vector_size
Definition: node.h:310
u32 max_clock
Maximum clock cycle for an invocation.
Definition: node.h:459
u32 next_frame_index
Start of next frames for this node.
Definition: node.h:470
never_inline void vlib_node_runtime_sync_stats(vlib_main_t *vm, vlib_node_runtime_t *r, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:540
static clib_error_t * show_frame_stats(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:243
#define hash_set(h, key, value)
Definition: hash.h:255
vlib_node_type_t
Definition: node.h:58
clib_error_t * vlib_call_all_main_loop_enter_functions(vlib_main_t *vm)
Definition: init.c:80
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
Definition: lock.h:89
static void vlib_next_frame_change_ownership(vlib_main_t *vm, vlib_node_runtime_t *node_runtime, u32 next_index)
Definition: main.c:273
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
Definition: lock.h:74
static vlib_next_frame_t * vlib_node_get_next_frame(vlib_main_t *vm, u32 node_index, u32 next_index)
Get pointer to frame by (node_index, next_index).
Definition: node_funcs.h:325
u32 interrupt_threshold_vector_length
Definition: node.h:696
vlib_process_t ** processes
Definition: node.h:717
static uword vlib_process_bootstrap(uword _a)
Definition: main.c:1215
vlib_node_runtime_t node_runtime
Definition: node.h:531
a
Definition: bitmap.h:538
u32 n_suspends
Definition: node.h:559
#define VLIB_PENDING_FRAME_NO_NEXT_FRAME
Definition: node.h:442
vlib_frame_t * vlib_get_next_frame_internal(vlib_main_t *vm, vlib_node_runtime_t *node, u32 next_index, u32 allocate_new_next_frame)
Definition: main.c:356
static void vlib_elog_main_loop_event(vlib_main_t *vm, u32 node_index, u64 time, u32 n_vectors, u32 is_return)
Definition: main.c:867
static clib_error_t * elog_save_buffer(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:669
int elog_post_mortem_dump
Definition: main.h:200
#define PREDICT_TRUE(x)
Definition: clib.h:106
uword random_seed
Definition: main.h:170
unsigned long u64
Definition: types.h:89
static void vlib_increment_main_loop_counter(vlib_main_t *vm)
Definition: main.h:330
elog_time_stamp_t init_time
Timestamps.
Definition: elog.h:169
void clib_random_buffer_init(clib_random_buffer_t *b, uword seed)
Definition: random_buffer.c:62
u32 index
Definition: node.h:273
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:228
static void dummy_queue_signal_callback(vlib_main_t *vm)
Definition: main.c:1677
u32 current_process_index
Definition: node.h:720
u32 *TW() tw_timer_expire_timers_vec(TWT(tw_timer_wheel)*tw, f64 now, u32 *vec)
u32 thread_index
Definition: main.h:179
static_always_inline uword vlib_process_startup(vlib_main_t *vm, vlib_process_t *p, vlib_frame_t *f)
Definition: main.c:1242
u32 main_loop_exit_set
Definition: main.h:91
static u64 dispatch_pending_node(vlib_main_t *vm, uword pending_frame_index, u64 last_time_stamp)
Definition: main.c:1087
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
static u64 clib_cpu_time_now(void)
Definition: time.h:73
elog_track_t elog_track
Definition: threads.h:116
u32 clocks_since_last_overflow
Number of clock cycles.
Definition: node.h:457
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:562
int i
static void vlib_process_update_stats(vlib_main_t *vm, vlib_process_t *p, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:643
static void elog_show_buffer_internal(vlib_main_t *vm, u32 n_events_to_show)
Definition: main.c:804
#define VLIB_MAIN_LOOP_EXIT_CLI
Definition: main.h:98
void clib_longjmp(clib_longjmp_t *save, uword return_value)
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
static vlib_frame_size_t * get_frame_size_info(vlib_node_main_t *nm, u32 n_scalar_bytes, u32 n_vector_bytes)
Definition: main.c:95
static clib_error_t * elog_show_buffer(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:832
u8 * format_elog_event(u8 *s, va_list *va)
Definition: elog.c:296
clib_time_t clib_time
Definition: main.h:63
clib_spinlock_t pending_interrupt_lock
Definition: node.h:690
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:228
uword clib_setjmp(clib_longjmp_t *save, uword return_value_not_taken)
#define vec_alloc(V, N)
Allocate space for N more elements (no header, unspecified alignment)
Definition: vec.h:278
unsigned char u8
Definition: types.h:56
static uword vlib_process_stack_is_valid(vlib_process_t *p)
Definition: main.c:1201
static vlib_frame_t * vlib_get_frame_no_check(vlib_main_t *vm, uword frame_index)
Definition: node_funcs.h:216
double f64
Definition: types.h:142
u16 scalar_size
Definition: node.h:310
u8 state
Definition: node.h:301
u16 thread_index
thread this node runs on
Definition: node.h:498
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:212
static void vlib_worker_thread_barrier_check(void)
Definition: threads.h:399
u32 * pending_interrupt_node_runtime_indices
Definition: node.h:689
u32 input_main_loops_per_call
For input nodes: decremented on each main loop interation until it reaches zero and function is calle...
Definition: node.h:475
static void vlib_main_loop(vlib_main_t *vm)
Definition: main.c:1633
#define foreach_weak_reference_stub
Definition: main.c:1681
#define static_always_inline
Definition: clib.h:93
vlib_node_function_t * function
Node function to call.
Definition: node.h:449
void elog_init(elog_main_t *em, u32 n_events)
Definition: elog.c:466
#define VLIB_FRAME_MAGIC
#define always_inline
Definition: clib.h:92
u32 main_loop_vectors_processed
Definition: main.h:78
#define VLIB_INVALID_NODE_INDEX
Definition: node.h:361
static u32 vlib_frame_alloc(vlib_main_t *vm, vlib_node_runtime_t *from_node_runtime, u32 to_next_index)
Definition: main.c:169
u16 log2_n_stack_bytes
Definition: node.h:554
vlib_node_t ** nodes
Definition: node.h:676
u32 vectors_since_last_overflow
Number of vector elements processed by this node.
Definition: node.h:467
vlib_frame_t * frame
Definition: main.c:1210
void di(unformat_input_t *i)
Definition: unformat.c:163
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:182
#define clib_error_return(e, args...)
Definition: error.h:99
#define VLIB_FRAME_ALIGN
Definition: node.h:365
static void vlib_process_sync_stats(vlib_main_t *vm, vlib_process_t *p, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:558
static uword vlib_timing_wheel_data_is_timed_event(u32 d)
Definition: node.h:650
#define VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE
Definition: node.h:297
#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES
Definition: buffer.h:440
#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
Definition: node.h:545
#define vec_resize(V, N)
Resize a vector (no header, unspecified alignment) Add N elements to end of given vector V...
Definition: vec.h:240
unsigned int u32
Definition: types.h:88
u64 cpu_time_main_loop_start
Definition: main.h:72
vlib_node_runtime_t * nodes_by_type[VLIB_N_NODE_TYPE]
Definition: node.h:686
static u32 vlib_frame_bytes(u32 n_scalar_bytes, u32 n_vector_bytes)
Definition: main.c:58
u32 * free_frame_indices
Definition: node.h:519
#define VLIB_MAIN_LOOP_EXIT_NONE
Definition: main.h:95
clib_error_t * vlib_node_main_init(vlib_main_t *vm)
Definition: node.c:629
#define VLIB_FRAME_SIZE
Definition: node.h:364
static vlib_next_frame_t * vlib_node_runtime_get_next_frame(vlib_main_t *vm, vlib_node_runtime_t *n, u32 next_index)
Definition: node_funcs.h:290
#define VLIB_PROCESS_IS_RUNNING
Definition: node.h:551
#define VLIB_PROCESS_RETURN_LONGJMP_RETURN
Definition: node.h:536
u64 max_clock_n
Definition: node.h:231
int callbacks_registered
Definition: buffer.h:455
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
u32 calls_since_last_overflow
Number of calls.
Definition: node.h:465
char * name
Definition: main.h:104
vlib_worker_thread_t * vlib_worker_threads
Definition: threads.c:35
#define hash_get(h, key)
Definition: hash.h:249
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:464
static clib_error_t * elog_write_file(elog_main_t *em, char *clib_file, int flush_ring)
Definition: elog.h:528
u32 next_frame_index
Definition: node.h:439
static void * vlib_process_signal_event_helper(vlib_node_main_t *nm, vlib_node_t *n, vlib_process_t *p, uword t, uword n_data_elts, uword n_data_elt_bytes)
Definition: node_funcs.h:769
vlib_node_stats_t stats_total
Definition: node.h:263
static void vlib_put_next_frame_validate(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 next_index, u32 n_vectors_left)
Definition: main.c:418
u16 state
Input node state.
Definition: node.h:488
u32 frame_index
Definition: node.h:389
vlib_signal_timed_event_data_t * signal_timed_event_data_pool
Definition: node.h:707
static u32 vlib_frame_index(vlib_main_t *vm, vlib_frame_t *f)
Definition: node_funcs.h:245
#define VLIB_FRAME_IS_ALLOCATED
Definition: node.h:406
#define VLIB_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:398
#define v
Definition: acl.c:491
struct _unformat_input_t unformat_input_t
static_always_inline u64 dispatch_node(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_node_type_t type, vlib_node_state_t dispatch_state, vlib_frame_t *frame, u64 last_time_stamp)
Definition: main.c:914
u32 n_alloc_frames
Definition: node.h:516
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:191
static uword vlib_timing_wheel_data_get_index(u32 d)
Definition: node.h:668
u8 inline_event_data[64-3 *sizeof(u32)-2 *sizeof(u16)]
Definition: node.h:641
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:274
elog_event_type_t * node_return_elog_event_types
Definition: main.h:165
#define TWT(a)
void vlib_frame_free(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_frame_t *f)
Definition: main.c:211
u32 polling_threshold_vector_length
Definition: node.h:695
u64 * n_vectors_by_next_node
Definition: node.h:333
#define PREDICT_FALSE(x)
Definition: clib.h:105
f64 seconds_per_clock
Definition: time.h:57
void elog_post_mortem_dump(void)
Definition: main.c:706
static vlib_frame_t * vlib_get_frame(vlib_main_t *vm, uword frame_index)
Definition: node_funcs.h:237
u32 node_index
Node index.
Definition: node.h:473
uword * init_functions_called
Definition: main.h:176
#define VLIB_ELOG_MAIN_LOOP
Definition: main.h:56
#define VLIB_PROCESS_STACK_MAGIC
Definition: node.h:602
#define VLIB_PROCESS_RESUME_LONGJMP_RESUME
Definition: node.h:542
void clib_time_init(clib_time_t *c)
Definition: time.c:178
uword * frame_size_hash
Definition: node.h:732
void elog_alloc(elog_main_t *em, u32 n_events)
Definition: elog.c:452
clib_error_t * vlibmemory_init(vlib_main_t *vm)
Definition: memory_api.c:891
volatile u32 queue_signal_pending
Definition: main.h:191
clib_error_t * map_api_segment_init(vlib_main_t *vm)
Definition: memory_api.c:463
u32 flags
Definition: vhost_user.h:110
static u64 dispatch_process(vlib_main_t *vm, vlib_process_t *p, vlib_frame_t *f, u64 last_time_stamp)
Definition: main.c:1273
u32 wraps
Definition: main.c:55
static u32 vlib_frame_index_no_check(vlib_main_t *vm, vlib_frame_t *f)
Definition: node_funcs.h:224
u32 owner_node_index
Definition: node.h:344
void vlib_register_all_static_nodes(vlib_main_t *vm)
Definition: node.c:538
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
Definition: init.h:195
static u32 vlib_node_runtime_update_main_loop_vector_stats(vlib_main_t *vm, vlib_node_runtime_t *node, uword n_vectors)
Definition: node_funcs.h:1047
#define UNFORMAT_END_OF_INPUT
Definition: format.h:144
svmdb_client_t * c
u16 n_vectors
Definition: node.h:380
u32 runtime_index
Definition: node.h:276
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:77
vlib_main_t * vm
Definition: buffer.c:294
static uword elog_n_events_in_buffer(elog_main_t *em)
Return number of events in the event-log buffer.
Definition: elog.h:188
static_always_inline void vlib_main_or_worker_loop(vlib_main_t *vm, int is_main)
Definition: main.c:1436
u32 node_runtime_index
Definition: node.h:433
vlib_pending_frame_t * pending_frames
Definition: node.h:702
int vlib_frame_queue_dequeue(vlib_main_t *vm, vlib_frame_queue_main_t *fqm)
Definition: threads.c:1656
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
clib_error_t * vpe_api_init(vlib_main_t *vm)
Definition: api.c:480
#define VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:286
static u32 vlib_node_runtime_update_stats(vlib_main_t *vm, vlib_node_runtime_t *node, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:607
#define clib_warning(format, args...)
Definition: error.h:59
#define VLIB_PROCESS_RESUME_PENDING
Definition: node.h:548
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
Definition: node_funcs.h:89
#define clib_memcpy(a, b, c)
Definition: string.h:75
#define VLIB_BUFFER_TRACE_TRAJECTORY
Compile time buffer trajectory tracing option Turn this on if you run into "bad monkey" contexts...
Definition: buffer.h:537
elog_main_t elog_main
Definition: main.h:161
clib_error_t * vlib_buffer_main_init(struct vlib_main_t *vm)
Definition: buffer.c:1018
#define VLIB_FRAME_PENDING
Definition: node.h:409
static u32 vlib_frame_vector_byte_offset(u32 scalar_size)
Definition: node_funcs.h:256
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:454
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:439
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:235
u32 n_total_events_disable_limit
When count reaches limit logging is disabled.
Definition: elog.h:138
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:154
#define never_inline
Definition: clib.h:89
#define hash_create(elts, value_bytes)
Definition: hash.h:696
#define uword_to_pointer(u, type)
Definition: types.h:136
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
Definition: node.h:492
#define ASSERT(truth)
u64 last_cpu_time
Definition: time.h:50
static void elog_reset_buffer(elog_main_t *em)
Reset the event buffer.
Definition: elog.h:207
vlib_frame_queue_main_t * frame_queue_mains
Definition: threads.h:335
#define ELOG_TRACK_DATA(em, f, track)
Definition: elog.h:475
uword event_ring_size
Power of 2 number of elements in ring.
Definition: elog.h:144
u16 flags
Definition: node.h:371
u32 main_loop_nodes_processed
Definition: main.h:79
u64 cpu
CPU cycle counter.
Definition: elog.h:125
#define clib_error_report(e)
Definition: error.h:113
static uword pointer_to_uword(const void *p)
Definition: types.h:131
static u32 vlib_frame_alloc_to_node(vlib_main_t *vm, u32 to_node_index, u32 frame_flags)
Definition: main.c:115
void(* queue_signal_callback)(struct vlib_main_t *)
Definition: main.h:193
void vlib_gdb_show_event_log(void)
Definition: main.c:861
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
static_always_inline uword vlib_process_resume(vlib_process_t *p)
Definition: main.c:1260
#define VLIB_FRAME_OWNER
Definition: node.h:403
void vlib_node_sync_stats(vlib_main_t *vm, vlib_node_t *n)
Definition: main.c:570
#define vec_elt(v, i)
Get vector value at index i.
int vlib_main(vlib_main_t *volatile vm, unformat_input_t *input)
Definition: main.c:1695
#define unformat_parse_error(input)
Definition: format.h:268
void vlib_worker_loop(vlib_main_t *vm)
Definition: main.c:1639
static vlib_node_t * vlib_get_next_node(vlib_main_t *vm, u32 node_index, u32 next_index)
Get vlib node by graph arc (next) index.
Definition: node_funcs.h:72
void vl_api_send_pending_rpc_requests(vlib_main_t *)
Definition: main.c:1430
#define clib_mem_alloc_aligned_no_fail(size, align)
Definition: mem.h:146
CJ_GLOBAL_LOG_PROTOTYPE
Definition: main.c:49
u32 * data_from_advancing_timing_wheel
Definition: node.h:710
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u32 input_node_counts_by_state[VLIB_N_NODE_STATE]
Definition: node.h:729
vlib_pending_frame_t * suspended_process_frames
Definition: node.h:723
vlib_node_main_t node_main
Definition: main.h:135
static u32 * vlib_frame_find_magic(vlib_frame_t *f, vlib_node_t *node)
Definition: main.c:83
u64 uword
Definition: types.h:112
u32 owner_next_index
Definition: node.h:344
u8 vector_size
Definition: node.h:377
vlib_next_frame_t * next_frames
Definition: node.h:699
static clib_error_t * elog_stop(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:732
static void unformat_free(unformat_input_t *i)
Definition: format.h:162
clib_error_t * vlib_call_all_config_functions(vlib_main_t *vm, unformat_input_t *input, int is_early)
Definition: init.c:94
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:267
#define VLIB_FRAME_TRACE
Definition: node.h:415
u32 stop_timer_handle
Definition: node.h:584
clib_error_t * vlib_call_all_init_functions(vlib_main_t *vm)
Definition: init.c:67
#define VLIB_FRAME_SIZE_EXTRA
static void elog_track(elog_main_t *em, elog_event_type_t *type, elog_track_t *track, u32 data)
Log a single-datum event to a specific track, non-inline version.
Definition: elog.h:395
clib_error_t * unix_physmem_init(vlib_main_t *vm)
Definition: physmem.c:216
static u64 dispatch_suspended_process(vlib_main_t *vm, uword process_index, u64 last_time_stamp)
Definition: main.c:1354
vlib_frame_size_t * frame_sizes
Definition: node.h:735
u64 resume_clock_interval
Definition: node.h:581
static void add_trajectory_trace(vlib_buffer_t *b, u32 node_index)
Definition: main.c:903
u16 flags
Definition: node.h:544
elog_event_type_t * node_call_elog_event_types
Definition: main.h:164
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:120
vlib_node_type_t type
Definition: node.h:270
static vlib_process_t * vlib_get_process_from_node(vlib_main_t *vm, vlib_node_t *node)
Definition: node_funcs.h:207
uword clib_calljmp(uword(*func)(uword func_arg), uword func_arg, void *stack)
u8 * format_unformat_error(u8 *s, va_list *va)
Definition: unformat.c:91
u32 name_elog_string
Definition: node.h:260
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1545
vlib_node_state_t
Definition: node.h:243
u64 init_cpu_time
Definition: time.h:60
u32 suspended_process_frame_index
Definition: node.h:556
static void validate_frame_magic(vlib_main_t *vm, vlib_frame_t *f, vlib_node_t *n, uword next_index)
Definition: main.c:347
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
vlib_buffer_main_t buffer_main
Definition: buffer.c:52
static clib_error_t * vlib_main_configure(vlib_main_t *vm, unformat_input_t *input)
Definition: main.c:1647
static vlib_node_t * vlib_get_node(vlib_main_t *vm, u32 i)
Get vlib node by index.
Definition: node_funcs.h:59
static clib_error_t * vlib_cli_elog_clear(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:652
#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT
Definition: node.h:546
#define vec_foreach(var, vec)
Vector iterator.
elog_track_t default_track
Default track.
Definition: elog.h:163
clib_longjmp_t return_longjmp
Definition: node.h:534
u16 flags
Copy of main node flags.
Definition: node.h:486
u32 node_runtime_index
Definition: node.h:392
u32 n_total_events
Total number of events in buffer.
Definition: elog.h:134
u8 scalar_size
Definition: node.h:374
clib_longjmp_t main_loop_exit
Definition: main.h:94
clib_longjmp_t resume_longjmp
Definition: node.h:540
void clib_mem_trace(int enable)
Definition: mem_mheap.c:154
static clib_error_t * elog_resize(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:772
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:295
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:62
#define TW(a)
clib_error_t * vlib_map_stat_segment_init(void)
Definition: stat_segment.c:190
static u32 vlib_timing_wheel_data_set_suspended_process(u32 i)
Definition: node.h:656
void vlib_start_process(vlib_main_t *vm, uword process_index)
Definition: main.c:1346
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:681
void * timing_wheel
Definition: node.h:705
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
clib_random_buffer_t random_buffer
Definition: main.h:173
static clib_error_t * elog_restart(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:752
#define VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE
Definition: node.h:298
vlib_buffer_free_list_index_t vlib_buffer_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
Definition: buffer.c:415
u8 *(* validate_frame)(struct vlib_main_t *vm, struct vlib_node_runtime_t *, struct vlib_frame_t *f)
Definition: node.h:354
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:972
clib_error_t * main_loop_error
Definition: main.h:101
vlib_process_t * process
Definition: main.c:1209
elog_event_t * elog_peek_events(elog_main_t *em)
convert event ring events to events, and return them as a vector.
Definition: elog.c:509
static uword elog_buffer_capacity(elog_main_t *em)
Return number of events which can fit in the event buffer.
Definition: elog.h:198
u64 cpu_time_last_node_dispatch
Definition: main.h:69
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:170
#define VLIB_FRAME_FREE_AFTER_DISPATCH
Definition: node.h:412
clib_error_t * vlib_thread_init(vlib_main_t *vm)
Definition: threads.c:322
u32 vectors_since_last_overflow
Definition: node.h:418
u32 max_clock_n
Number of vectors in the recorded max_clock.
Definition: node.h:462
#define VLIB_PROCESS_RETURN_LONGJMP_SUSPEND
Definition: node.h:537