FD.io VPP  v18.10-34-gcce845e
Vector Packet Processing
main.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * main.c: main vector processing loop
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #include <math.h>
41 #include <vppinfra/format.h>
42 #include <vlib/vlib.h>
43 #include <vlib/threads.h>
45 
46 #include <vlib/unix/unix.h>
47 #include <vlib/unix/cj.h>
48 
50 
51 /* Actually allocate a few extra slots of vector data to support
52  speculative vector enqueues which overflow vector data in next frame. */
53 #define VLIB_FRAME_SIZE_ALLOC (VLIB_FRAME_SIZE + 4)
54 
56 
58 vlib_frame_bytes (u32 n_scalar_bytes, u32 n_vector_bytes)
59 {
60  u32 n_bytes;
61 
62  /* Make room for vlib_frame_t plus scalar arguments. */
63  n_bytes = vlib_frame_vector_byte_offset (n_scalar_bytes);
64 
65  /* Make room for vector arguments.
66  Allocate a few extra slots of vector data to support
67  speculative vector enqueues which overflow vector data in next frame. */
68 #define VLIB_FRAME_SIZE_EXTRA 4
69  n_bytes += (VLIB_FRAME_SIZE + VLIB_FRAME_SIZE_EXTRA) * n_vector_bytes;
70 
71  /* Magic number is first 32bit number after vector data.
72  Used to make sure that vector data is never overrun. */
73 #define VLIB_FRAME_MAGIC (0xabadc0ed)
74  n_bytes += sizeof (u32);
75 
76  /* Pad to cache line. */
77  n_bytes = round_pow2 (n_bytes, CLIB_CACHE_LINE_BYTES);
78 
79  return n_bytes;
80 }
81 
84 {
85  void *p = f;
86 
88 
90 
91  return p;
92 }
93 
94 static vlib_frame_size_t *
96  u32 n_scalar_bytes, u32 n_vector_bytes)
97 {
98  uword key = (n_scalar_bytes << 16) | n_vector_bytes;
99  uword *p, i;
100 
101  p = hash_get (nm->frame_size_hash, key);
102  if (p)
103  i = p[0];
104  else
105  {
106  i = vec_len (nm->frame_sizes);
107  vec_validate (nm->frame_sizes, i);
108  hash_set (nm->frame_size_hash, key, i);
109  }
110 
111  return vec_elt_at_index (nm->frame_sizes, i);
112 }
113 
114 static u32
116  u32 frame_flags)
117 {
118  vlib_node_main_t *nm = &vm->node_main;
119  vlib_frame_size_t *fs;
120  vlib_node_t *to_node;
121  vlib_frame_t *f;
122  u32 fi, l, n, scalar_size, vector_size;
123 
124  to_node = vlib_get_node (vm, to_node_index);
125 
126  scalar_size = to_node->scalar_size;
127  vector_size = to_node->vector_size;
128 
129  fs = get_frame_size_info (nm, scalar_size, vector_size);
130  n = vlib_frame_bytes (scalar_size, vector_size);
131  if ((l = vec_len (fs->free_frame_indices)) > 0)
132  {
133  /* Allocate from end of free list. */
134  fi = fs->free_frame_indices[l - 1];
135  f = vlib_get_frame_no_check (vm, fi);
136  _vec_len (fs->free_frame_indices) = l - 1;
137  }
138  else
139  {
141  fi = vlib_frame_index_no_check (vm, f);
142  }
143 
144  /* Poison frame when debugging. */
145  if (CLIB_DEBUG > 0)
146  memset (f, 0xfe, n);
147 
148  /* Insert magic number. */
149  {
150  u32 *magic;
151 
152  magic = vlib_frame_find_magic (f, to_node);
153  *magic = VLIB_FRAME_MAGIC;
154  }
155 
156  f->frame_flags = VLIB_FRAME_IS_ALLOCATED | frame_flags;
157  f->n_vectors = 0;
158  f->scalar_size = scalar_size;
159  f->vector_size = vector_size;
160 
161  fs->n_alloc_frames += 1;
162 
163  return fi;
164 }
165 
166 /* Allocate a frame for from FROM_NODE to TO_NODE via TO_NEXT_INDEX.
167  Returns frame index. */
168 static u32
170  u32 to_next_index)
171 {
172  vlib_node_t *from_node;
173 
174  from_node = vlib_get_node (vm, from_node_runtime->node_index);
175  ASSERT (to_next_index < vec_len (from_node->next_nodes));
176 
177  return vlib_frame_alloc_to_node (vm, from_node->next_nodes[to_next_index],
178  /* frame_flags */ 0);
179 }
180 
181 vlib_frame_t *
183 {
184  u32 fi = vlib_frame_alloc_to_node (vm, to_node_index,
185  /* frame_flags */
187  return vlib_get_frame (vm, fi);
188 }
189 
190 void
192 {
194  vlib_node_t *to_node;
195 
196  if (f->n_vectors == 0)
197  return;
198 
199  to_node = vlib_get_node (vm, to_node_index);
200 
201  vec_add2 (vm->node_main.pending_frames, p, 1);
202 
204  p->frame_index = vlib_frame_index (vm, f);
205  p->node_runtime_index = to_node->runtime_index;
207 }
208 
209 /* Free given frame. */
210 void
212 {
213  vlib_node_main_t *nm = &vm->node_main;
214  vlib_node_t *node;
215  vlib_frame_size_t *fs;
216  u32 frame_index;
217 
219 
220  node = vlib_get_node (vm, r->node_index);
221  fs = get_frame_size_info (nm, node->scalar_size, node->vector_size);
222 
223  frame_index = vlib_frame_index (vm, f);
224 
226 
227  /* No next frames may point to freed frame. */
228  if (CLIB_DEBUG > 0)
229  {
230  vlib_next_frame_t *nf;
232  ASSERT (nf->frame_index != frame_index);
233  }
234 
236 
237  vec_add1 (fs->free_frame_indices, frame_index);
238  ASSERT (fs->n_alloc_frames > 0);
239  fs->n_alloc_frames -= 1;
240 }
241 
242 static clib_error_t *
244  unformat_input_t * input, vlib_cli_command_t * cmd)
245 {
246  vlib_node_main_t *nm = &vm->node_main;
247  vlib_frame_size_t *fs;
248 
249  vlib_cli_output (vm, "%=6s%=12s%=12s", "Size", "# Alloc", "# Free");
250  vec_foreach (fs, nm->frame_sizes)
251  {
252  u32 n_alloc = fs->n_alloc_frames;
253  u32 n_free = vec_len (fs->free_frame_indices);
254 
255  if (n_alloc + n_free > 0)
256  vlib_cli_output (vm, "%=6d%=12d%=12d",
257  fs - nm->frame_sizes, n_alloc, n_free);
258  }
259 
260  return 0;
261 }
262 
263 /* *INDENT-OFF* */
264 VLIB_CLI_COMMAND (show_frame_stats_cli, static) = {
265  .path = "show vlib frame-allocation",
266  .short_help = "Show node dispatch frame statistics",
267  .function = show_frame_stats,
268 };
269 /* *INDENT-ON* */
270 
271 /* Change ownership of enqueue rights to given next node. */
272 static void
274  vlib_node_runtime_t * node_runtime,
275  u32 next_index)
276 {
277  vlib_node_main_t *nm = &vm->node_main;
278  vlib_next_frame_t *next_frame;
279  vlib_node_t *node, *next_node;
280 
281  node = vec_elt (nm->nodes, node_runtime->node_index);
282 
283  /* Only internal & input nodes are allowed to call other nodes. */
285  || node->type == VLIB_NODE_TYPE_INPUT
286  || node->type == VLIB_NODE_TYPE_PROCESS);
287 
288  ASSERT (vec_len (node->next_nodes) == node_runtime->n_next_nodes);
289 
290  next_frame =
291  vlib_node_runtime_get_next_frame (vm, node_runtime, next_index);
292  next_node = vec_elt (nm->nodes, node->next_nodes[next_index]);
293 
294  if (next_node->owner_node_index != VLIB_INVALID_NODE_INDEX)
295  {
296  /* Get frame from previous owner. */
297  vlib_next_frame_t *owner_next_frame;
298  vlib_next_frame_t tmp;
299 
300  owner_next_frame =
302  next_node->owner_node_index,
303  next_node->owner_next_index);
304 
305  /* Swap target next frame with owner's. */
306  tmp = owner_next_frame[0];
307  owner_next_frame[0] = next_frame[0];
308  next_frame[0] = tmp;
309 
310  /*
311  * If next_frame is already pending, we have to track down
312  * all pending frames and fix their next_frame_index fields.
313  */
314  if (next_frame->flags & VLIB_FRAME_PENDING)
315  {
317  if (next_frame->frame_index != ~0)
318  {
319  vec_foreach (p, nm->pending_frames)
320  {
321  if (p->frame_index == next_frame->frame_index)
322  {
323  p->next_frame_index =
324  next_frame - vm->node_main.next_frames;
325  }
326  }
327  }
328  }
329  }
330  else
331  {
332  /* No previous owner. Take ownership. */
333  next_frame->flags |= VLIB_FRAME_OWNER;
334  }
335 
336  /* Record new owner. */
337  next_node->owner_node_index = node->index;
338  next_node->owner_next_index = next_index;
339 
340  /* Now we should be owner. */
341  ASSERT (next_frame->flags & VLIB_FRAME_OWNER);
342 }
343 
344 /* Make sure that magic number is still there.
345  Otherwise, it is likely that caller has overrun frame arguments. */
346 always_inline void
348  vlib_frame_t * f, vlib_node_t * n, uword next_index)
349 {
350  vlib_node_t *next_node = vlib_get_node (vm, n->next_nodes[next_index]);
351  u32 *magic = vlib_frame_find_magic (f, next_node);
352  ASSERT (VLIB_FRAME_MAGIC == magic[0]);
353 }
354 
355 vlib_frame_t *
357  vlib_node_runtime_t * node,
358  u32 next_index, u32 allocate_new_next_frame)
359 {
360  vlib_frame_t *f;
361  vlib_next_frame_t *nf;
362  u32 n_used;
363 
364  nf = vlib_node_runtime_get_next_frame (vm, node, next_index);
365 
366  /* Make sure this next frame owns right to enqueue to destination frame. */
367  if (PREDICT_FALSE (!(nf->flags & VLIB_FRAME_OWNER)))
368  vlib_next_frame_change_ownership (vm, node, next_index);
369 
370  /* ??? Don't need valid flag: can use frame_index == ~0 */
372  {
373  nf->frame_index = vlib_frame_alloc (vm, node, next_index);
375  }
376 
377  f = vlib_get_frame (vm, nf->frame_index);
378 
379  /* Has frame been removed from pending vector (e.g. finished dispatching)?
380  If so we can reuse frame. */
381  if ((nf->flags & VLIB_FRAME_PENDING)
382  && !(f->frame_flags & VLIB_FRAME_PENDING))
383  {
384  nf->flags &= ~VLIB_FRAME_PENDING;
385  f->n_vectors = 0;
386  }
387 
388  /* Allocate new frame if current one is already full. */
389  n_used = f->n_vectors;
390  if (n_used >= VLIB_FRAME_SIZE || (allocate_new_next_frame && n_used > 0))
391  {
392  /* Old frame may need to be freed after dispatch, since we'll have
393  two redundant frames from node -> next node. */
395  {
396  vlib_frame_t *f_old = vlib_get_frame (vm, nf->frame_index);
398  }
399 
400  /* Allocate new frame to replace full one. */
401  nf->frame_index = vlib_frame_alloc (vm, node, next_index);
402  f = vlib_get_frame (vm, nf->frame_index);
403  n_used = f->n_vectors;
404  }
405 
406  /* Should have free vectors in frame now. */
407  ASSERT (n_used < VLIB_FRAME_SIZE);
408 
409  if (CLIB_DEBUG > 0)
410  {
411  validate_frame_magic (vm, f,
412  vlib_get_node (vm, node->node_index), next_index);
413  }
414 
415  return f;
416 }
417 
418 static void
420  vlib_node_runtime_t * rt,
421  u32 next_index, u32 n_vectors_left)
422 {
423  vlib_node_main_t *nm = &vm->node_main;
424  vlib_next_frame_t *nf;
425  vlib_frame_t *f;
426  vlib_node_runtime_t *next_rt;
427  vlib_node_t *next_node;
428  u32 n_before, n_after;
429 
430  nf = vlib_node_runtime_get_next_frame (vm, rt, next_index);
431  f = vlib_get_frame (vm, nf->frame_index);
432 
433  ASSERT (n_vectors_left <= VLIB_FRAME_SIZE);
434  n_after = VLIB_FRAME_SIZE - n_vectors_left;
435  n_before = f->n_vectors;
436 
437  ASSERT (n_after >= n_before);
438 
440  nf->node_runtime_index);
441  next_node = vlib_get_node (vm, next_rt->node_index);
442  if (n_after > 0 && next_node->validate_frame)
443  {
444  u8 *msg = next_node->validate_frame (vm, rt, f);
445  if (msg)
446  {
447  clib_warning ("%v", msg);
448  ASSERT (0);
449  }
450  vec_free (msg);
451  }
452 }
453 
454 void
457  u32 next_index, u32 n_vectors_left)
458 {
459  vlib_node_main_t *nm = &vm->node_main;
460  vlib_next_frame_t *nf;
461  vlib_frame_t *f;
462  u32 n_vectors_in_frame;
463 
464  if (buffer_main.callbacks_registered == 0 && CLIB_DEBUG > 0)
465  vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left);
466 
467  nf = vlib_node_runtime_get_next_frame (vm, r, next_index);
468  f = vlib_get_frame (vm, nf->frame_index);
469 
470  /* Make sure that magic number is still there. Otherwise, caller
471  has overrun frame meta data. */
472  if (CLIB_DEBUG > 0)
473  {
474  vlib_node_t *node = vlib_get_node (vm, r->node_index);
475  validate_frame_magic (vm, f, node, next_index);
476  }
477 
478  /* Convert # of vectors left -> number of vectors there. */
479  ASSERT (n_vectors_left <= VLIB_FRAME_SIZE);
480  n_vectors_in_frame = VLIB_FRAME_SIZE - n_vectors_left;
481 
482  f->n_vectors = n_vectors_in_frame;
483 
484  /* If vectors were added to frame, add to pending vector. */
485  if (PREDICT_TRUE (n_vectors_in_frame > 0))
486  {
488  u32 v0, v1;
489 
490  r->cached_next_index = next_index;
491 
492  if (!(f->frame_flags & VLIB_FRAME_PENDING))
493  {
494  __attribute__ ((unused)) vlib_node_t *node;
495  vlib_node_t *next_node;
496  vlib_node_runtime_t *next_runtime;
497 
498  node = vlib_get_node (vm, r->node_index);
499  next_node = vlib_get_next_node (vm, r->node_index, next_index);
500  next_runtime = vlib_node_get_runtime (vm, next_node->index);
501 
502  vec_add2 (nm->pending_frames, p, 1);
503 
504  p->frame_index = nf->frame_index;
506  p->next_frame_index = nf - nm->next_frames;
507  nf->flags |= VLIB_FRAME_PENDING;
509 
510  /*
511  * If we're going to dispatch this frame on another thread,
512  * force allocation of a new frame. Otherwise, we create
513  * a dangling frame reference. Each thread has its own copy of
514  * the next_frames vector.
515  */
516  if (0 && r->thread_index != next_runtime->thread_index)
517  {
518  nf->frame_index = ~0;
520  }
521  }
522 
523  /* Copy trace flag from next_frame and from runtime. */
524  nf->flags |=
525  (nf->flags & VLIB_NODE_FLAG_TRACE) | (r->
527 
529  v1 = v0 + n_vectors_in_frame;
531  if (PREDICT_FALSE (v1 < v0))
532  {
533  vlib_node_t *node = vlib_get_node (vm, r->node_index);
534  vec_elt (node->n_vectors_by_next_node, next_index) += v0;
535  }
536  }
537 }
538 
539 /* Sync up runtime (32 bit counters) and main node stats (64 bit counters). */
540 never_inline void
543  uword n_calls, uword n_vectors, uword n_clocks)
544 {
545  vlib_node_t *n = vlib_get_node (vm, r->node_index);
546 
547  n->stats_total.calls += n_calls + r->calls_since_last_overflow;
548  n->stats_total.vectors += n_vectors + r->vectors_since_last_overflow;
549  n->stats_total.clocks += n_clocks + r->clocks_since_last_overflow;
552 
556 }
557 
558 always_inline void __attribute__ ((unused))
560  vlib_process_t * p,
561  uword n_calls, uword n_vectors, uword n_clocks)
562 {
563  vlib_node_runtime_t *rt = &p->node_runtime;
565  vlib_node_runtime_sync_stats (vm, rt, n_calls, n_vectors, n_clocks);
566  n->stats_total.suspends += p->n_suspends;
567  p->n_suspends = 0;
568 }
569 
570 void
572 {
574 
575  if (n->type == VLIB_NODE_TYPE_PROCESS)
576  {
577  /* Nothing to do for PROCESS nodes except in main thread */
578  if (vm != &vlib_global_main)
579  return;
580 
583  p->n_suspends = 0;
584  rt = &p->node_runtime;
585  }
586  else
587  rt =
589  n->runtime_index);
590 
591  vlib_node_runtime_sync_stats (vm, rt, 0, 0, 0);
592 
593  /* Sync up runtime next frame vector counters with main node structure. */
594  {
595  vlib_next_frame_t *nf;
596  uword i;
597  for (i = 0; i < rt->n_next_nodes; i++)
598  {
599  nf = vlib_node_runtime_get_next_frame (vm, rt, i);
603  }
604  }
605 }
606 
609  vlib_node_runtime_t * node,
610  uword n_calls,
611  uword n_vectors, uword n_clocks)
612 {
613  u32 ca0, ca1, v0, v1, cl0, cl1, r;
614 
615  cl0 = cl1 = node->clocks_since_last_overflow;
616  ca0 = ca1 = node->calls_since_last_overflow;
617  v0 = v1 = node->vectors_since_last_overflow;
618 
619  ca1 = ca0 + n_calls;
620  v1 = v0 + n_vectors;
621  cl1 = cl0 + n_clocks;
622 
623  node->calls_since_last_overflow = ca1;
624  node->clocks_since_last_overflow = cl1;
625  node->vectors_since_last_overflow = v1;
626  node->max_clock_n = node->max_clock > n_clocks ?
627  node->max_clock_n : n_vectors;
628  node->max_clock = node->max_clock > n_clocks ? node->max_clock : n_clocks;
629 
630  r = vlib_node_runtime_update_main_loop_vector_stats (vm, node, n_vectors);
631 
632  if (PREDICT_FALSE (ca1 < ca0 || v1 < v0 || cl1 < cl0))
633  {
634  node->calls_since_last_overflow = ca0;
635  node->clocks_since_last_overflow = cl0;
636  node->vectors_since_last_overflow = v0;
637  vlib_node_runtime_sync_stats (vm, node, n_calls, n_vectors, n_clocks);
638  }
639 
640  return r;
641 }
642 
643 always_inline void
645  vlib_process_t * p,
646  uword n_calls, uword n_vectors, uword n_clocks)
647 {
649  n_calls, n_vectors, n_clocks);
650 }
651 
652 static clib_error_t *
654  unformat_input_t * input, vlib_cli_command_t * cmd)
655 {
657  return 0;
658 }
659 
660 /* *INDENT-OFF* */
661 VLIB_CLI_COMMAND (elog_clear_cli, static) = {
662  .path = "event-logger clear",
663  .short_help = "Clear the event log",
664  .function = vlib_cli_elog_clear,
665 };
666 /* *INDENT-ON* */
667 
668 #ifdef CLIB_UNIX
669 static clib_error_t *
671  unformat_input_t * input, vlib_cli_command_t * cmd)
672 {
673  elog_main_t *em = &vm->elog_main;
674  char *file, *chroot_file;
675  clib_error_t *error = 0;
676 
677  if (!unformat (input, "%s", &file))
678  {
679  vlib_cli_output (vm, "expected file name, got `%U'",
680  format_unformat_error, input);
681  return 0;
682  }
683 
684  /* It's fairly hard to get "../oopsie" through unformat; just in case */
685  if (strstr (file, "..") || index (file, '/'))
686  {
687  vlib_cli_output (vm, "illegal characters in filename '%s'", file);
688  return 0;
689  }
690 
691  chroot_file = (char *) format (0, "/tmp/%s%c", file, 0);
692 
693  vec_free (file);
694 
695  vlib_cli_output (vm, "Saving %wd of %wd events to %s",
697  elog_buffer_capacity (em), chroot_file);
698 
700  error = elog_write_file (em, chroot_file, 1 /* flush ring */ );
702  vec_free (chroot_file);
703  return error;
704 }
705 
706 void
708 {
710  elog_main_t *em = &vm->elog_main;
711  u8 *filename;
712  clib_error_t *error;
713 
714  if (!vm->elog_post_mortem_dump)
715  return;
716 
717  filename = format (0, "/tmp/elog_post_mortem.%d%c", getpid (), 0);
718  error = elog_write_file (em, (char *) filename, 1 /* flush ring */ );
719  if (error)
720  clib_error_report (error);
721  vec_free (filename);
722 }
723 
724 /* *INDENT-OFF* */
725 VLIB_CLI_COMMAND (elog_save_cli, static) = {
726  .path = "event-logger save",
727  .short_help = "event-logger save <filename> (saves log in /tmp/<filename>)",
728  .function = elog_save_buffer,
729 };
730 /* *INDENT-ON* */
731 
732 static clib_error_t *
734  unformat_input_t * input, vlib_cli_command_t * cmd)
735 {
736  elog_main_t *em = &vm->elog_main;
737 
739 
740  vlib_cli_output (vm, "Stopped the event logger...");
741  return 0;
742 }
743 
744 /* *INDENT-OFF* */
745 VLIB_CLI_COMMAND (elog_stop_cli, static) = {
746  .path = "event-logger stop",
747  .short_help = "Stop the event-logger",
748  .function = elog_stop,
749 };
750 /* *INDENT-ON* */
751 
752 static clib_error_t *
754  unformat_input_t * input, vlib_cli_command_t * cmd)
755 {
756  elog_main_t *em = &vm->elog_main;
757 
759 
760  vlib_cli_output (vm, "Restarted the event logger...");
761  return 0;
762 }
763 
764 /* *INDENT-OFF* */
765 VLIB_CLI_COMMAND (elog_restart_cli, static) = {
766  .path = "event-logger restart",
767  .short_help = "Restart the event-logger",
768  .function = elog_restart,
769 };
770 /* *INDENT-ON* */
771 
772 static clib_error_t *
774  unformat_input_t * input, vlib_cli_command_t * cmd)
775 {
776  elog_main_t *em = &vm->elog_main;
777  u32 tmp;
778 
779  /* Stop the parade */
781 
782  if (unformat (input, "%d", &tmp))
783  {
784  elog_alloc (em, tmp);
786  }
787  else
788  return clib_error_return (0, "Must specify how many events in the ring");
789 
790  vlib_cli_output (vm, "Resized ring and restarted the event logger...");
791  return 0;
792 }
793 
794 /* *INDENT-OFF* */
795 VLIB_CLI_COMMAND (elog_resize_cli, static) = {
796  .path = "event-logger resize",
797  .short_help = "event-logger resize <nnn>",
798  .function = elog_resize,
799 };
800 /* *INDENT-ON* */
801 
802 #endif /* CLIB_UNIX */
803 
804 static void
806 {
807  elog_main_t *em = &vm->elog_main;
808  elog_event_t *e, *es;
809  f64 dt;
810 
811  /* Show events in VLIB time since log clock starts after VLIB clock. */
812  dt = (em->init_time.cpu - vm->clib_time.init_cpu_time)
814 
815  es = elog_peek_events (em);
816  vlib_cli_output (vm, "%d of %d events in buffer, logger %s", vec_len (es),
817  em->event_ring_size,
819  "running" : "stopped");
820  vec_foreach (e, es)
821  {
822  vlib_cli_output (vm, "%18.9f: %U",
823  e->time + dt, format_elog_event, em, e);
824  n_events_to_show--;
825  if (n_events_to_show == 0)
826  break;
827  }
828  vec_free (es);
829 
830 }
831 
832 static clib_error_t *
834  unformat_input_t * input, vlib_cli_command_t * cmd)
835 {
836  u32 n_events_to_show;
837  clib_error_t *error = 0;
838 
839  n_events_to_show = 250;
841  {
842  if (unformat (input, "%d", &n_events_to_show))
843  ;
844  else if (unformat (input, "all"))
845  n_events_to_show = ~0;
846  else
847  return unformat_parse_error (input);
848  }
849  elog_show_buffer_internal (vm, n_events_to_show);
850  return error;
851 }
852 
853 /* *INDENT-OFF* */
854 VLIB_CLI_COMMAND (elog_show_cli, static) = {
855  .path = "show event-logger",
856  .short_help = "Show event logger info",
857  .function = elog_show_buffer,
858 };
859 /* *INDENT-ON* */
860 
861 void
863 {
865 }
866 
867 static inline void
869  u32 node_index,
870  u64 time, u32 n_vectors, u32 is_return)
871 {
873  elog_main_t *em = &evm->elog_main;
874 
875  if (VLIB_ELOG_MAIN_LOOP && n_vectors)
876  elog_track (em,
877  /* event type */
878  vec_elt_at_index (is_return
881  node_index),
882  /* track */
884  elog_track : &em->default_track),
885  /* data to log */ n_vectors);
886 }
887 
888 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
889 void (*vlib_buffer_trace_trajectory_cb) (vlib_buffer_t * b, u32 node_index);
890 void (*vlib_buffer_trace_trajectory_init_cb) (vlib_buffer_t * b);
891 
892 void
893 vlib_buffer_trace_trajectory_init (vlib_buffer_t * b)
894 {
895  if (PREDICT_TRUE (vlib_buffer_trace_trajectory_init_cb != 0))
896  {
897  (*vlib_buffer_trace_trajectory_init_cb) (b);
898  }
899 }
900 
901 #endif
902 
903 static inline void
905 {
906 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
907  if (PREDICT_TRUE (vlib_buffer_trace_trajectory_cb != 0))
908  {
909  (*vlib_buffer_trace_trajectory_cb) (b, node_index);
910  }
911 #endif
912 }
913 
916  vlib_node_runtime_t * node,
917  vlib_node_type_t type,
918  vlib_node_state_t dispatch_state,
919  vlib_frame_t * frame, u64 last_time_stamp)
920 {
921  uword n, v;
922  u64 t;
923  vlib_node_main_t *nm = &vm->node_main;
924  vlib_next_frame_t *nf;
925 
926  if (CLIB_DEBUG > 0)
927  {
928  vlib_node_t *n = vlib_get_node (vm, node->node_index);
929  ASSERT (n->type == type);
930  }
931 
932  /* Only non-internal nodes may be disabled. */
933  if (type != VLIB_NODE_TYPE_INTERNAL && node->state != dispatch_state)
934  {
936  return last_time_stamp;
937  }
938 
939  if ((type == VLIB_NODE_TYPE_PRE_INPUT || type == VLIB_NODE_TYPE_INPUT)
940  && dispatch_state != VLIB_NODE_STATE_INTERRUPT)
941  {
943  /* Only call node when count reaches zero. */
944  if (c)
945  {
946  node->input_main_loops_per_call = c - 1;
947  return last_time_stamp;
948  }
949  }
950 
951  /* Speculatively prefetch next frames. */
952  if (node->n_next_nodes > 0)
953  {
955  CLIB_PREFETCH (nf, 4 * sizeof (nf[0]), WRITE);
956  }
957 
958  vm->cpu_time_last_node_dispatch = last_time_stamp;
959 
960  if (1 /* || vm->thread_index == node->thread_index */ )
961  {
962  vlib_main_t *stat_vm;
963 
964  stat_vm = /* vlib_mains ? vlib_mains[0] : */ vm;
965 
967  last_time_stamp,
968  frame ? frame->n_vectors : 0,
969  /* is_after */ 0);
970 
971  /*
972  * Turn this on if you run into
973  * "bad monkey" contexts, and you want to know exactly
974  * which nodes they've visited... See ixge.c...
975  */
976  if (VLIB_BUFFER_TRACE_TRAJECTORY && frame)
977  {
978  int i;
979  u32 *from;
980  from = vlib_frame_vector_args (frame);
981  for (i = 0; i < frame->n_vectors; i++)
982  {
983  vlib_buffer_t *b = vlib_get_buffer (vm, from[i]);
984  add_trajectory_trace (b, node->node_index);
985  }
986  n = node->function (vm, node, frame);
987  }
988  else
989  n = node->function (vm, node, frame);
990 
991  t = clib_cpu_time_now ();
992 
993  vlib_elog_main_loop_event (vm, node->node_index, t, n, /* is_after */
994  1);
995 
997  vm->main_loop_nodes_processed += n > 0;
998 
999  v = vlib_node_runtime_update_stats (stat_vm, node,
1000  /* n_calls */ 1,
1001  /* n_vectors */ n,
1002  /* n_clocks */ t - last_time_stamp);
1003 
1004  /* When in interrupt mode and vector rate crosses threshold switch to
1005  polling mode. */
1006  if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT)
1007  || (dispatch_state == VLIB_NODE_STATE_POLLING
1008  && (node->flags
1010  {
1011 #ifdef DISPATCH_NODE_ELOG_REQUIRED
1012  ELOG_TYPE_DECLARE (e) =
1013  {
1014  .function = (char *) __FUNCTION__,.format =
1015  "%s vector length %d, switching to %s",.format_args =
1016  "T4i4t4",.n_enum_strings = 2,.enum_strings =
1017  {
1018  "interrupt", "polling",},};
1019  struct
1020  {
1021  u32 node_name, vector_length, is_polling;
1022  } *ed;
1024 #endif
1025 
1026  if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT
1027  && v >= nm->polling_threshold_vector_length) &&
1028  !(node->flags &
1030  {
1031  vlib_node_t *n = vlib_get_node (vm, node->node_index);
1032  n->state = VLIB_NODE_STATE_POLLING;
1033  node->state = VLIB_NODE_STATE_POLLING;
1034  node->flags &=
1036  node->flags |=
1038  nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] -= 1;
1039  nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] += 1;
1040 
1041 #ifdef DISPATCH_NODE_ELOG_REQUIRED
1042  ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
1043  w->elog_track);
1044  ed->node_name = n->name_elog_string;
1045  ed->vector_length = v;
1046  ed->is_polling = 1;
1047 #endif
1048  }
1049  else if (dispatch_state == VLIB_NODE_STATE_POLLING
1050  && v <= nm->interrupt_threshold_vector_length)
1051  {
1052  vlib_node_t *n = vlib_get_node (vm, node->node_index);
1053  if (node->flags &
1055  {
1056  /* Switch to interrupt mode after dispatch in polling one more time.
1057  This allows driver to re-enable interrupts. */
1058  n->state = VLIB_NODE_STATE_INTERRUPT;
1059  node->state = VLIB_NODE_STATE_INTERRUPT;
1060  node->flags &=
1062  nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] -=
1063  1;
1064  nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] +=
1065  1;
1066 
1067  }
1068  else
1069  {
1070  node->flags |=
1072 #ifdef DISPATCH_NODE_ELOG_REQUIRED
1073  ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
1074  w->elog_track);
1075  ed->node_name = n->name_elog_string;
1076  ed->vector_length = v;
1077  ed->is_polling = 0;
1078 #endif
1079  }
1080  }
1081  }
1082  }
1083 
1084  return t;
1085 }
1086 
1087 static u64
1088 dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index,
1089  u64 last_time_stamp)
1090 {
1091  vlib_node_main_t *nm = &vm->node_main;
1092  vlib_frame_t *f;
1093  vlib_next_frame_t *nf, nf_dummy;
1095  u32 restore_frame_index;
1097 
1098  /* See comment below about dangling references to nm->pending_frames */
1099  p = nm->pending_frames + pending_frame_index;
1100 
1102  p->node_runtime_index);
1103 
1104  f = vlib_get_frame (vm, p->frame_index);
1106  {
1107  /* No next frame: so use dummy on stack. */
1108  nf = &nf_dummy;
1110  nf->frame_index = ~p->frame_index;
1111  }
1112  else
1114 
1116 
1117  /* Force allocation of new frame while current frame is being
1118  dispatched. */
1119  restore_frame_index = ~0;
1120  if (nf->frame_index == p->frame_index)
1121  {
1122  nf->frame_index = ~0;
1125  restore_frame_index = p->frame_index;
1126  }
1127 
1128  /* Frame must be pending. */
1130  ASSERT (f->n_vectors > 0);
1131 
1132  /* Copy trace flag from next frame to node.
1133  Trace flag indicates that at least one vector in the dispatched
1134  frame is traced. */
1135  n->flags &= ~VLIB_NODE_FLAG_TRACE;
1136  n->flags |= (nf->flags & VLIB_FRAME_TRACE) ? VLIB_NODE_FLAG_TRACE : 0;
1137  nf->flags &= ~VLIB_FRAME_TRACE;
1138 
1139  last_time_stamp = dispatch_node (vm, n,
1141  VLIB_NODE_STATE_POLLING,
1142  f, last_time_stamp);
1143 
1145 
1146  /* Frame is ready to be used again, so restore it. */
1147  if (restore_frame_index != ~0)
1148  {
1149  /*
1150  * We musn't restore a frame that is flagged to be freed. This
1151  * shouldn't happen since frames to be freed post dispatch are
1152  * those used when the to-node frame becomes full i.e. they form a
1153  * sort of queue of frames to a single node. If we get here then
1154  * the to-node frame and the pending frame *were* the same, and so
1155  * we removed the to-node frame. Therefore this frame is no
1156  * longer part of the queue for that node and hence it cannot be
1157  * it's overspill.
1158  */
1160 
1161  /*
1162  * NB: dispatching node n can result in the creation and scheduling
1163  * of new frames, and hence in the reallocation of nm->pending_frames.
1164  * Recompute p, or no supper. This was broken for more than 10 years.
1165  */
1166  p = nm->pending_frames + pending_frame_index;
1167 
1168  /*
1169  * p->next_frame_index can change during node dispatch if node
1170  * function decides to change graph hook up.
1171  */
1174 
1175  if (~0 == nf->frame_index)
1176  {
1177  /* no new frame has been assigned to this node, use the saved one */
1178  nf->frame_index = restore_frame_index;
1179  f->n_vectors = 0;
1180  }
1181  else
1182  {
1183  /* The node has gained a frame, implying packets from the current frame
1184  were re-queued to this same node. we don't need the saved one
1185  anymore */
1186  vlib_frame_free (vm, n, f);
1187  }
1188  }
1189  else
1190  {
1192  {
1194  vlib_frame_free (vm, n, f);
1195  }
1196  }
1197 
1198  return last_time_stamp;
1199 }
1200 
1203 {
1204  return p->stack[0] == VLIB_PROCESS_STACK_MAGIC;
1205 }
1206 
1207 typedef struct
1208 {
1213 
1214 /* Called in process stack. */
1215 static uword
1217 {
1219  vlib_main_t *vm;
1220  vlib_node_runtime_t *node;
1221  vlib_frame_t *f;
1222  vlib_process_t *p;
1223  uword n;
1224 
1226 
1227  vm = a->vm;
1228  p = a->process;
1229  f = a->frame;
1230  node = &p->node_runtime;
1231 
1232  n = node->function (vm, node, f);
1233 
1235 
1236  clib_longjmp (&p->return_longjmp, n);
1237 
1238  return n;
1239 }
1240 
1241 /* Called in main stack. */
1244 {
1246  uword r;
1247 
1248  a.vm = vm;
1249  a.process = p;
1250  a.frame = f;
1251 
1255  (void *) p->stack + (1 << p->log2_n_stack_bytes));
1256 
1257  return r;
1258 }
1259 
1262 {
1263  uword r;
1270  return r;
1271 }
1272 
1273 static u64
1275  vlib_process_t * p, vlib_frame_t * f, u64 last_time_stamp)
1276 {
1277  vlib_node_main_t *nm = &vm->node_main;
1278  vlib_node_runtime_t *node_runtime = &p->node_runtime;
1279  vlib_node_t *node = vlib_get_node (vm, node_runtime->node_index);
1280  u32 old_process_index;
1281  u64 t;
1282  uword n_vectors, is_suspend;
1283 
1284  if (node->state != VLIB_NODE_STATE_POLLING
1287  return last_time_stamp;
1288 
1290 
1291  t = last_time_stamp;
1292  vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
1293  f ? f->n_vectors : 0, /* is_after */ 0);
1294 
1295  /* Save away current process for suspend. */
1296  old_process_index = nm->current_process_index;
1298 
1299  n_vectors = vlib_process_startup (vm, p, f);
1300 
1301  nm->current_process_index = old_process_index;
1302 
1304  is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND;
1305  if (is_suspend)
1306  {
1308 
1309  n_vectors = 0;
1311  pf->node_runtime_index = node->runtime_index;
1312  pf->frame_index = f ? vlib_frame_index (vm, f) : ~0;
1313  pf->next_frame_index = ~0;
1314 
1315  p->n_suspends += 1;
1317 
1319  {
1320  TWT (tw_timer_wheel) * tw =
1321  (TWT (tw_timer_wheel) *) nm->timing_wheel;
1322  p->stop_timer_handle =
1323  TW (tw_timer_start) (tw,
1325  (node->runtime_index) /* [sic] pool idex */ ,
1326  0 /* timer_id */ ,
1328  }
1329  }
1330  else
1332 
1333  t = clib_cpu_time_now ();
1334 
1335  vlib_elog_main_loop_event (vm, node_runtime->node_index, t, is_suspend,
1336  /* is_after */ 1);
1337 
1339  /* n_calls */ !is_suspend,
1340  /* n_vectors */ n_vectors,
1341  /* n_clocks */ t - last_time_stamp);
1342 
1343  return t;
1344 }
1345 
1346 void
1348 {
1349  vlib_node_main_t *nm = &vm->node_main;
1350  vlib_process_t *p = vec_elt (nm->processes, process_index);
1351  dispatch_process (vm, p, /* frame */ 0, /* cpu_time_now */ 0);
1352 }
1353 
1354 static u64
1356  uword process_index, u64 last_time_stamp)
1357 {
1358  vlib_node_main_t *nm = &vm->node_main;
1359  vlib_node_runtime_t *node_runtime;
1360  vlib_node_t *node;
1361  vlib_frame_t *f;
1362  vlib_process_t *p;
1364  u64 t, n_vectors, is_suspend;
1365 
1366  t = last_time_stamp;
1367 
1368  p = vec_elt (nm->processes, process_index);
1370  return last_time_stamp;
1371 
1374 
1375  pf =
1378 
1379  node_runtime = &p->node_runtime;
1380  node = vlib_get_node (vm, node_runtime->node_index);
1381  f = pf->frame_index != ~0 ? vlib_get_frame (vm, pf->frame_index) : 0;
1382 
1383  vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
1384  f ? f->n_vectors : 0, /* is_after */ 0);
1385 
1386  /* Save away current process for suspend. */
1388 
1389  n_vectors = vlib_process_resume (p);
1390  t = clib_cpu_time_now ();
1391 
1392  nm->current_process_index = ~0;
1393 
1394  is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND;
1395  if (is_suspend)
1396  {
1397  /* Suspend it again. */
1398  n_vectors = 0;
1399  p->n_suspends += 1;
1401  {
1402  p->stop_timer_handle =
1403  TW (tw_timer_start) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
1405  (node->runtime_index) /* [sic] pool idex */ ,
1406  0 /* timer_id */ ,
1408  }
1409  }
1410  else
1411  {
1415  }
1416 
1417  t = clib_cpu_time_now ();
1418  vlib_elog_main_loop_event (vm, node_runtime->node_index, t, !is_suspend,
1419  /* is_after */ 1);
1420 
1422  /* n_calls */ !is_suspend,
1423  /* n_vectors */ n_vectors,
1424  /* n_clocks */ t - last_time_stamp);
1425 
1426  return t;
1427 }
1428 
1429 void vl_api_send_pending_rpc_requests (vlib_main_t *) __attribute__ ((weak));
1430 void
1432 {
1433 }
1434 
1435 
1438 {
1439  vlib_node_main_t *nm = &vm->node_main;
1441  uword i;
1442  u64 cpu_time_now;
1444  u32 *last_node_runtime_indices = 0;
1445 
1446  /* Initialize pending node vector. */
1447  if (is_main)
1448  {
1449  vec_resize (nm->pending_frames, 32);
1450  _vec_len (nm->pending_frames) = 0;
1451  }
1452 
1453  /* Mark time of main loop start. */
1454  if (is_main)
1455  {
1456  cpu_time_now = vm->clib_time.last_cpu_time;
1457  vm->cpu_time_main_loop_start = cpu_time_now;
1458  }
1459  else
1460  cpu_time_now = clib_cpu_time_now ();
1461 
1462  /* Pre-allocate interupt runtime indices and lock. */
1464  vec_alloc (last_node_runtime_indices, 32);
1465  if (!is_main)
1467 
1468  /* Pre-allocate expired nodes. */
1473 
1474  /* Start all processes. */
1475  if (is_main)
1476  {
1477  uword i;
1478  nm->current_process_index = ~0;
1479  for (i = 0; i < vec_len (nm->processes); i++)
1480  cpu_time_now = dispatch_process (vm, nm->processes[i], /* frame */ 0,
1481  cpu_time_now);
1482  }
1483 
1484  while (1)
1485  {
1487 
1488  if (PREDICT_FALSE (_vec_len (vm->pending_rpc_requests) > 0))
1490 
1491  if (!is_main)
1492  {
1494  vec_foreach (fqm, tm->frame_queue_mains)
1495  vlib_frame_queue_dequeue (vm, fqm);
1496  }
1497 
1498  /* Process pre-input nodes. */
1500  cpu_time_now = dispatch_node (vm, n,
1502  VLIB_NODE_STATE_POLLING,
1503  /* frame */ 0,
1504  cpu_time_now);
1505 
1506  /* Next process input nodes. */
1508  cpu_time_now = dispatch_node (vm, n,
1510  VLIB_NODE_STATE_POLLING,
1511  /* frame */ 0,
1512  cpu_time_now);
1513 
1514  if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0))
1515  vm->queue_signal_callback (vm);
1516 
1517  /* Next handle interrupts. */
1518  {
1519  /* unlocked read, for performance */
1520  uword l = _vec_len (nm->pending_interrupt_node_runtime_indices);
1521  uword i;
1522  if (PREDICT_FALSE (l > 0))
1523  {
1524  u32 *tmp;
1525  if (!is_main)
1526  {
1528  /* Re-read w/ lock held, in case another thread added an item */
1529  l = _vec_len (nm->pending_interrupt_node_runtime_indices);
1530  }
1531 
1534  last_node_runtime_indices;
1535  last_node_runtime_indices = tmp;
1536  _vec_len (last_node_runtime_indices) = 0;
1537  if (!is_main)
1539  for (i = 0; i < l; i++)
1540  {
1542  last_node_runtime_indices[i]);
1543  cpu_time_now =
1545  VLIB_NODE_STATE_INTERRUPT,
1546  /* frame */ 0,
1547  cpu_time_now);
1548  }
1549  }
1550  }
1551  /* Input nodes may have added work to the pending vector.
1552  Process pending vector until there is nothing left.
1553  All pending vectors will be processed from input -> output. */
1554  for (i = 0; i < _vec_len (nm->pending_frames); i++)
1555  cpu_time_now = dispatch_pending_node (vm, i, cpu_time_now);
1556  /* Reset pending vector for next iteration. */
1557  _vec_len (nm->pending_frames) = 0;
1558 
1559  if (is_main)
1560  {
1561  /* Check if process nodes have expired from timing wheel. */
1563 
1566  ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm),
1568 
1570 
1571  if (PREDICT_FALSE
1572  (_vec_len (nm->data_from_advancing_timing_wheel) > 0))
1573  {
1574  uword i;
1575 
1576  for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel);
1577  i++)
1578  {
1581 
1583  {
1586  di);
1587  vlib_node_t *n =
1589  vlib_process_t *p =
1590  vec_elt (nm->processes, n->runtime_index);
1591  void *data;
1592  data =
1594  te->event_type_index,
1595  te->n_data_elts,
1596  te->n_data_elt_bytes);
1597  if (te->n_data_bytes < sizeof (te->inline_event_data))
1598  clib_memcpy (data, te->inline_event_data,
1599  te->n_data_bytes);
1600  else
1601  {
1602  clib_memcpy (data, te->event_data_as_vector,
1603  te->n_data_bytes);
1605  }
1607  }
1608  else
1609  {
1610  cpu_time_now = clib_cpu_time_now ();
1611  cpu_time_now =
1612  dispatch_suspended_process (vm, di, cpu_time_now);
1613  }
1614  }
1615  _vec_len (nm->data_from_advancing_timing_wheel) = 0;
1616  }
1617  }
1619 
1620  /* Record time stamp in case there are no enabled nodes and above
1621  calls do not update time stamp. */
1622  cpu_time_now = clib_cpu_time_now ();
1623  }
1624 }
1625 
1626 static void
1628 {
1629  vlib_main_or_worker_loop (vm, /* is_main */ 1);
1630 }
1631 
1632 void
1634 {
1635  vlib_main_or_worker_loop (vm, /* is_main */ 0);
1636 }
1637 
1639 
1640 static clib_error_t *
1642 {
1643  int turn_on_mem_trace = 0;
1644 
1645  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1646  {
1647  if (unformat (input, "memory-trace"))
1648  turn_on_mem_trace = 1;
1649 
1650  else if (unformat (input, "elog-events %d",
1651  &vm->elog_main.event_ring_size))
1652  ;
1653  else if (unformat (input, "elog-post-mortem-dump"))
1654  vm->elog_post_mortem_dump = 1;
1655  else
1656  return unformat_parse_error (input);
1657  }
1658 
1659  unformat_free (input);
1660 
1661  /* Enable memory trace as early as possible. */
1662  if (turn_on_mem_trace)
1663  clib_mem_trace (1);
1664 
1665  return 0;
1666 }
1667 
1669 
1670 static void
1672 {
1673 }
1674 
1675 #define foreach_weak_reference_stub \
1676 _(vlib_map_stat_segment_init) \
1677 _(vpe_api_init) \
1678 _(vlibmemory_init) \
1679 _(map_api_segment_init)
1680 
1681 #define _(name) \
1682 clib_error_t *name (vlib_main_t *vm) __attribute__((weak)); \
1683 clib_error_t *name (vlib_main_t *vm) { return 0; }
1685 #undef _
1686 
1687 /* Main function. */
1688 int
1690 {
1691  clib_error_t *volatile error;
1692  vlib_node_main_t *nm = &vm->node_main;
1693 
1695 
1696  clib_time_init (&vm->clib_time);
1697 
1698  /* Turn on event log. */
1699  if (!vm->elog_main.event_ring_size)
1700  vm->elog_main.event_ring_size = 128 << 10;
1702  elog_enable_disable (&vm->elog_main, 1);
1703 
1704  /* Default name. */
1705  if (!vm->name)
1706  vm->name = "VLIB";
1707 
1708  if ((error = unix_physmem_init (vm)))
1709  {
1710  clib_error_report (error);
1711  goto done;
1712  }
1713 
1714  if ((error = vlib_buffer_main_init (vm)))
1715  {
1716  clib_error_report (error);
1717  goto done;
1718  }
1719 
1720  if ((error = vlib_thread_init (vm)))
1721  {
1722  clib_error_report (error);
1723  goto done;
1724  }
1725 
1726  if ((error = vlib_map_stat_segment_init (vm)))
1727  {
1728  clib_error_report (error);
1729  goto done;
1730  }
1731 
1732  /* Register static nodes so that init functions may use them. */
1734 
1735  /* Set seed for random number generator.
1736  Allow user to specify seed to make random sequence deterministic. */
1737  if (!unformat (input, "seed %wd", &vm->random_seed))
1738  vm->random_seed = clib_cpu_time_now ();
1740 
1741  /* Initialize node graph. */
1742  if ((error = vlib_node_main_init (vm)))
1743  {
1744  /* Arrange for graph hook up error to not be fatal when debugging. */
1745  if (CLIB_DEBUG > 0)
1746  clib_error_report (error);
1747  else
1748  goto done;
1749  }
1750 
1751  /* Direct call / weak reference, for vlib standalone use-cases */
1752  if ((error = vpe_api_init (vm)))
1753  {
1754  clib_error_report (error);
1755  goto done;
1756  }
1757 
1758  if ((error = vlibmemory_init (vm)))
1759  {
1760  clib_error_report (error);
1761  goto done;
1762  }
1763 
1764  if ((error = map_api_segment_init (vm)))
1765  {
1766  clib_error_report (error);
1767  goto done;
1768  }
1769 
1770  /* See unix/main.c; most likely already set up */
1771  if (vm->init_functions_called == 0)
1772  vm->init_functions_called = hash_create (0, /* value bytes */ 0);
1773  if ((error = vlib_call_all_init_functions (vm)))
1774  goto done;
1775 
1776  /* Create default buffer free list. */
1778  "default");
1779 
1780  nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)),
1782 
1784  _vec_len (nm->data_from_advancing_timing_wheel) = 0;
1785 
1786  /* Create the process timing wheel */
1787  TW (tw_timer_wheel_init) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
1788  0 /* no callback */ ,
1789  10e-6 /* timer period 10us */ ,
1790  ~0 /* max expirations per call */ );
1791 
1793  _vec_len (vm->pending_rpc_requests) = 0;
1794 
1796  {
1798  vm->main_loop_exit_set = 1;
1799  break;
1800 
1802  goto done;
1803 
1804  default:
1805  error = vm->main_loop_error;
1806  goto done;
1807  }
1808 
1809  if ((error = vlib_call_all_config_functions (vm, input, 0 /* is_early */ )))
1810  goto done;
1811 
1812  /* Call all main loop enter functions. */
1813  {
1814  clib_error_t *sub_error;
1815  sub_error = vlib_call_all_main_loop_enter_functions (vm);
1816  if (sub_error)
1817  clib_error_report (sub_error);
1818  }
1819 
1820  vlib_main_loop (vm);
1821 
1822 done:
1823  /* Call all exit functions. */
1824  {
1825  clib_error_t *sub_error;
1826  sub_error = vlib_call_all_main_loop_exit_functions (vm);
1827  if (sub_error)
1828  clib_error_report (sub_error);
1829  }
1830 
1831  if (error)
1832  clib_error_report (error);
1833 
1834  return 0;
1835 }
1836 
1837 /*
1838  * fd.io coding-style-patch-verification: ON
1839  *
1840  * Local Variables:
1841  * eval: (c-set-style "gnu")
1842  * End:
1843  */
u32 * next_nodes
Definition: node.h:339
static void elog_enable_disable(elog_main_t *em, int is_enabled)
Enable or disable event logging.
Definition: elog.h:217
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:437
u32 TW() tw_timer_start(TWT(tw_timer_wheel)*tw, u32 user_id, u32 timer_id, u64 interval)
Start a Tw Timer.
clib_error_t * vlib_call_all_main_loop_exit_functions(vlib_main_t *vm)
Definition: init.c:87
uword * pending_rpc_requests
Definition: main.h:217
vlib_main_t vlib_global_main
Definition: main.c:1638
f64 time
Absolute time as floating point number in seconds.
Definition: elog.h:66
void TW() tw_timer_wheel_init(TWT(tw_timer_wheel)*tw, void *expired_timer_callback, f64 timer_interval_in_seconds, u32 max_expirations)
Initialize a tw timer wheel template instance.
u16 vector_size
Definition: node.h:325
u32 max_clock
Maximum clock cycle for an invocation.
Definition: node.h:480
u32 next_frame_index
Start of next frames for this node.
Definition: node.h:491
never_inline void vlib_node_runtime_sync_stats(vlib_main_t *vm, vlib_node_runtime_t *r, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:541
static clib_error_t * show_frame_stats(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:243
#define hash_set(h, key, value)
Definition: hash.h:255
vlib_node_type_t
Definition: node.h:58
clib_error_t * vlib_call_all_main_loop_enter_functions(vlib_main_t *vm)
Definition: init.c:80
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
Definition: lock.h:89
static void vlib_next_frame_change_ownership(vlib_main_t *vm, vlib_node_runtime_t *node_runtime, u32 next_index)
Definition: main.c:273
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
Definition: lock.h:74
static vlib_next_frame_t * vlib_node_get_next_frame(vlib_main_t *vm, u32 node_index, u32 next_index)
Get pointer to frame by (node_index, next_index).
Definition: node_funcs.h:325
u32 interrupt_threshold_vector_length
Definition: node.h:717
vlib_process_t ** processes
Definition: node.h:738
static uword vlib_process_bootstrap(uword _a)
Definition: main.c:1216
vlib_node_runtime_t node_runtime
Definition: node.h:552
a
Definition: bitmap.h:538
u32 n_suspends
Definition: node.h:580
#define VLIB_PENDING_FRAME_NO_NEXT_FRAME
Definition: node.h:463
vlib_frame_t * vlib_get_next_frame_internal(vlib_main_t *vm, vlib_node_runtime_t *node, u32 next_index, u32 allocate_new_next_frame)
Definition: main.c:356
static void vlib_elog_main_loop_event(vlib_main_t *vm, u32 node_index, u64 time, u32 n_vectors, u32 is_return)
Definition: main.c:868
static clib_error_t * elog_save_buffer(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:670
int elog_post_mortem_dump
Definition: main.h:199
#define PREDICT_TRUE(x)
Definition: clib.h:108
uword random_seed
Definition: main.h:170
unsigned long u64
Definition: types.h:89
static void vlib_increment_main_loop_counter(vlib_main_t *vm)
Definition: main.h:329
elog_time_stamp_t init_time
Timestamps.
Definition: elog.h:169
void clib_random_buffer_init(clib_random_buffer_t *b, uword seed)
Definition: random_buffer.c:62
u32 index
Definition: node.h:288
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:227
static void dummy_queue_signal_callback(vlib_main_t *vm)
Definition: main.c:1671
u32 current_process_index
Definition: node.h:741
u32 *TW() tw_timer_expire_timers_vec(TWT(tw_timer_wheel)*tw, f64 now, u32 *vec)
u32 thread_index
Definition: main.h:179
static_always_inline uword vlib_process_startup(vlib_main_t *vm, vlib_process_t *p, vlib_frame_t *f)
Definition: main.c:1243
u32 main_loop_exit_set
Definition: main.h:88
static u64 dispatch_pending_node(vlib_main_t *vm, uword pending_frame_index, u64 last_time_stamp)
Definition: main.c:1088
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
static u64 clib_cpu_time_now(void)
Definition: time.h:73
elog_track_t elog_track
Definition: threads.h:100
u32 clocks_since_last_overflow
Number of clock cycles.
Definition: node.h:478
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:562
int i
static void vlib_process_update_stats(vlib_main_t *vm, vlib_process_t *p, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:644
static void elog_show_buffer_internal(vlib_main_t *vm, u32 n_events_to_show)
Definition: main.c:805
#define VLIB_MAIN_LOOP_EXIT_CLI
Definition: main.h:95
void clib_longjmp(clib_longjmp_t *save, uword return_value)
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
static vlib_frame_size_t * get_frame_size_info(vlib_node_main_t *nm, u32 n_scalar_bytes, u32 n_vector_bytes)
Definition: main.c:95
static clib_error_t * elog_show_buffer(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:833
u8 * format_elog_event(u8 *s, va_list *va)
Definition: elog.c:296
clib_time_t clib_time
Definition: main.h:63
clib_spinlock_t pending_interrupt_lock
Definition: node.h:711
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:228
uword clib_setjmp(clib_longjmp_t *save, uword return_value_not_taken)
#define vec_alloc(V, N)
Allocate space for N more elements (no header, unspecified alignment)
Definition: vec.h:278
unsigned char u8
Definition: types.h:56
static uword vlib_process_stack_is_valid(vlib_process_t *p)
Definition: main.c:1202
static vlib_frame_t * vlib_get_frame_no_check(vlib_main_t *vm, uword frame_index)
Definition: node_funcs.h:216
double f64
Definition: types.h:142
u16 scalar_size
Definition: node.h:325
u8 state
Definition: node.h:316
u16 thread_index
thread this node runs on
Definition: node.h:519
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:204
static void vlib_worker_thread_barrier_check(void)
Definition: threads.h:389
u32 * pending_interrupt_node_runtime_indices
Definition: node.h:710
u32 input_main_loops_per_call
For input nodes: decremented on each main loop interation until it reaches zero and function is calle...
Definition: node.h:496
static void vlib_main_loop(vlib_main_t *vm)
Definition: main.c:1627
#define foreach_weak_reference_stub
Definition: main.c:1675
memset(h->entries, 0, sizeof(h->entries[0])*entries)
#define static_always_inline
Definition: clib.h:95
vlib_node_function_t * function
Node function to call.
Definition: node.h:470
void elog_init(elog_main_t *em, u32 n_events)
Definition: elog.c:470
#define VLIB_FRAME_MAGIC
#define always_inline
Definition: clib.h:94
u32 main_loop_vectors_processed
Definition: main.h:75
#define VLIB_INVALID_NODE_INDEX
Definition: node.h:379
static u32 vlib_frame_alloc(vlib_main_t *vm, vlib_node_runtime_t *from_node_runtime, u32 to_next_index)
Definition: main.c:169
u16 log2_n_stack_bytes
Definition: node.h:575
vlib_node_t ** nodes
Definition: node.h:697
u32 vectors_since_last_overflow
Number of vector elements processed by this node.
Definition: node.h:488
vlib_frame_t * frame
Definition: main.c:1211
void di(unformat_input_t *i)
Definition: unformat.c:163
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:182
#define clib_error_return(e, args...)
Definition: error.h:99
#define VLIB_FRAME_ALIGN
Definition: node.h:383
static void vlib_process_sync_stats(vlib_main_t *vm, vlib_process_t *p, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:559
static uword vlib_timing_wheel_data_is_timed_event(u32 d)
Definition: node.h:671
#define VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE
Definition: node.h:312
#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES
Definition: buffer.h:441
#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
Definition: node.h:566
#define vec_resize(V, N)
Resize a vector (no header, unspecified alignment) Add N elements to end of given vector V...
Definition: vec.h:240
unsigned int u32
Definition: types.h:88
u64 cpu_time_main_loop_start
Definition: main.h:69
vlib_node_runtime_t * nodes_by_type[VLIB_N_NODE_TYPE]
Definition: node.h:707
static u32 vlib_frame_bytes(u32 n_scalar_bytes, u32 n_vector_bytes)
Definition: main.c:58
u32 * free_frame_indices
Definition: node.h:540
#define VLIB_MAIN_LOOP_EXIT_NONE
Definition: main.h:92
clib_error_t * vlib_node_main_init(vlib_main_t *vm)
Definition: node.c:630
#define VLIB_FRAME_SIZE
Definition: node.h:382
static vlib_next_frame_t * vlib_node_runtime_get_next_frame(vlib_main_t *vm, vlib_node_runtime_t *n, u32 next_index)
Definition: node_funcs.h:290
#define VLIB_PROCESS_IS_RUNNING
Definition: node.h:572
#define VLIB_PROCESS_RETURN_LONGJMP_RETURN
Definition: node.h:557
u64 max_clock_n
Definition: node.h:246
int callbacks_registered
Definition: buffer.h:456
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
u32 calls_since_last_overflow
Number of calls.
Definition: node.h:486
char * name
Definition: main.h:101
vlib_worker_thread_t * vlib_worker_threads
Definition: threads.c:36
#define hash_get(h, key)
Definition: hash.h:249
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:464
static clib_error_t * elog_write_file(elog_main_t *em, char *clib_file, int flush_ring)
Definition: elog.h:528
u32 next_frame_index
Definition: node.h:460
static void * vlib_process_signal_event_helper(vlib_node_main_t *nm, vlib_node_t *n, vlib_process_t *p, uword t, uword n_data_elts, uword n_data_elt_bytes)
Definition: node_funcs.h:769
vlib_node_stats_t stats_total
Definition: node.h:278
static void vlib_put_next_frame_validate(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 next_index, u32 n_vectors_left)
Definition: main.c:419
u16 state
Input node state.
Definition: node.h:509
u32 frame_index
Definition: node.h:410
vlib_signal_timed_event_data_t * signal_timed_event_data_pool
Definition: node.h:728
static u32 vlib_frame_index(vlib_main_t *vm, vlib_frame_t *f)
Definition: node_funcs.h:245
u16 frame_flags
Definition: node.h:389
#define VLIB_FRAME_IS_ALLOCATED
Definition: node.h:427
#define VLIB_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:419
#define v
Definition: acl.c:496
struct _unformat_input_t unformat_input_t
static_always_inline u64 dispatch_node(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_node_type_t type, vlib_node_state_t dispatch_state, vlib_frame_t *frame, u64 last_time_stamp)
Definition: main.c:915
u32 n_alloc_frames
Definition: node.h:537
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:191
static uword vlib_timing_wheel_data_get_index(u32 d)
Definition: node.h:689
u8 inline_event_data[64-3 *sizeof(u32)-2 *sizeof(u16)]
Definition: node.h:662
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:274
elog_event_type_t * node_return_elog_event_types
Definition: main.h:165
#define TWT(a)
void vlib_frame_free(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_frame_t *f)
Definition: main.c:211
u32 polling_threshold_vector_length
Definition: node.h:716
u64 * n_vectors_by_next_node
Definition: node.h:348
#define PREDICT_FALSE(x)
Definition: clib.h:107
f64 seconds_per_clock
Definition: time.h:57
void elog_post_mortem_dump(void)
Definition: main.c:707
static vlib_frame_t * vlib_get_frame(vlib_main_t *vm, uword frame_index)
Definition: node_funcs.h:237
u32 node_index
Node index.
Definition: node.h:494
uword * init_functions_called
Definition: main.h:176
#define VLIB_ELOG_MAIN_LOOP
Definition: main.h:56
#define VLIB_PROCESS_STACK_MAGIC
Definition: node.h:623
#define VLIB_PROCESS_RESUME_LONGJMP_RESUME
Definition: node.h:563
void clib_time_init(clib_time_t *c)
Definition: time.c:178
uword * frame_size_hash
Definition: node.h:753
void elog_alloc(elog_main_t *em, u32 n_events)
Definition: elog.c:456
clib_error_t * vlibmemory_init(vlib_main_t *vm)
Definition: memory_api.c:896
volatile u32 queue_signal_pending
Definition: main.h:190
clib_error_t * map_api_segment_init(vlib_main_t *vm)
Definition: memory_api.c:468
u32 flags
Definition: vhost_user.h:115
static u64 dispatch_process(vlib_main_t *vm, vlib_process_t *p, vlib_frame_t *f, u64 last_time_stamp)
Definition: main.c:1274
u32 wraps
Definition: main.c:55
static u32 vlib_frame_index_no_check(vlib_main_t *vm, vlib_frame_t *f)
Definition: node_funcs.h:224
u32 owner_node_index
Definition: node.h:359
void vlib_register_all_static_nodes(vlib_main_t *vm)
Definition: node.c:539
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
Definition: init.h:216
static u32 vlib_node_runtime_update_main_loop_vector_stats(vlib_main_t *vm, vlib_node_runtime_t *node, uword n_vectors)
Definition: node_funcs.h:1055
#define UNFORMAT_END_OF_INPUT
Definition: format.h:144
svmdb_client_t * c
u16 n_vectors
Definition: node.h:401
u32 runtime_index
Definition: node.h:291
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:79
vlib_main_t * vm
Definition: buffer.c:294
static uword elog_n_events_in_buffer(elog_main_t *em)
Return number of events in the event-log buffer.
Definition: elog.h:188
static_always_inline void vlib_main_or_worker_loop(vlib_main_t *vm, int is_main)
Definition: main.c:1437
u32 node_runtime_index
Definition: node.h:454
vlib_pending_frame_t * pending_frames
Definition: node.h:723
int vlib_frame_queue_dequeue(vlib_main_t *vm, vlib_frame_queue_main_t *fqm)
Definition: threads.c:1558
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
clib_error_t * vpe_api_init(vlib_main_t *vm)
Definition: api.c:544
#define VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:301
static u32 vlib_node_runtime_update_stats(vlib_main_t *vm, vlib_node_runtime_t *node, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:608
#define clib_warning(format, args...)
Definition: error.h:59
#define VLIB_PROCESS_RESUME_PENDING
Definition: node.h:569
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
Definition: node_funcs.h:89
#define clib_memcpy(a, b, c)
Definition: string.h:75
#define VLIB_BUFFER_TRACE_TRAJECTORY
Compile time buffer trajectory tracing option Turn this on if you run into "bad monkey" contexts...
Definition: buffer.h:538
elog_main_t elog_main
Definition: main.h:157
clib_error_t * vlib_buffer_main_init(struct vlib_main_t *vm)
Definition: buffer.c:983
#define VLIB_FRAME_PENDING
Definition: node.h:430
static u32 vlib_frame_vector_byte_offset(u32 scalar_size)
Definition: node_funcs.h:256
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:455
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:439
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:237
u32 n_total_events_disable_limit
When count reaches limit logging is disabled.
Definition: elog.h:138
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:155
#define never_inline
Definition: clib.h:91
#define hash_create(elts, value_bytes)
Definition: hash.h:696
#define uword_to_pointer(u, type)
Definition: types.h:136
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
Definition: node.h:513
#define ASSERT(truth)
u64 last_cpu_time
Definition: time.h:50
static void elog_reset_buffer(elog_main_t *em)
Reset the event buffer.
Definition: elog.h:207
vlib_frame_queue_main_t * frame_queue_mains
Definition: threads.h:325
#define ELOG_TRACK_DATA(em, f, track)
Definition: elog.h:475
uword event_ring_size
Power of 2 number of elements in ring.
Definition: elog.h:144
u32 main_loop_nodes_processed
Definition: main.h:76
u64 cpu
CPU cycle counter.
Definition: elog.h:125
#define clib_error_report(e)
Definition: error.h:113
static uword pointer_to_uword(const void *p)
Definition: types.h:131
static u32 vlib_frame_alloc_to_node(vlib_main_t *vm, u32 to_node_index, u32 frame_flags)
Definition: main.c:115
void(* queue_signal_callback)(struct vlib_main_t *)
Definition: main.h:192
void vlib_gdb_show_event_log(void)
Definition: main.c:862
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
static_always_inline uword vlib_process_resume(vlib_process_t *p)
Definition: main.c:1261
#define VLIB_FRAME_OWNER
Definition: node.h:424
void vlib_node_sync_stats(vlib_main_t *vm, vlib_node_t *n)
Definition: main.c:571
#define vec_elt(v, i)
Get vector value at index i.
int vlib_main(vlib_main_t *volatile vm, unformat_input_t *input)
Definition: main.c:1689
#define unformat_parse_error(input)
Definition: format.h:268
void vlib_worker_loop(vlib_main_t *vm)
Definition: main.c:1633
static vlib_node_t * vlib_get_next_node(vlib_main_t *vm, u32 node_index, u32 next_index)
Get vlib node by graph arc (next) index.
Definition: node_funcs.h:72
void vl_api_send_pending_rpc_requests(vlib_main_t *)
Definition: main.c:1431
#define clib_mem_alloc_aligned_no_fail(size, align)
Definition: mem.h:166
CJ_GLOBAL_LOG_PROTOTYPE
Definition: main.c:49
u32 * data_from_advancing_timing_wheel
Definition: node.h:731
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u32 input_node_counts_by_state[VLIB_N_NODE_STATE]
Definition: node.h:750
vlib_pending_frame_t * suspended_process_frames
Definition: node.h:744
vlib_node_main_t node_main
Definition: main.h:134
static u32 * vlib_frame_find_magic(vlib_frame_t *f, vlib_node_t *node)
Definition: main.c:83
u64 uword
Definition: types.h:112
u32 owner_next_index
Definition: node.h:359
u8 vector_size
Definition: node.h:398
vlib_next_frame_t * next_frames
Definition: node.h:720
static clib_error_t * elog_stop(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:733
static void unformat_free(unformat_input_t *i)
Definition: format.h:162
clib_error_t * vlib_call_all_config_functions(vlib_main_t *vm, unformat_input_t *input, int is_early)
Definition: init.c:94
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:267
#define VLIB_FRAME_TRACE
Definition: node.h:436
u32 stop_timer_handle
Definition: node.h:605
clib_error_t * vlib_call_all_init_functions(vlib_main_t *vm)
Definition: init.c:67
#define VLIB_FRAME_SIZE_EXTRA
static void elog_track(elog_main_t *em, elog_event_type_t *type, elog_track_t *track, u32 data)
Log a single-datum event to a specific track, non-inline version.
Definition: elog.h:395
clib_error_t * unix_physmem_init(vlib_main_t *vm)
Definition: physmem.c:243
static u64 dispatch_suspended_process(vlib_main_t *vm, uword process_index, u64 last_time_stamp)
Definition: main.c:1355
vlib_frame_size_t * frame_sizes
Definition: node.h:756
u64 resume_clock_interval
Definition: node.h:602
static void add_trajectory_trace(vlib_buffer_t *b, u32 node_index)
Definition: main.c:904
u16 flags
Definition: node.h:565
elog_event_type_t * node_call_elog_event_types
Definition: main.h:164
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:140
vlib_node_type_t type
Definition: node.h:285
static vlib_process_t * vlib_get_process_from_node(vlib_main_t *vm, vlib_node_t *node)
Definition: node_funcs.h:207
uword clib_calljmp(uword(*func)(uword func_arg), uword func_arg, void *stack)
u8 * format_unformat_error(u8 *s, va_list *va)
Definition: unformat.c:91
u32 name_elog_string
Definition: node.h:275
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1455
vlib_node_state_t
Definition: node.h:258
u64 init_cpu_time
Definition: time.h:60
u32 suspended_process_frame_index
Definition: node.h:577
static void validate_frame_magic(vlib_main_t *vm, vlib_frame_t *f, vlib_node_t *n, uword next_index)
Definition: main.c:347
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
vlib_buffer_main_t buffer_main
Definition: buffer.c:52
static clib_error_t * vlib_main_configure(vlib_main_t *vm, unformat_input_t *input)
Definition: main.c:1641
static vlib_node_t * vlib_get_node(vlib_main_t *vm, u32 i)
Get vlib node by index.
Definition: node_funcs.h:59
static clib_error_t * vlib_cli_elog_clear(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:653
#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT
Definition: node.h:567
#define vec_foreach(var, vec)
Vector iterator.
elog_track_t default_track
Default track.
Definition: elog.h:163
clib_longjmp_t return_longjmp
Definition: node.h:555
u16 flags
Copy of main node flags.
Definition: node.h:507
u32 node_runtime_index
Definition: node.h:413
u32 n_total_events
Total number of events in buffer.
Definition: elog.h:134
u8 scalar_size
Definition: node.h:395
clib_longjmp_t main_loop_exit
Definition: main.h:91
clib_longjmp_t resume_longjmp
Definition: node.h:561
void clib_mem_trace(int enable)
Definition: mem_dlmalloc.c:431
static clib_error_t * elog_resize(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:773
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:310
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
#define TW(a)
clib_error_t * vlib_map_stat_segment_init(void)
Definition: stat_segment.c:213
static u32 vlib_timing_wheel_data_set_suspended_process(u32 i)
Definition: node.h:677
void vlib_start_process(vlib_main_t *vm, uword process_index)
Definition: main.c:1347
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:725
void * timing_wheel
Definition: node.h:726
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:58
clib_random_buffer_t random_buffer
Definition: main.h:173
static clib_error_t * elog_restart(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:753
#define VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE
Definition: node.h:313
vlib_buffer_free_list_index_t vlib_buffer_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
Definition: buffer.c:415
u8 *(* validate_frame)(struct vlib_main_t *vm, struct vlib_node_runtime_t *, struct vlib_frame_t *f)
Definition: node.h:369
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:972
clib_error_t * main_loop_error
Definition: main.h:98
vlib_process_t * process
Definition: main.c:1210
elog_event_t * elog_peek_events(elog_main_t *em)
convert event ring events to events, and return them as a vector.
Definition: elog.c:513
static uword elog_buffer_capacity(elog_main_t *em)
Return number of events which can fit in the event buffer.
Definition: elog.h:198
u64 cpu_time_last_node_dispatch
Definition: main.h:66
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:170
#define VLIB_FRAME_FREE_AFTER_DISPATCH
Definition: node.h:433
clib_error_t * vlib_thread_init(vlib_main_t *vm)
Definition: threads.c:237
u32 vectors_since_last_overflow
Definition: node.h:439
u32 max_clock_n
Number of vectors in the recorded max_clock.
Definition: node.h:483
#define VLIB_PROCESS_RETURN_LONGJMP_SUSPEND
Definition: node.h:558