FD.io VPP  v20.05.1-6-gf53edbc3b
Vector Packet Processing
main.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * main.c: main vector processing loop
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #include <math.h>
41 #include <vppinfra/format.h>
42 #include <vlib/vlib.h>
43 #include <vlib/threads.h>
45 
46 #include <vlib/unix/unix.h>
47 
48 /* Actually allocate a few extra slots of vector data to support
49  speculative vector enqueues which overflow vector data in next frame. */
50 #define VLIB_FRAME_SIZE_ALLOC (VLIB_FRAME_SIZE + 4)
51 
53 vlib_frame_bytes (u32 n_scalar_bytes, u32 n_vector_bytes)
54 {
55  u32 n_bytes;
56 
57  /* Make room for vlib_frame_t plus scalar arguments. */
58  n_bytes = vlib_frame_vector_byte_offset (n_scalar_bytes);
59 
60  /* Make room for vector arguments.
61  Allocate a few extra slots of vector data to support
62  speculative vector enqueues which overflow vector data in next frame. */
63 #define VLIB_FRAME_SIZE_EXTRA 4
64  n_bytes += (VLIB_FRAME_SIZE + VLIB_FRAME_SIZE_EXTRA) * n_vector_bytes;
65 
66  /* Magic number is first 32bit number after vector data.
67  Used to make sure that vector data is never overrun. */
68 #define VLIB_FRAME_MAGIC (0xabadc0ed)
69  n_bytes += sizeof (u32);
70 
71  /* Pad to cache line. */
72  n_bytes = round_pow2 (n_bytes, CLIB_CACHE_LINE_BYTES);
73 
74  return n_bytes;
75 }
76 
79 {
80  void *p = f;
81 
83 
85 
86  return p;
87 }
88 
89 static inline vlib_frame_size_t *
91  u32 n_scalar_bytes, u32 n_vector_bytes)
92 {
93 #ifdef VLIB_SUPPORTS_ARBITRARY_SCALAR_SIZES
94  uword key = (n_scalar_bytes << 16) | n_vector_bytes;
95  uword *p, i;
96 
97  p = hash_get (nm->frame_size_hash, key);
98  if (p)
99  i = p[0];
100  else
101  {
102  i = vec_len (nm->frame_sizes);
103  vec_validate (nm->frame_sizes, i);
104  hash_set (nm->frame_size_hash, key, i);
105  }
106 
107  return vec_elt_at_index (nm->frame_sizes, i);
108 #else
109  ASSERT (vlib_frame_bytes (n_scalar_bytes, n_vector_bytes)
110  == (vlib_frame_bytes (0, 4)));
111  return vec_elt_at_index (nm->frame_sizes, 0);
112 #endif
113 }
114 
115 static vlib_frame_t *
117  u32 frame_flags)
118 {
119  vlib_node_main_t *nm = &vm->node_main;
120  vlib_frame_size_t *fs;
121  vlib_node_t *to_node;
122  vlib_frame_t *f;
123  u32 l, n, scalar_size, vector_size;
124 
125  to_node = vlib_get_node (vm, to_node_index);
126 
127  scalar_size = to_node->scalar_size;
128  vector_size = to_node->vector_size;
129 
130  fs = get_frame_size_info (nm, scalar_size, vector_size);
131  n = vlib_frame_bytes (scalar_size, vector_size);
132  if ((l = vec_len (fs->free_frames)) > 0)
133  {
134  /* Allocate from end of free list. */
135  f = fs->free_frames[l - 1];
136  _vec_len (fs->free_frames) = l - 1;
137  }
138  else
139  {
141  }
142 
143  /* Poison frame when debugging. */
144  if (CLIB_DEBUG > 0)
145  clib_memset (f, 0xfe, n);
146 
147  /* Insert magic number. */
148  {
149  u32 *magic;
150 
151  magic = vlib_frame_find_magic (f, to_node);
152  *magic = VLIB_FRAME_MAGIC;
153  }
154 
155  f->frame_flags = VLIB_FRAME_IS_ALLOCATED | frame_flags;
156  f->n_vectors = 0;
157  f->scalar_size = scalar_size;
158  f->vector_size = vector_size;
159  f->flags = 0;
160 
161  fs->n_alloc_frames += 1;
162 
163  return f;
164 }
165 
166 /* Allocate a frame for from FROM_NODE to TO_NODE via TO_NEXT_INDEX.
167  Returns frame index. */
168 static vlib_frame_t *
170  u32 to_next_index)
171 {
172  vlib_node_t *from_node;
173 
174  from_node = vlib_get_node (vm, from_node_runtime->node_index);
175  ASSERT (to_next_index < vec_len (from_node->next_nodes));
176 
177  return vlib_frame_alloc_to_node (vm, from_node->next_nodes[to_next_index],
178  /* frame_flags */ 0);
179 }
180 
181 vlib_frame_t *
183 {
184  vlib_frame_t *f = vlib_frame_alloc_to_node (vm, to_node_index,
185  /* frame_flags */
187  return vlib_get_frame (vm, f);
188 }
189 
190 static inline void
192 {
193  if (CLIB_DEBUG > 0)
194  {
195  int i;
196  u32 *from = vlib_frame_vector_args (f);
197 
198  /* Check for bad buffer index values */
199  for (i = 0; i < f->n_vectors; i++)
200  {
201  if (from[i] == 0)
202  {
203  clib_warning ("BUG: buffer index 0 at index %d", i);
204  ASSERT (0);
205  }
206  else if (from[i] == 0xfefefefe)
207  {
208  clib_warning ("BUG: frame poison pattern at index %d", i);
209  ASSERT (0);
210  }
211  }
212  }
213 }
214 
215 void
217 {
219  vlib_node_t *to_node;
220 
221  if (f->n_vectors == 0)
222  return;
223 
225 
226  to_node = vlib_get_node (vm, to_node_index);
227 
228  vec_add2 (vm->node_main.pending_frames, p, 1);
229 
231  p->frame = vlib_get_frame (vm, f);
232  p->node_runtime_index = to_node->runtime_index;
234 }
235 
236 /* Free given frame. */
237 void
239 {
240  vlib_node_main_t *nm = &vm->node_main;
241  vlib_node_t *node;
242  vlib_frame_size_t *fs;
243 
245 
246  node = vlib_get_node (vm, r->node_index);
247  fs = get_frame_size_info (nm, node->scalar_size, node->vector_size);
248 
250 
251  /* No next frames may point to freed frame. */
252  if (CLIB_DEBUG > 0)
253  {
254  vlib_next_frame_t *nf;
255  vec_foreach (nf, vm->node_main.next_frames) ASSERT (nf->frame != f);
256  }
257 
259 
260  vec_add1 (fs->free_frames, f);
261  ASSERT (fs->n_alloc_frames > 0);
262  fs->n_alloc_frames -= 1;
263 }
264 
265 static clib_error_t *
267  unformat_input_t * input, vlib_cli_command_t * cmd)
268 {
269  vlib_node_main_t *nm = &vm->node_main;
270  vlib_frame_size_t *fs;
271 
272  vlib_cli_output (vm, "%=6s%=12s%=12s", "Size", "# Alloc", "# Free");
273  vec_foreach (fs, nm->frame_sizes)
274  {
275  u32 n_alloc = fs->n_alloc_frames;
276  u32 n_free = vec_len (fs->free_frames);
277 
278  if (n_alloc + n_free > 0)
279  vlib_cli_output (vm, "%=6d%=12d%=12d",
280  fs - nm->frame_sizes, n_alloc, n_free);
281  }
282 
283  return 0;
284 }
285 
286 /* *INDENT-OFF* */
287 VLIB_CLI_COMMAND (show_frame_stats_cli, static) = {
288  .path = "show vlib frame-allocation",
289  .short_help = "Show node dispatch frame statistics",
290  .function = show_frame_stats,
291 };
292 /* *INDENT-ON* */
293 
294 /* Change ownership of enqueue rights to given next node. */
295 static void
297  vlib_node_runtime_t * node_runtime,
298  u32 next_index)
299 {
300  vlib_node_main_t *nm = &vm->node_main;
301  vlib_next_frame_t *next_frame;
302  vlib_node_t *node, *next_node;
303 
304  node = vec_elt (nm->nodes, node_runtime->node_index);
305 
306  /* Only internal & input nodes are allowed to call other nodes. */
308  || node->type == VLIB_NODE_TYPE_INPUT
309  || node->type == VLIB_NODE_TYPE_PROCESS);
310 
311  ASSERT (vec_len (node->next_nodes) == node_runtime->n_next_nodes);
312 
313  next_frame =
314  vlib_node_runtime_get_next_frame (vm, node_runtime, next_index);
315  next_node = vec_elt (nm->nodes, node->next_nodes[next_index]);
316 
317  if (next_node->owner_node_index != VLIB_INVALID_NODE_INDEX)
318  {
319  /* Get frame from previous owner. */
320  vlib_next_frame_t *owner_next_frame;
321  vlib_next_frame_t tmp;
322 
323  owner_next_frame =
325  next_node->owner_node_index,
326  next_node->owner_next_index);
327 
328  /* Swap target next frame with owner's. */
329  tmp = owner_next_frame[0];
330  owner_next_frame[0] = next_frame[0];
331  next_frame[0] = tmp;
332 
333  /*
334  * If next_frame is already pending, we have to track down
335  * all pending frames and fix their next_frame_index fields.
336  */
337  if (next_frame->flags & VLIB_FRAME_PENDING)
338  {
340  if (next_frame->frame != NULL)
341  {
342  vec_foreach (p, nm->pending_frames)
343  {
344  if (p->frame == next_frame->frame)
345  {
346  p->next_frame_index =
347  next_frame - vm->node_main.next_frames;
348  }
349  }
350  }
351  }
352  }
353  else
354  {
355  /* No previous owner. Take ownership. */
356  next_frame->flags |= VLIB_FRAME_OWNER;
357  }
358 
359  /* Record new owner. */
360  next_node->owner_node_index = node->index;
361  next_node->owner_next_index = next_index;
362 
363  /* Now we should be owner. */
364  ASSERT (next_frame->flags & VLIB_FRAME_OWNER);
365 }
366 
367 /* Make sure that magic number is still there.
368  Otherwise, it is likely that caller has overrun frame arguments. */
369 always_inline void
371  vlib_frame_t * f, vlib_node_t * n, uword next_index)
372 {
373  vlib_node_t *next_node = vlib_get_node (vm, n->next_nodes[next_index]);
374  u32 *magic = vlib_frame_find_magic (f, next_node);
375  ASSERT (VLIB_FRAME_MAGIC == magic[0]);
376 }
377 
378 vlib_frame_t *
381  u32 next_index, u32 allocate_new_next_frame)
382 {
383  vlib_frame_t *f;
384  vlib_next_frame_t *nf;
385  u32 n_used;
386 
387  nf = vlib_node_runtime_get_next_frame (vm, node, next_index);
388 
389  /* Make sure this next frame owns right to enqueue to destination frame. */
390  if (PREDICT_FALSE (!(nf->flags & VLIB_FRAME_OWNER)))
391  vlib_next_frame_change_ownership (vm, node, next_index);
392 
393  /* ??? Don't need valid flag: can use frame_index == ~0 */
395  {
396  nf->frame = vlib_frame_alloc (vm, node, next_index);
398  }
399 
400  f = nf->frame;
401 
402  /* Has frame been removed from pending vector (e.g. finished dispatching)?
403  If so we can reuse frame. */
404  if ((nf->flags & VLIB_FRAME_PENDING)
405  && !(f->frame_flags & VLIB_FRAME_PENDING))
406  {
407  nf->flags &= ~VLIB_FRAME_PENDING;
408  f->n_vectors = 0;
409  f->flags = 0;
410  }
411 
412  /* Allocate new frame if current one is marked as no-append or
413  it is already full. */
414  n_used = f->n_vectors;
415  if (n_used >= VLIB_FRAME_SIZE || (allocate_new_next_frame && n_used > 0) ||
417  {
418  /* Old frame may need to be freed after dispatch, since we'll have
419  two redundant frames from node -> next node. */
421  {
422  vlib_frame_t *f_old = vlib_get_frame (vm, nf->frame);
424  }
425 
426  /* Allocate new frame to replace full one. */
427  f = nf->frame = vlib_frame_alloc (vm, node, next_index);
428  n_used = f->n_vectors;
429  }
430 
431  /* Should have free vectors in frame now. */
432  ASSERT (n_used < VLIB_FRAME_SIZE);
433 
434  if (CLIB_DEBUG > 0)
435  {
436  validate_frame_magic (vm, f,
437  vlib_get_node (vm, node->node_index), next_index);
438  }
439 
440  return f;
441 }
442 
443 static void
445  vlib_node_runtime_t * rt,
446  u32 next_index, u32 n_vectors_left)
447 {
448  vlib_node_main_t *nm = &vm->node_main;
449  vlib_next_frame_t *nf;
450  vlib_frame_t *f;
451  vlib_node_runtime_t *next_rt;
452  vlib_node_t *next_node;
453  u32 n_before, n_after;
454 
455  nf = vlib_node_runtime_get_next_frame (vm, rt, next_index);
456  f = vlib_get_frame (vm, nf->frame);
457 
458  ASSERT (n_vectors_left <= VLIB_FRAME_SIZE);
459 
461 
462  n_after = VLIB_FRAME_SIZE - n_vectors_left;
463  n_before = f->n_vectors;
464 
465  ASSERT (n_after >= n_before);
466 
468  nf->node_runtime_index);
469  next_node = vlib_get_node (vm, next_rt->node_index);
470  if (n_after > 0 && next_node->validate_frame)
471  {
472  u8 *msg = next_node->validate_frame (vm, rt, f);
473  if (msg)
474  {
475  clib_warning ("%v", msg);
476  ASSERT (0);
477  }
478  vec_free (msg);
479  }
480 }
481 
482 void
485  u32 next_index, u32 n_vectors_left)
486 {
487  vlib_node_main_t *nm = &vm->node_main;
488  vlib_next_frame_t *nf;
489  vlib_frame_t *f;
490  u32 n_vectors_in_frame;
491 
492  if (CLIB_DEBUG > 0)
493  vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left);
494 
495  nf = vlib_node_runtime_get_next_frame (vm, r, next_index);
496  f = vlib_get_frame (vm, nf->frame);
497 
498  /* Make sure that magic number is still there. Otherwise, caller
499  has overrun frame meta data. */
500  if (CLIB_DEBUG > 0)
501  {
503  validate_frame_magic (vm, f, node, next_index);
504  }
505 
506  /* Convert # of vectors left -> number of vectors there. */
507  ASSERT (n_vectors_left <= VLIB_FRAME_SIZE);
508  n_vectors_in_frame = VLIB_FRAME_SIZE - n_vectors_left;
509 
510  f->n_vectors = n_vectors_in_frame;
511 
512  /* If vectors were added to frame, add to pending vector. */
513  if (PREDICT_TRUE (n_vectors_in_frame > 0))
514  {
516  u32 v0, v1;
517 
518  r->cached_next_index = next_index;
519 
520  if (!(f->frame_flags & VLIB_FRAME_PENDING))
521  {
522  __attribute__ ((unused)) vlib_node_t *node;
523  vlib_node_t *next_node;
524  vlib_node_runtime_t *next_runtime;
525 
526  node = vlib_get_node (vm, r->node_index);
527  next_node = vlib_get_next_node (vm, r->node_index, next_index);
528  next_runtime = vlib_node_get_runtime (vm, next_node->index);
529 
530  vec_add2 (nm->pending_frames, p, 1);
531 
532  p->frame = nf->frame;
534  p->next_frame_index = nf - nm->next_frames;
535  nf->flags |= VLIB_FRAME_PENDING;
537 
538  /*
539  * If we're going to dispatch this frame on another thread,
540  * force allocation of a new frame. Otherwise, we create
541  * a dangling frame reference. Each thread has its own copy of
542  * the next_frames vector.
543  */
544  if (0 && r->thread_index != next_runtime->thread_index)
545  {
546  nf->frame = NULL;
548  }
549  }
550 
551  /* Copy trace flag from next_frame and from runtime. */
552  nf->flags |=
553  (nf->flags & VLIB_NODE_FLAG_TRACE) | (r->
555 
557  v1 = v0 + n_vectors_in_frame;
559  if (PREDICT_FALSE (v1 < v0))
560  {
562  vec_elt (node->n_vectors_by_next_node, next_index) += v0;
563  }
564  }
565 }
566 
567 /* Sync up runtime (32 bit counters) and main node stats (64 bit counters). */
568 never_inline void
571  uword n_calls, uword n_vectors, uword n_clocks,
572  uword n_ticks0, uword n_ticks1)
573 {
574  vlib_node_t *n = vlib_get_node (vm, r->node_index);
575 
576  n->stats_total.calls += n_calls + r->calls_since_last_overflow;
577  n->stats_total.vectors += n_vectors + r->vectors_since_last_overflow;
578  n->stats_total.clocks += n_clocks + r->clocks_since_last_overflow;
579  n->stats_total.perf_counter0_ticks += n_ticks0 +
581  n->stats_total.perf_counter1_ticks += n_ticks1 +
583  n->stats_total.perf_counter_vectors += n_vectors +
587 
594 }
595 
596 always_inline void __attribute__ ((unused))
598  vlib_process_t * p,
599  uword n_calls, uword n_vectors, uword n_clocks,
600  uword n_ticks0, uword n_ticks1)
601 {
602  vlib_node_runtime_t *rt = &p->node_runtime;
603  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
604  vlib_node_runtime_sync_stats (vm, rt, n_calls, n_vectors, n_clocks,
605  n_ticks0, n_ticks1);
606  n->stats_total.suspends += p->n_suspends;
607  p->n_suspends = 0;
608 }
609 
610 void
612 {
614 
615  if (n->type == VLIB_NODE_TYPE_PROCESS)
616  {
617  /* Nothing to do for PROCESS nodes except in main thread */
618  if (vm != &vlib_global_main)
619  return;
620 
623  p->n_suspends = 0;
624  rt = &p->node_runtime;
625  }
626  else
627  rt =
629  n->runtime_index);
630 
631  vlib_node_runtime_sync_stats (vm, rt, 0, 0, 0, 0, 0);
632 
633  /* Sync up runtime next frame vector counters with main node structure. */
634  {
635  vlib_next_frame_t *nf;
636  uword i;
637  for (i = 0; i < rt->n_next_nodes; i++)
638  {
639  nf = vlib_node_runtime_get_next_frame (vm, rt, i);
643  }
644  }
645 }
646 
650  uword n_calls,
651  uword n_vectors, uword n_clocks,
652  uword n_ticks0, uword n_ticks1)
653 {
654  u32 ca0, ca1, v0, v1, cl0, cl1, r;
655  u32 ptick00, ptick01, ptick10, ptick11, pvec0, pvec1;
656 
657  cl0 = cl1 = node->clocks_since_last_overflow;
658  ca0 = ca1 = node->calls_since_last_overflow;
659  v0 = v1 = node->vectors_since_last_overflow;
660  ptick00 = ptick01 = node->perf_counter0_ticks_since_last_overflow;
661  ptick10 = ptick11 = node->perf_counter1_ticks_since_last_overflow;
662  pvec0 = pvec1 = node->perf_counter_vectors_since_last_overflow;
663 
664  ca1 = ca0 + n_calls;
665  v1 = v0 + n_vectors;
666  cl1 = cl0 + n_clocks;
667  ptick01 = ptick00 + n_ticks0;
668  ptick11 = ptick10 + n_ticks1;
669  pvec1 = pvec0 + n_vectors;
670 
671  node->calls_since_last_overflow = ca1;
672  node->clocks_since_last_overflow = cl1;
673  node->vectors_since_last_overflow = v1;
677 
678  node->max_clock_n = node->max_clock > n_clocks ?
679  node->max_clock_n : n_vectors;
680  node->max_clock = node->max_clock > n_clocks ? node->max_clock : n_clocks;
681 
682  r = vlib_node_runtime_update_main_loop_vector_stats (vm, node, n_vectors);
683 
684  if (PREDICT_FALSE (ca1 < ca0 || v1 < v0 || cl1 < cl0) || (ptick01 < ptick00)
685  || (ptick11 < ptick10) || (pvec1 < pvec0))
686  {
687  node->calls_since_last_overflow = ca0;
688  node->clocks_since_last_overflow = cl0;
689  node->vectors_since_last_overflow = v0;
693 
694  vlib_node_runtime_sync_stats (vm, node, n_calls, n_vectors, n_clocks,
695  n_ticks0, n_ticks1);
696  }
697 
698  return r;
699 }
700 
701 always_inline void
704  vlib_frame_t * frame, int before_or_after)
705 {
706  *pmc0 = 0;
707  *pmc1 = 0;
710  pmc1, node, frame, before_or_after);
711 }
712 
713 always_inline void
715  vlib_process_t * p,
716  uword n_calls, uword n_vectors, uword n_clocks)
717 {
719  n_calls, n_vectors, n_clocks, 0ULL, 0ULL);
720 }
721 
722 static clib_error_t *
724  unformat_input_t * input, vlib_cli_command_t * cmd)
725 {
727  return 0;
728 }
729 
730 /* *INDENT-OFF* */
731 VLIB_CLI_COMMAND (elog_clear_cli, static) = {
732  .path = "event-logger clear",
733  .short_help = "Clear the event log",
734  .function = vlib_cli_elog_clear,
735 };
736 /* *INDENT-ON* */
737 
738 #ifdef CLIB_UNIX
739 static clib_error_t *
741  unformat_input_t * input, vlib_cli_command_t * cmd)
742 {
743  elog_main_t *em = &vm->elog_main;
744  char *file, *chroot_file;
745  clib_error_t *error = 0;
746 
747  if (!unformat (input, "%s", &file))
748  {
749  vlib_cli_output (vm, "expected file name, got `%U'",
750  format_unformat_error, input);
751  return 0;
752  }
753 
754  /* It's fairly hard to get "../oopsie" through unformat; just in case */
755  if (strstr (file, "..") || index (file, '/'))
756  {
757  vlib_cli_output (vm, "illegal characters in filename '%s'", file);
758  return 0;
759  }
760 
761  chroot_file = (char *) format (0, "/tmp/%s%c", file, 0);
762 
763  vec_free (file);
764 
765  vlib_cli_output (vm, "Saving %wd of %wd events to %s",
767  elog_buffer_capacity (em), chroot_file);
768 
770  error = elog_write_file (em, chroot_file, 1 /* flush ring */ );
772  vec_free (chroot_file);
773  return error;
774 }
775 
776 void
778 {
780  elog_main_t *em = &vm->elog_main;
781  u8 *filename;
782  clib_error_t *error;
783 
784  if (!vm->elog_post_mortem_dump)
785  return;
786 
787  filename = format (0, "/tmp/elog_post_mortem.%d%c", getpid (), 0);
788  error = elog_write_file (em, (char *) filename, 1 /* flush ring */ );
789  if (error)
790  clib_error_report (error);
791  vec_free (filename);
792 }
793 
794 /* *INDENT-OFF* */
795 VLIB_CLI_COMMAND (elog_save_cli, static) = {
796  .path = "event-logger save",
797  .short_help = "event-logger save <filename> (saves log in /tmp/<filename>)",
798  .function = elog_save_buffer,
799 };
800 /* *INDENT-ON* */
801 
802 static clib_error_t *
804  unformat_input_t * input, vlib_cli_command_t * cmd)
805 {
806  elog_main_t *em = &vm->elog_main;
807 
809 
810  vlib_cli_output (vm, "Stopped the event logger...");
811  return 0;
812 }
813 
814 /* *INDENT-OFF* */
815 VLIB_CLI_COMMAND (elog_stop_cli, static) = {
816  .path = "event-logger stop",
817  .short_help = "Stop the event-logger",
818  .function = elog_stop,
819 };
820 /* *INDENT-ON* */
821 
822 static clib_error_t *
824  unformat_input_t * input, vlib_cli_command_t * cmd)
825 {
826  elog_main_t *em = &vm->elog_main;
827 
829 
830  vlib_cli_output (vm, "Restarted the event logger...");
831  return 0;
832 }
833 
834 /* *INDENT-OFF* */
835 VLIB_CLI_COMMAND (elog_restart_cli, static) = {
836  .path = "event-logger restart",
837  .short_help = "Restart the event-logger",
838  .function = elog_restart,
839 };
840 /* *INDENT-ON* */
841 
842 static clib_error_t *
844  unformat_input_t * input, vlib_cli_command_t * cmd)
845 {
846  elog_main_t *em = &vm->elog_main;
847  u32 tmp;
848 
849  /* Stop the parade */
851 
852  if (unformat (input, "%d", &tmp))
853  {
854  elog_alloc (em, tmp);
856  }
857  else
858  return clib_error_return (0, "Must specify how many events in the ring");
859 
860  vlib_cli_output (vm, "Resized ring and restarted the event logger...");
861  return 0;
862 }
863 
864 /* *INDENT-OFF* */
865 VLIB_CLI_COMMAND (elog_resize_cli, static) = {
866  .path = "event-logger resize",
867  .short_help = "event-logger resize <nnn>",
868  .function = elog_resize,
869 };
870 /* *INDENT-ON* */
871 
872 #endif /* CLIB_UNIX */
873 
874 static void
876 {
877  elog_main_t *em = &vm->elog_main;
878  elog_event_t *e, *es;
879  f64 dt;
880 
881  /* Show events in VLIB time since log clock starts after VLIB clock. */
882  dt = (em->init_time.cpu - vm->clib_time.init_cpu_time)
884 
885  es = elog_peek_events (em);
886  vlib_cli_output (vm, "%d of %d events in buffer, logger %s", vec_len (es),
887  em->event_ring_size,
889  "running" : "stopped");
890  vec_foreach (e, es)
891  {
892  vlib_cli_output (vm, "%18.9f: %U",
893  e->time + dt, format_elog_event, em, e);
894  n_events_to_show--;
895  if (n_events_to_show == 0)
896  break;
897  }
898  vec_free (es);
899 
900 }
901 
902 static clib_error_t *
904  unformat_input_t * input, vlib_cli_command_t * cmd)
905 {
906  u32 n_events_to_show;
907  clib_error_t *error = 0;
908 
909  n_events_to_show = 250;
911  {
912  if (unformat (input, "%d", &n_events_to_show))
913  ;
914  else if (unformat (input, "all"))
915  n_events_to_show = ~0;
916  else
917  return unformat_parse_error (input);
918  }
919  elog_show_buffer_internal (vm, n_events_to_show);
920  return error;
921 }
922 
923 /* *INDENT-OFF* */
924 VLIB_CLI_COMMAND (elog_show_cli, static) = {
925  .path = "show event-logger",
926  .short_help = "Show event logger info",
927  .function = elog_show_buffer,
928 };
929 /* *INDENT-ON* */
930 
931 void
933 {
935 }
936 
937 static inline void
939  u32 node_index,
940  u64 time, u32 n_vectors, u32 is_return)
941 {
943  elog_main_t *em = &evm->elog_main;
944  int enabled = evm->elog_trace_graph_dispatch |
946 
947  if (PREDICT_FALSE (enabled && n_vectors))
948  {
949  if (PREDICT_FALSE (!elog_is_enabled (em)))
950  {
951  evm->elog_trace_graph_dispatch = 0;
952  evm->elog_trace_graph_circuit = 0;
953  return;
954  }
955  if (PREDICT_TRUE
957  (evm->elog_trace_graph_circuit &&
958  node_index == evm->elog_trace_graph_circuit_node_index)))
959  {
960  elog_track (em,
961  /* event type */
962  vec_elt_at_index (is_return
965  node_index),
966  /* track */
967  (vm->thread_index ?
969  : &em->default_track),
970  /* data to log */ n_vectors);
971  }
972  }
973 }
974 
975 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
976 void (*vlib_buffer_trace_trajectory_cb) (vlib_buffer_t * b, u32 node_index);
977 void (*vlib_buffer_trace_trajectory_init_cb) (vlib_buffer_t * b);
978 
979 void
980 vlib_buffer_trace_trajectory_init (vlib_buffer_t * b)
981 {
982  if (PREDICT_TRUE (vlib_buffer_trace_trajectory_init_cb != 0))
983  {
984  (*vlib_buffer_trace_trajectory_init_cb) (b);
985  }
986 }
987 
988 #endif
989 
990 static inline void
992 {
993 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
994  if (PREDICT_TRUE (vlib_buffer_trace_trajectory_cb != 0))
995  {
996  (*vlib_buffer_trace_trajectory_cb) (b, node_index);
997  }
998 #endif
999 }
1000 
1001 u8 *format_vnet_buffer_flags (u8 * s, va_list * args) __attribute__ ((weak));
1002 u8 *
1003 format_vnet_buffer_flags (u8 * s, va_list * args)
1004 {
1005  s = format (s, "BUG STUB %s", __FUNCTION__);
1006  return s;
1007 }
1008 
1009 u8 *format_vnet_buffer_opaque (u8 * s, va_list * args) __attribute__ ((weak));
1010 u8 *
1011 format_vnet_buffer_opaque (u8 * s, va_list * args)
1012 {
1013  s = format (s, "BUG STUB %s", __FUNCTION__);
1014  return s;
1015 }
1016 
1017 u8 *format_vnet_buffer_opaque2 (u8 * s, va_list * args)
1018  __attribute__ ((weak));
1019 u8 *
1020 format_vnet_buffer_opaque2 (u8 * s, va_list * args)
1021 {
1022  s = format (s, "BUG STUB %s", __FUNCTION__);
1023  return s;
1024 }
1025 
1026 static u8 *
1027 format_buffer_metadata (u8 * s, va_list * args)
1028 {
1029  vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
1030 
1031  s = format (s, "flags: %U\n", format_vnet_buffer_flags, b);
1032  s = format (s, "current_data: %d, current_length: %d\n",
1033  (i32) (b->current_data), (i32) (b->current_length));
1034  s = format
1035  (s,
1036  "current_config_index/punt_reason: %d, flow_id: %x, next_buffer: %x\n",
1038  s =
1039  format (s, "error: %d, ref_count: %d, buffer_pool_index: %d\n",
1040  (u32) (b->error), (u32) (b->ref_count),
1041  (u32) (b->buffer_pool_index));
1042  s =
1043  format (s, "trace_handle: 0x%x, len_not_first_buf: %d\n", b->trace_handle,
1045  return s;
1046 }
1047 
1048 #define A(x) vec_add1(vm->pcap_buffer, (x))
1049 
1050 static void
1053 {
1054  int i;
1055  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **bufp, *b;
1056  pcap_main_t *pm = &vlib_global_main.dispatch_pcap_main;
1057  vlib_trace_main_t *tm = &vm->trace_main;
1058  u32 capture_size;
1059  vlib_node_t *n;
1060  i32 n_left;
1061  f64 time_now = vlib_time_now (vm);
1062  u32 *from;
1063  u8 *d;
1064  u8 string_count;
1065 
1066  /* Input nodes don't have frames yet */
1067  if (frame == 0 || frame->n_vectors == 0)
1068  return;
1069 
1070  from = vlib_frame_vector_args (frame);
1071  vlib_get_buffers (vm, from, bufs, frame->n_vectors);
1072  bufp = bufs;
1073 
1074  n = vlib_get_node (vm, node->node_index);
1075 
1076  for (i = 0; i < frame->n_vectors; i++)
1077  {
1079  {
1080  b = bufp[i];
1081 
1083  string_count = 0;
1084 
1085  /* Version, flags */
1088  A (0 /* string_count */ );
1089  A (n->protocol_hint);
1090 
1091  /* Buffer index (big endian) */
1092  A ((from[i] >> 24) & 0xff);
1093  A ((from[i] >> 16) & 0xff);
1094  A ((from[i] >> 8) & 0xff);
1095  A ((from[i] >> 0) & 0xff);
1096 
1097  /* Node name, NULL-terminated ASCII */
1098  vm->pcap_buffer = format (vm->pcap_buffer, "%v%c", n->name, 0);
1099  string_count++;
1100 
1101  vm->pcap_buffer = format (vm->pcap_buffer, "%U%c",
1102  format_buffer_metadata, b, 0);
1103  string_count++;
1104  vm->pcap_buffer = format (vm->pcap_buffer, "%U%c",
1106  string_count++;
1107  vm->pcap_buffer = format (vm->pcap_buffer, "%U%c",
1109  string_count++;
1110 
1111  /* Is this packet traced? */
1112  if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED))
1113  {
1117 
1118  vm->pcap_buffer = format (vm->pcap_buffer, "%U%c",
1119  format_vlib_trace, vm, h[0], 0);
1120  string_count++;
1121  }
1122 
1123  /* Save the string count */
1124  vm->pcap_buffer[2] = string_count;
1125 
1126  /* Figure out how many bytes in the pcap trace */
1127  capture_size = vec_len (vm->pcap_buffer) +
1128  +vlib_buffer_length_in_chain (vm, b);
1129 
1131  n_left = clib_min (capture_size, 16384);
1132  d = pcap_add_packet (pm, time_now, n_left, capture_size);
1133 
1134  /* Copy the header */
1136  d += vec_len (vm->pcap_buffer);
1137 
1138  n_left = clib_min
1139  (vlib_buffer_length_in_chain (vm, b),
1140  (16384 - vec_len (vm->pcap_buffer)));
1141  /* Copy the packet data */
1142  while (1)
1143  {
1144  u32 copy_length = clib_min ((u32) n_left, b->current_length);
1145  clib_memcpy_fast (d, b->data + b->current_data, copy_length);
1146  n_left -= b->current_length;
1147  if (n_left <= 0)
1148  break;
1149  d += b->current_length;
1150  ASSERT (b->flags & VLIB_BUFFER_NEXT_PRESENT);
1151  b = vlib_get_buffer (vm, b->next_buffer);
1152  }
1154  }
1155  }
1156 }
1157 
1162  vlib_node_state_t dispatch_state,
1163  vlib_frame_t * frame, u64 last_time_stamp)
1164 {
1165  uword n, v;
1166  u64 t;
1167  vlib_node_main_t *nm = &vm->node_main;
1168  vlib_next_frame_t *nf;
1169  u64 pmc_before[2], pmc_after[2], pmc_delta[2];
1170 
1171  if (CLIB_DEBUG > 0)
1172  {
1173  vlib_node_t *n = vlib_get_node (vm, node->node_index);
1174  ASSERT (n->type == type);
1175  }
1176 
1177  /* Only non-internal nodes may be disabled. */
1178  if (type != VLIB_NODE_TYPE_INTERNAL && node->state != dispatch_state)
1179  {
1180  ASSERT (type != VLIB_NODE_TYPE_INTERNAL);
1181  return last_time_stamp;
1182  }
1183 
1184  if ((type == VLIB_NODE_TYPE_PRE_INPUT || type == VLIB_NODE_TYPE_INPUT)
1185  && dispatch_state != VLIB_NODE_STATE_INTERRUPT)
1186  {
1188  /* Only call node when count reaches zero. */
1189  if (c)
1190  {
1191  node->input_main_loops_per_call = c - 1;
1192  return last_time_stamp;
1193  }
1194  }
1195 
1196  /* Speculatively prefetch next frames. */
1197  if (node->n_next_nodes > 0)
1198  {
1199  nf = vec_elt_at_index (nm->next_frames, node->next_frame_index);
1200  CLIB_PREFETCH (nf, 4 * sizeof (nf[0]), WRITE);
1201  }
1202 
1203  vm->cpu_time_last_node_dispatch = last_time_stamp;
1204 
1206  last_time_stamp, frame ? frame->n_vectors : 0,
1207  /* is_after */ 0);
1208 
1209  vlib_node_runtime_perf_counter (vm, &pmc_before[0], &pmc_before[1],
1210  node, frame, 0 /* before */ );
1211 
1212  /*
1213  * Turn this on if you run into
1214  * "bad monkey" contexts, and you want to know exactly
1215  * which nodes they've visited... See ixge.c...
1216  */
1217  if (VLIB_BUFFER_TRACE_TRAJECTORY && frame)
1218  {
1219  int i;
1220  u32 *from;
1221  from = vlib_frame_vector_args (frame);
1222  for (i = 0; i < frame->n_vectors; i++)
1223  {
1224  vlib_buffer_t *b = vlib_get_buffer (vm, from[i]);
1225  add_trajectory_trace (b, node->node_index);
1226  }
1228  dispatch_pcap_trace (vm, node, frame);
1229  n = node->function (vm, node, frame);
1230  }
1231  else
1232  {
1234  dispatch_pcap_trace (vm, node, frame);
1235  n = node->function (vm, node, frame);
1236  }
1237 
1238  t = clib_cpu_time_now ();
1239 
1240  /*
1241  * To validate accounting: pmc_delta = t - pmc_before;
1242  * perf ticks should equal clocks/pkt...
1243  */
1244  vlib_node_runtime_perf_counter (vm, &pmc_after[0], &pmc_after[1], node,
1245  frame, 1 /* after */ );
1246 
1247  pmc_delta[0] = pmc_after[0] - pmc_before[0];
1248  pmc_delta[1] = pmc_after[1] - pmc_before[1];
1249 
1250  vlib_elog_main_loop_event (vm, node->node_index, t, n, 1 /* is_after */ );
1251 
1252  vm->main_loop_vectors_processed += n;
1253  vm->main_loop_nodes_processed += n > 0;
1254 
1255  v = vlib_node_runtime_update_stats (vm, node,
1256  /* n_calls */ 1,
1257  /* n_vectors */ n,
1258  /* n_clocks */ t - last_time_stamp,
1259  pmc_delta[0] /* PMC0 */ ,
1260  pmc_delta[1] /* PMC1 */ );
1261 
1262  /* When in interrupt mode and vector rate crosses threshold switch to
1263  polling mode. */
1264  if (PREDICT_FALSE ((dispatch_state == VLIB_NODE_STATE_INTERRUPT)
1265  || (dispatch_state == VLIB_NODE_STATE_POLLING
1266  && (node->flags
1267  &
1269  {
1270  /* *INDENT-OFF* */
1271  ELOG_TYPE_DECLARE (e) =
1272  {
1273  .function = (char *) __FUNCTION__,
1274  .format = "%s vector length %d, switching to %s",
1275  .format_args = "T4i4t4",
1276  .n_enum_strings = 2,
1277  .enum_strings = {
1278  "interrupt", "polling",
1279  },
1280  };
1281  /* *INDENT-ON* */
1282  struct
1283  {
1284  u32 node_name, vector_length, is_polling;
1285  } *ed;
1286 
1287  if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT
1288  && v >= nm->polling_threshold_vector_length) &&
1289  !(node->flags &
1291  {
1292  vlib_node_t *n = vlib_get_node (vm, node->node_index);
1293  n->state = VLIB_NODE_STATE_POLLING;
1294  node->state = VLIB_NODE_STATE_POLLING;
1295  node->flags &=
1298  nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] -= 1;
1299  nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] += 1;
1300 
1301  if (PREDICT_FALSE (vlib_global_main.elog_trace_graph_dispatch))
1302  {
1304  + vm->thread_index;
1305 
1306  ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
1307  w->elog_track);
1308  ed->node_name = n->name_elog_string;
1309  ed->vector_length = v;
1310  ed->is_polling = 1;
1311  }
1312  }
1313  else if (dispatch_state == VLIB_NODE_STATE_POLLING
1314  && v <= nm->interrupt_threshold_vector_length)
1315  {
1316  vlib_node_t *n = vlib_get_node (vm, node->node_index);
1317  if (node->flags &
1319  {
1320  /* Switch to interrupt mode after dispatch in polling one more time.
1321  This allows driver to re-enable interrupts. */
1322  n->state = VLIB_NODE_STATE_INTERRUPT;
1323  node->state = VLIB_NODE_STATE_INTERRUPT;
1324  node->flags &=
1326  nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] -= 1;
1327  nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] += 1;
1328 
1329  }
1330  else
1331  {
1333  + vm->thread_index;
1334  node->flags |=
1336  if (PREDICT_FALSE (vlib_global_main.elog_trace_graph_dispatch))
1337  {
1338  ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
1339  w->elog_track);
1340  ed->node_name = n->name_elog_string;
1341  ed->vector_length = v;
1342  ed->is_polling = 0;
1343  }
1344  }
1345  }
1346  }
1347 
1348  return t;
1349 }
1350 
1351 static u64
1352 dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index,
1353  u64 last_time_stamp)
1354 {
1355  vlib_node_main_t *nm = &vm->node_main;
1356  vlib_frame_t *f;
1357  vlib_next_frame_t *nf, nf_dummy;
1359  vlib_frame_t *restore_frame;
1361 
1362  /* See comment below about dangling references to nm->pending_frames */
1363  p = nm->pending_frames + pending_frame_index;
1364 
1366  p->node_runtime_index);
1367 
1368  f = vlib_get_frame (vm, p->frame);
1370  {
1371  /* No next frame: so use dummy on stack. */
1372  nf = &nf_dummy;
1374  nf->frame = NULL;
1375  }
1376  else
1378 
1380 
1381  /* Force allocation of new frame while current frame is being
1382  dispatched. */
1383  restore_frame = NULL;
1384  if (nf->frame == p->frame)
1385  {
1386  nf->frame = NULL;
1389  restore_frame = p->frame;
1390  }
1391 
1392  /* Frame must be pending. */
1394  ASSERT (f->n_vectors > 0);
1395 
1396  /* Copy trace flag from next frame to node.
1397  Trace flag indicates that at least one vector in the dispatched
1398  frame is traced. */
1399  n->flags &= ~VLIB_NODE_FLAG_TRACE;
1400  n->flags |= (nf->flags & VLIB_FRAME_TRACE) ? VLIB_NODE_FLAG_TRACE : 0;
1401  nf->flags &= ~VLIB_FRAME_TRACE;
1402 
1403  last_time_stamp = dispatch_node (vm, n,
1405  VLIB_NODE_STATE_POLLING,
1406  f, last_time_stamp);
1407  /* Internal node vector-rate accounting, for summary stats */
1408  vm->internal_node_vectors += f->n_vectors;
1409  vm->internal_node_calls++;
1413 
1415 
1416  /* Frame is ready to be used again, so restore it. */
1417  if (restore_frame != NULL)
1418  {
1419  /*
1420  * We musn't restore a frame that is flagged to be freed. This
1421  * shouldn't happen since frames to be freed post dispatch are
1422  * those used when the to-node frame becomes full i.e. they form a
1423  * sort of queue of frames to a single node. If we get here then
1424  * the to-node frame and the pending frame *were* the same, and so
1425  * we removed the to-node frame. Therefore this frame is no
1426  * longer part of the queue for that node and hence it cannot be
1427  * it's overspill.
1428  */
1430 
1431  /*
1432  * NB: dispatching node n can result in the creation and scheduling
1433  * of new frames, and hence in the reallocation of nm->pending_frames.
1434  * Recompute p, or no supper. This was broken for more than 10 years.
1435  */
1436  p = nm->pending_frames + pending_frame_index;
1437 
1438  /*
1439  * p->next_frame_index can change during node dispatch if node
1440  * function decides to change graph hook up.
1441  */
1444 
1445  if (NULL == nf->frame)
1446  {
1447  /* no new frame has been assigned to this node, use the saved one */
1448  nf->frame = restore_frame;
1449  f->n_vectors = 0;
1450  }
1451  else
1452  {
1453  /* The node has gained a frame, implying packets from the current frame
1454  were re-queued to this same node. we don't need the saved one
1455  anymore */
1456  vlib_frame_free (vm, n, f);
1457  }
1458  }
1459  else
1460  {
1462  {
1464  vlib_frame_free (vm, n, f);
1465  }
1466  }
1467 
1468  return last_time_stamp;
1469 }
1470 
1473 {
1474  return p->stack[0] == VLIB_PROCESS_STACK_MAGIC;
1475 }
1476 
1477 typedef struct
1478 {
1483 
1484 /* Called in process stack. */
1485 static uword
1487 {
1489  vlib_main_t *vm;
1491  vlib_frame_t *f;
1492  vlib_process_t *p;
1493  uword n;
1494 
1496 
1497  vm = a->vm;
1498  p = a->process;
1499  f = a->frame;
1500  node = &p->node_runtime;
1501 
1502  n = node->function (vm, node, f);
1503 
1505 
1506  clib_longjmp (&p->return_longjmp, n);
1507 
1508  return n;
1509 }
1510 
1511 /* Called in main stack. */
1514 {
1516  uword r;
1517 
1518  a.vm = vm;
1519  a.process = p;
1520  a.frame = f;
1521 
1525  (void *) p->stack + (1 << p->log2_n_stack_bytes));
1526 
1527  return r;
1528 }
1529 
1532 {
1533  uword r;
1540  return r;
1541 }
1542 
1543 static u64
1545  vlib_process_t * p, vlib_frame_t * f, u64 last_time_stamp)
1546 {
1547  vlib_node_main_t *nm = &vm->node_main;
1548  vlib_node_runtime_t *node_runtime = &p->node_runtime;
1549  vlib_node_t *node = vlib_get_node (vm, node_runtime->node_index);
1550  u32 old_process_index;
1551  u64 t;
1552  uword n_vectors, is_suspend;
1553 
1554  if (node->state != VLIB_NODE_STATE_POLLING
1557  return last_time_stamp;
1558 
1560 
1561  t = last_time_stamp;
1562  vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
1563  f ? f->n_vectors : 0, /* is_after */ 0);
1564 
1565  /* Save away current process for suspend. */
1566  old_process_index = nm->current_process_index;
1568 
1569  n_vectors = vlib_process_startup (vm, p, f);
1570 
1571  nm->current_process_index = old_process_index;
1572 
1574  is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND;
1575  if (is_suspend)
1576  {
1578 
1579  n_vectors = 0;
1581  pf->node_runtime_index = node->runtime_index;
1582  pf->frame = f;
1583  pf->next_frame_index = ~0;
1584 
1585  p->n_suspends += 1;
1587 
1589  {
1590  TWT (tw_timer_wheel) * tw =
1591  (TWT (tw_timer_wheel) *) nm->timing_wheel;
1592  p->stop_timer_handle =
1593  TW (tw_timer_start) (tw,
1595  (node->runtime_index) /* [sic] pool idex */ ,
1596  0 /* timer_id */ ,
1598  }
1599  }
1600  else
1602 
1603  t = clib_cpu_time_now ();
1604 
1605  vlib_elog_main_loop_event (vm, node_runtime->node_index, t, is_suspend,
1606  /* is_after */ 1);
1607 
1609  /* n_calls */ !is_suspend,
1610  /* n_vectors */ n_vectors,
1611  /* n_clocks */ t - last_time_stamp);
1612 
1613  return t;
1614 }
1615 
1616 void
1618 {
1619  vlib_node_main_t *nm = &vm->node_main;
1620  vlib_process_t *p = vec_elt (nm->processes, process_index);
1621  dispatch_process (vm, p, /* frame */ 0, /* cpu_time_now */ 0);
1622 }
1623 
1624 static u64
1626  uword process_index, u64 last_time_stamp)
1627 {
1628  vlib_node_main_t *nm = &vm->node_main;
1629  vlib_node_runtime_t *node_runtime;
1630  vlib_node_t *node;
1631  vlib_frame_t *f;
1632  vlib_process_t *p;
1634  u64 t, n_vectors, is_suspend;
1635 
1636  t = last_time_stamp;
1637 
1638  p = vec_elt (nm->processes, process_index);
1640  return last_time_stamp;
1641 
1644 
1647 
1648  node_runtime = &p->node_runtime;
1649  node = vlib_get_node (vm, node_runtime->node_index);
1650  f = pf->frame;
1651 
1652  vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
1653  f ? f->n_vectors : 0, /* is_after */ 0);
1654 
1655  /* Save away current process for suspend. */
1657 
1658  n_vectors = vlib_process_resume (p);
1659  t = clib_cpu_time_now ();
1660 
1661  nm->current_process_index = ~0;
1662 
1663  is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND;
1664  if (is_suspend)
1665  {
1666  /* Suspend it again. */
1667  n_vectors = 0;
1668  p->n_suspends += 1;
1670  {
1671  p->stop_timer_handle =
1672  TW (tw_timer_start) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
1674  (node->runtime_index) /* [sic] pool idex */ ,
1675  0 /* timer_id */ ,
1677  }
1678  }
1679  else
1680  {
1685  }
1686 
1687  t = clib_cpu_time_now ();
1688  vlib_elog_main_loop_event (vm, node_runtime->node_index, t, !is_suspend,
1689  /* is_after */ 1);
1690 
1692  /* n_calls */ !is_suspend,
1693  /* n_vectors */ n_vectors,
1694  /* n_clocks */ t - last_time_stamp);
1695 
1696  return t;
1697 }
1698 
1699 void vl_api_send_pending_rpc_requests (vlib_main_t *) __attribute__ ((weak));
1700 void
1702 {
1703 }
1704 
1705 
1708 {
1709  vlib_node_main_t *nm = &vm->node_main;
1711  uword i;
1712  u64 cpu_time_now;
1713  f64 now;
1715  u32 *last_node_runtime_indices = 0;
1716  u32 frame_queue_check_counter = 0;
1717 
1718  /* Initialize pending node vector. */
1719  if (is_main)
1720  {
1721  vec_resize (nm->pending_frames, 32);
1722  _vec_len (nm->pending_frames) = 0;
1723  }
1724 
1725  /* Mark time of main loop start. */
1726  if (is_main)
1727  {
1728  cpu_time_now = vm->clib_time.last_cpu_time;
1729  vm->cpu_time_main_loop_start = cpu_time_now;
1730  }
1731  else
1732  cpu_time_now = clib_cpu_time_now ();
1733 
1734  /* Pre-allocate interupt runtime indices and lock. */
1736  vec_alloc (last_node_runtime_indices, 32);
1737  if (!is_main)
1739 
1740  /* Pre-allocate expired nodes. */
1745 
1749 
1750  /* Start all processes. */
1751  if (is_main)
1752  {
1753  uword i;
1754 
1755  /*
1756  * Perform an initial barrier sync. Pays no attention to
1757  * the barrier sync hold-down timer scheme, which won't work
1758  * at this point in time.
1759  */
1761 
1762  nm->current_process_index = ~0;
1763  for (i = 0; i < vec_len (nm->processes); i++)
1764  cpu_time_now = dispatch_process (vm, nm->processes[i], /* frame */ 0,
1765  cpu_time_now);
1766  }
1767 
1768  while (1)
1769  {
1771 
1772  if (PREDICT_FALSE (_vec_len (vm->pending_rpc_requests) > 0))
1773  {
1774  if (!is_main)
1776  }
1777 
1778  if (!is_main)
1779  {
1782  frame_queue_check_counter))
1783  {
1784  u32 processed = 0;
1785 
1786  if (vm->check_frame_queues)
1787  {
1788  frame_queue_check_counter = 100;
1789  vm->check_frame_queues = 0;
1790  }
1791 
1792  vec_foreach (fqm, tm->frame_queue_mains)
1793  processed += vlib_frame_queue_dequeue (vm, fqm);
1794 
1795  /* No handoff queue work found? */
1796  if (processed)
1797  frame_queue_check_counter = 100;
1798  else
1799  frame_queue_check_counter--;
1800  }
1803  }
1804 
1805  /* Process pre-input nodes. */
1807  cpu_time_now = dispatch_node (vm, n,
1809  VLIB_NODE_STATE_POLLING,
1810  /* frame */ 0,
1811  cpu_time_now);
1812 
1813  /* Next process input nodes. */
1815  cpu_time_now = dispatch_node (vm, n,
1817  VLIB_NODE_STATE_POLLING,
1818  /* frame */ 0,
1819  cpu_time_now);
1820 
1821  if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0))
1822  vm->queue_signal_callback (vm);
1823 
1824  /* Next handle interrupts. */
1825  {
1826  /* unlocked read, for performance */
1827  uword l = _vec_len (nm->pending_interrupt_node_runtime_indices);
1828  uword i;
1829  if (PREDICT_FALSE (l > 0))
1830  {
1831  u32 *tmp;
1832  if (!is_main)
1833  {
1835  /* Re-read w/ lock held, in case another thread added an item */
1836  l = _vec_len (nm->pending_interrupt_node_runtime_indices);
1837  }
1838 
1841  last_node_runtime_indices;
1842  last_node_runtime_indices = tmp;
1843  _vec_len (last_node_runtime_indices) = 0;
1844  if (!is_main)
1846  for (i = 0; i < l; i++)
1847  {
1849  last_node_runtime_indices[i]);
1850  cpu_time_now =
1852  VLIB_NODE_STATE_INTERRUPT,
1853  /* frame */ 0,
1854  cpu_time_now);
1855  }
1856  }
1857  }
1858  /* Input nodes may have added work to the pending vector.
1859  Process pending vector until there is nothing left.
1860  All pending vectors will be processed from input -> output. */
1861  for (i = 0; i < _vec_len (nm->pending_frames); i++)
1862  cpu_time_now = dispatch_pending_node (vm, i, cpu_time_now);
1863  /* Reset pending vector for next iteration. */
1864  _vec_len (nm->pending_frames) = 0;
1865 
1866  if (is_main)
1867  {
1868  /* *INDENT-OFF* */
1869  ELOG_TYPE_DECLARE (es) =
1870  {
1871  .format = "process tw start",
1872  .format_args = "",
1873  };
1874  ELOG_TYPE_DECLARE (ee) =
1875  {
1876  .format = "process tw end: %d",
1877  .format_args = "i4",
1878  };
1879  /* *INDENT-ON* */
1880 
1881  struct
1882  {
1883  int nready_procs;
1884  } *ed;
1885 
1886  /* Check if process nodes have expired from timing wheel. */
1888 
1890  ed = ELOG_DATA (&vlib_global_main.elog_main, es);
1891 
1894  ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm),
1896 
1898 
1900  {
1901  ed = ELOG_DATA (&vlib_global_main.elog_main, ee);
1902  ed->nready_procs =
1903  _vec_len (nm->data_from_advancing_timing_wheel);
1904  }
1905 
1906  if (PREDICT_FALSE
1907  (_vec_len (nm->data_from_advancing_timing_wheel) > 0))
1908  {
1909  uword i;
1910 
1911  for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel);
1912  i++)
1913  {
1916 
1918  {
1921  di);
1922  vlib_node_t *n =
1924  vlib_process_t *p =
1925  vec_elt (nm->processes, n->runtime_index);
1926  void *data;
1927  data =
1929  te->event_type_index,
1930  te->n_data_elts,
1931  te->n_data_elt_bytes);
1932  if (te->n_data_bytes < sizeof (te->inline_event_data))
1934  te->n_data_bytes);
1935  else
1936  {
1938  te->n_data_bytes);
1940  }
1942  }
1943  else
1944  {
1945  cpu_time_now = clib_cpu_time_now ();
1946  cpu_time_now =
1947  dispatch_suspended_process (vm, di, cpu_time_now);
1948  }
1949  }
1950  _vec_len (nm->data_from_advancing_timing_wheel) = 0;
1951  }
1952  }
1954  /* Record time stamp in case there are no enabled nodes and above
1955  calls do not update time stamp. */
1956  cpu_time_now = clib_cpu_time_now ();
1958  now = clib_time_now_internal (&vm->clib_time, cpu_time_now);
1959  /* Time to update loops_per_second? */
1960  if (PREDICT_FALSE (now >= vm->loop_interval_end))
1961  {
1962  /* Next sample ends in 20ms */
1963  if (vm->loop_interval_start)
1964  {
1965  f64 this_loops_per_second;
1966 
1967  this_loops_per_second =
1968  ((f64) vm->loops_this_reporting_interval) / (now -
1969  vm->loop_interval_start);
1970 
1971  vm->loops_per_second =
1973  (1.0 - vm->damping_constant) * this_loops_per_second;
1974  if (vm->loops_per_second != 0.0)
1975  vm->seconds_per_loop = 1.0 / vm->loops_per_second;
1976  else
1977  vm->seconds_per_loop = 0.0;
1978  }
1979  /* New interval starts now, and ends in 20ms */
1980  vm->loop_interval_start = now;
1981  vm->loop_interval_end = now + 2e-4;
1983  }
1984  }
1985 }
1986 
1987 static void
1989 {
1990  vlib_main_or_worker_loop (vm, /* is_main */ 1);
1991 }
1992 
1993 void
1995 {
1996  vlib_main_or_worker_loop (vm, /* is_main */ 0);
1997 }
1998 
2000 
2001 static clib_error_t *
2003 {
2004  int turn_on_mem_trace = 0;
2005 
2006  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2007  {
2008  if (unformat (input, "memory-trace"))
2009  turn_on_mem_trace = 1;
2010 
2011  else if (unformat (input, "elog-events %d",
2012  &vm->elog_main.event_ring_size))
2013  ;
2014  else if (unformat (input, "elog-post-mortem-dump"))
2015  vm->elog_post_mortem_dump = 1;
2016  else if (unformat (input, "buffer-alloc-success-rate %f",
2018  {
2019  if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR == 0)
2020  return clib_error_return
2021  (0, "Buffer fault injection not configured");
2022  }
2023  else if (unformat (input, "buffer-alloc-success-seed %u",
2025  {
2026  if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR == 0)
2027  return clib_error_return
2028  (0, "Buffer fault injection not configured");
2029  }
2030  else
2031  return unformat_parse_error (input);
2032  }
2033 
2034  unformat_free (input);
2035 
2036  /* Enable memory trace as early as possible. */
2037  if (turn_on_mem_trace)
2038  clib_mem_trace (1);
2039 
2040  return 0;
2041 }
2042 
2044 
2045 static void
2047 {
2048 }
2049 
2050 #define foreach_weak_reference_stub \
2051 _(vlib_map_stat_segment_init) \
2052 _(vpe_api_init) \
2053 _(vlibmemory_init) \
2054 _(map_api_segment_init)
2055 
2056 #define _(name) \
2057 clib_error_t *name (vlib_main_t *vm) __attribute__((weak)); \
2058 clib_error_t *name (vlib_main_t *vm) { return 0; }
2060 #undef _
2061 
2062 void vl_api_set_elog_main (elog_main_t * m) __attribute__ ((weak));
2063 void
2065 {
2066  clib_warning ("STUB");
2067 }
2068 
2069 int vl_api_set_elog_trace_api_messages (int enable) __attribute__ ((weak));
2070 int
2072 {
2073  clib_warning ("STUB");
2074  return 0;
2075 }
2076 
2077 int vl_api_get_elog_trace_api_messages (void) __attribute__ ((weak));
2078 int
2080 {
2081  clib_warning ("STUB");
2082  return 0;
2083 }
2084 
2085 /* Main function. */
2086 int
2088 {
2089  clib_error_t *volatile error;
2090  vlib_node_main_t *nm = &vm->node_main;
2091 
2093 
2094  /* Turn on event log. */
2095  if (!vm->elog_main.event_ring_size)
2096  vm->elog_main.event_ring_size = 128 << 10;
2098  elog_enable_disable (&vm->elog_main, 1);
2101 
2102  /* Default name. */
2103  if (!vm->name)
2104  vm->name = "VLIB";
2105 
2106  if ((error = vlib_physmem_init (vm)))
2107  {
2108  clib_error_report (error);
2109  goto done;
2110  }
2111 
2112  if ((error = vlib_map_stat_segment_init (vm)))
2113  {
2114  clib_error_report (error);
2115  goto done;
2116  }
2117 
2118  if ((error = vlib_buffer_main_init (vm)))
2119  {
2120  clib_error_report (error);
2121  goto done;
2122  }
2123 
2124  if ((error = vlib_thread_init (vm)))
2125  {
2126  clib_error_report (error);
2127  goto done;
2128  }
2129 
2130  /* Register static nodes so that init functions may use them. */
2132 
2133  /* Set seed for random number generator.
2134  Allow user to specify seed to make random sequence deterministic. */
2135  if (!unformat (input, "seed %wd", &vm->random_seed))
2136  vm->random_seed = clib_cpu_time_now ();
2138 
2139  /* Initialize node graph. */
2140  if ((error = vlib_node_main_init (vm)))
2141  {
2142  /* Arrange for graph hook up error to not be fatal when debugging. */
2143  if (CLIB_DEBUG > 0)
2144  clib_error_report (error);
2145  else
2146  goto done;
2147  }
2148 
2149  /* Direct call / weak reference, for vlib standalone use-cases */
2150  if ((error = vpe_api_init (vm)))
2151  {
2152  clib_error_report (error);
2153  goto done;
2154  }
2155 
2156  if ((error = vlibmemory_init (vm)))
2157  {
2158  clib_error_report (error);
2159  goto done;
2160  }
2161 
2162  if ((error = map_api_segment_init (vm)))
2163  {
2164  clib_error_report (error);
2165  goto done;
2166  }
2167 
2168  /* See unix/main.c; most likely already set up */
2169  if (vm->init_functions_called == 0)
2170  vm->init_functions_called = hash_create (0, /* value bytes */ 0);
2171  if ((error = vlib_call_all_init_functions (vm)))
2172  goto done;
2173 
2174  nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)),
2176 
2178  _vec_len (nm->data_from_advancing_timing_wheel) = 0;
2179 
2180  /* Create the process timing wheel */
2181  TW (tw_timer_wheel_init) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
2182  0 /* no callback */ ,
2183  10e-6 /* timer period 10us */ ,
2184  ~0 /* max expirations per call */ );
2185 
2187  _vec_len (vm->pending_rpc_requests) = 0;
2189  _vec_len (vm->processing_rpc_requests) = 0;
2190 
2191  /* Default params for the buffer allocator fault injector, if configured */
2192  if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR > 0)
2193  {
2194  vm->buffer_alloc_success_seed = 0xdeaddabe;
2195  vm->buffer_alloc_success_rate = 0.80;
2196  }
2197 
2198  if ((error = vlib_call_all_config_functions (vm, input, 0 /* is_early */ )))
2199  goto done;
2200 
2201  /*
2202  * Use exponential smoothing, with a half-life of 1 second
2203  * reported_rate(t) = reported_rate(t-1) * K + rate(t)*(1-K)
2204  *
2205  * Sample every 20ms, aka 50 samples per second
2206  * K = exp (-1.0/20.0);
2207  * K = 0.95
2208  */
2209  vm->damping_constant = exp (-1.0 / 20.0);
2210 
2211  /* Sort per-thread init functions before we start threads */
2213 
2214  /* Call all main loop enter functions. */
2215  {
2216  clib_error_t *sub_error;
2217  sub_error = vlib_call_all_main_loop_enter_functions (vm);
2218  if (sub_error)
2219  clib_error_report (sub_error);
2220  }
2221 
2223  {
2225  vm->main_loop_exit_set = 1;
2226  break;
2227 
2229  goto done;
2230 
2231  default:
2232  error = vm->main_loop_error;
2233  goto done;
2234  }
2235 
2236  vlib_main_loop (vm);
2237 
2238 done:
2239  /* Call all exit functions. */
2240  {
2241  clib_error_t *sub_error;
2242  sub_error = vlib_call_all_main_loop_exit_functions (vm);
2243  if (sub_error)
2244  clib_error_report (sub_error);
2245  }
2246 
2247  if (error)
2248  clib_error_report (error);
2249 
2250  return 0;
2251 }
2252 
2253 int
2255 {
2257  pcap_main_t *pm = &vm->dispatch_pcap_main;
2258  vlib_trace_main_t *tm;
2259  vlib_trace_node_t *tn;
2260 
2261  if (a->status)
2262  {
2263  if (vm->dispatch_pcap_enable)
2264  {
2265  int i;
2267  (vm, "pcap dispatch capture enabled: %d of %d pkts...",
2269  vlib_cli_output (vm, "capture to file %s", pm->file_name);
2270 
2271  for (i = 0; i < vec_len (vm->dispatch_buffer_trace_nodes); i++)
2272  {
2273  vlib_cli_output (vm,
2274  "Buffer trace of %d pkts from %U enabled...",
2278  }
2279  }
2280  else
2281  vlib_cli_output (vm, "pcap dispatch capture disabled");
2282  return 0;
2283  }
2284 
2285  /* Consistency checks */
2286 
2287  /* Enable w/ capture already enabled not allowed */
2288  if (vm->dispatch_pcap_enable && a->enable)
2289  return -7; /* VNET_API_ERROR_INVALID_VALUE */
2290 
2291  /* Disable capture with capture already disabled, not interesting */
2292  if (vm->dispatch_pcap_enable == 0 && a->enable == 0)
2293  return -81; /* VNET_API_ERROR_VALUE_EXIST */
2294 
2295  /* Change number of packets to capture while capturing */
2296  if (vm->dispatch_pcap_enable && a->enable
2298  return -8; /* VNET_API_ERROR_INVALID_VALUE_2 */
2299 
2300  /* Independent of enable/disable, to allow buffer trace multi nodes */
2301  if (a->buffer_trace_node_index != ~0)
2302  {
2303  /* *INDENT-OFF* */
2305  {
2306  tm = &this_vlib_main->trace_main;
2307  tm->verbose = 0; /* not sure this ever did anything... */
2309  tn = tm->nodes + a->buffer_trace_node_index;
2310  tn->limit += a->buffer_traces_to_capture;
2311  tm->trace_enable = 1;
2312  }));
2313  /* *INDENT-ON* */
2315  }
2316 
2317  if (a->enable)
2318  {
2319  /* Clean up from previous run, if any */
2320  vec_free (pm->file_name);
2321  vec_free (pm->pcap_data);
2322  memset (pm, 0, sizeof (*pm));
2323 
2325  if (pm->lock == 0)
2326  clib_spinlock_init (&(pm->lock));
2327 
2328  if (a->filename == 0)
2329  a->filename = format (0, "/tmp/dispatch.pcap%c", 0);
2330 
2331  pm->file_name = (char *) a->filename;
2332  pm->n_packets_captured = 0;
2333  pm->packet_type = PCAP_PACKET_TYPE_vpp;
2335  /* *INDENT-OFF* */
2336  foreach_vlib_main (({this_vlib_main->dispatch_pcap_enable = 1;}));
2337  /* *INDENT-ON* */
2338  }
2339  else
2340  {
2341  /* *INDENT-OFF* */
2342  foreach_vlib_main (({this_vlib_main->dispatch_pcap_enable = 0;}));
2343  /* *INDENT-ON* */
2345  if (pm->n_packets_captured)
2346  {
2347  clib_error_t *error;
2349  vlib_cli_output (vm, "Write %d packets to %s, and stop capture...",
2350  pm->n_packets_captured, pm->file_name);
2351  error = pcap_write (pm);
2352  if (pm->flags & PCAP_MAIN_INIT_DONE)
2353  pcap_close (pm);
2354  /* Report I/O errors... */
2355  if (error)
2356  {
2357  clib_error_report (error);
2358  return -11; /* VNET_API_ERROR_SYSCALL_ERROR_1 */
2359  }
2360  return 0;
2361  }
2362  else
2363  return -6; /* VNET_API_ERROR_NO_SUCH_ENTRY */
2364  }
2365 
2366  return 0;
2367 }
2368 
2369 static clib_error_t *
2371  unformat_input_t * input, vlib_cli_command_t * cmd)
2372 {
2373  unformat_input_t _line_input, *line_input = &_line_input;
2375  u8 *filename = 0;
2376  u32 max = 1000;
2377  int rv;
2378  int enable = 0;
2379  int status = 0;
2380  u32 node_index = ~0, buffer_traces_to_capture = 100;
2381 
2382  /* Get a line of input. */
2383  if (!unformat_user (input, unformat_line_input, line_input))
2384  return 0;
2385 
2386  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
2387  {
2388  if (unformat (line_input, "on %=", &enable, 1))
2389  ;
2390  else if (unformat (line_input, "enable %=", &enable, 1))
2391  ;
2392  else if (unformat (line_input, "off %=", &enable, 0))
2393  ;
2394  else if (unformat (line_input, "disable %=", &enable, 0))
2395  ;
2396  else if (unformat (line_input, "max %d", &max))
2397  ;
2398  else if (unformat (line_input, "packets-to-capture %d", &max))
2399  ;
2400  else if (unformat (line_input, "file %U", unformat_vlib_tmpfile,
2401  &filename))
2402  ;
2403  else if (unformat (line_input, "status %=", &status, 1))
2404  ;
2405  else if (unformat (line_input, "buffer-trace %U %d",
2406  unformat_vlib_node, vm, &node_index,
2407  &buffer_traces_to_capture))
2408  ;
2409  else
2410  {
2411  return clib_error_return (0, "unknown input `%U'",
2412  format_unformat_error, line_input);
2413  }
2414  }
2415 
2416  unformat_free (line_input);
2417 
2418  /* no need for memset (a, 0, sizeof (*a)), set all fields here. */
2419  a->filename = filename;
2420  a->enable = enable;
2421  a->status = status;
2422  a->packets_to_capture = max;
2423  a->buffer_trace_node_index = node_index;
2424  a->buffer_traces_to_capture = buffer_traces_to_capture;
2425 
2427 
2428  switch (rv)
2429  {
2430  case 0:
2431  break;
2432 
2433  case -7:
2434  return clib_error_return (0, "dispatch trace already enabled...");
2435 
2436  case -81:
2437  return clib_error_return (0, "dispatch trace already disabled...");
2438 
2439  case -8:
2440  return clib_error_return
2441  (0, "can't change number of records to capture while tracing...");
2442 
2443  case -11:
2444  return clib_error_return (0, "I/O writing trace capture...");
2445 
2446  case -6:
2447  return clib_error_return (0, "No packets captured...");
2448 
2449  default:
2450  vlib_cli_output (vm, "WARNING: trace configure returned %d", rv);
2451  break;
2452  }
2453  return 0;
2454 }
2455 
2456 /*?
2457  * This command is used to start or stop pcap dispatch trace capture, or show
2458  * the capture status.
2459  *
2460  * This command has the following optional parameters:
2461  *
2462  * - <b>on|off</b> - Used to start or stop capture.
2463  *
2464  * - <b>max <nn></b> - Depth of local buffer. Once '<em>nn</em>' number
2465  * of packets have been received, buffer is flushed to file. Once another
2466  * '<em>nn</em>' number of packets have been received, buffer is flushed
2467  * to file, overwriting previous write. If not entered, value defaults
2468  * to 100. Can only be updated if packet capture is off.
2469  *
2470  * - <b>file <name></b> - Used to specify the output filename. The file will
2471  * be placed in the '<em>/tmp</em>' directory, so only the filename is
2472  * supported. Directory should not be entered. If file already exists, file
2473  * will be overwritten. If no filename is provided, '<em>/tmp/vpe.pcap</em>'
2474  * will be used. Can only be updated if packet capture is off.
2475  *
2476  * - <b>status</b> - Displays the current status and configured attributes
2477  * associated with a packet capture. If packet capture is in progress,
2478  * '<em>status</em>' also will return the number of packets currently in
2479  * the local buffer. All additional attributes entered on command line
2480  * with '<em>status</em>' will be ignored and not applied.
2481  *
2482  * @cliexpar
2483  * Example of how to display the status of capture when off:
2484  * @cliexstart{pcap dispatch trace status}
2485  * max is 100, for any interface to file /tmp/vpe.pcap
2486  * pcap dispatch capture is off...
2487  * @cliexend
2488  * Example of how to start a dispatch trace capture:
2489  * @cliexstart{pcap dispatch trace on max 35 file dispatchTrace.pcap}
2490  * pcap dispatch capture on...
2491  * @cliexend
2492  * Example of how to start a dispatch trace capture with buffer tracing
2493  * @cliexstart{pcap dispatch trace on max 10000 file dispatchTrace.pcap buffer-trace dpdk-input 1000}
2494  * pcap dispatch capture on...
2495  * @cliexend
2496  * Example of how to display the status of a tx packet capture in progress:
2497  * @cliexstart{pcap tx trace status}
2498  * max is 35, dispatch trace to file /tmp/vppTest.pcap
2499  * pcap tx capture is on: 20 of 35 pkts...
2500  * @cliexend
2501  * Example of how to stop a tx packet capture:
2502  * @cliexstart{vppctl pcap dispatch trace off}
2503  * captured 21 pkts...
2504  * saved to /tmp/dispatchTrace.pcap...
2505  * @cliexend
2506 ?*/
2507 /* *INDENT-OFF* */
2509  .path = "pcap dispatch trace",
2510  .short_help =
2511  "pcap dispatch trace [on|off] [max <nn>] [file <name>] [status]\n"
2512  " [buffer-trace <input-node-name> <nn>]",
2513  .function = dispatch_trace_command_fn,
2514 };
2515 /* *INDENT-ON* */
2516 
2517 /*
2518  * fd.io coding-style-patch-verification: ON
2519  *
2520  * Local Variables:
2521  * eval: (c-set-style "gnu")
2522  * End:
2523  */
_vlib_init_function_list_elt_t * worker_init_function_registrations
Definition: main.h:224
u32 * next_nodes
Definition: node.h:337
static void elog_enable_disable(elog_main_t *em, int is_enabled)
Enable or disable event logging.
Definition: elog.h:220
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:507
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
clib_error_t * vlib_call_all_main_loop_exit_functions(vlib_main_t *vm)
Definition: init.c:398
static vlib_frame_t * vlib_frame_alloc(vlib_main_t *vm, vlib_node_runtime_t *from_node_runtime, u32 to_next_index)
Definition: main.c:169
uword * pending_rpc_requests
Definition: main.h:275
u64 internal_node_vectors
Definition: main.h:106
vlib_main_t vlib_global_main
Definition: main.c:1999
f64 time
Absolute time as floating point number in seconds.
Definition: elog.h:67
void TW() tw_timer_wheel_init(TWT(tw_timer_wheel) *tw, void *expired_timer_callback, f64 timer_interval_in_seconds, u32 max_expirations)
Initialize a tw timer wheel template instance.
u16 vector_size
Definition: node.h:323
u32 max_clock
Maximum clock cycle for an invocation.
Definition: node.h:480
u32 next_frame_index
Start of next frames for this node.
Definition: node.h:495
static clib_error_t * show_frame_stats(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:266
#define hash_set(h, key, value)
Definition: hash.h:255
char * file_name
File name of pcap output.
Definition: pcap.h:162
vlib_node_type_t
Definition: node.h:69
#define clib_min(x, y)
Definition: clib.h:319
clib_error_t * vlib_call_all_main_loop_enter_functions(vlib_main_t *vm)
Definition: init.c:391
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
Definition: lock.h:102
static void vlib_next_frame_change_ownership(vlib_main_t *vm, vlib_node_runtime_t *node_runtime, u32 next_index)
Definition: main.c:296
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
Definition: lock.h:80
static vlib_next_frame_t * vlib_node_get_next_frame(vlib_main_t *vm, u32 node_index, u32 next_index)
Get pointer to frame by (node_index, next_index).
Definition: node_funcs.h:299
u32 interrupt_threshold_vector_length
Definition: node.h:721
vlib_process_t ** processes
Definition: node.h:742
int vl_api_get_elog_trace_api_messages(void)
Definition: main.c:2079
u32 flags
flags
Definition: pcap.h:174
static uword vlib_process_bootstrap(uword _a)
Definition: main.c:1486
format_function_t format_vlib_node_name
Definition: node_funcs.h:1141
vlib_node_runtime_t node_runtime
Definition: node.h:556
a
Definition: bitmap.h:538
u32 n_suspends
Definition: node.h:584
#define VLIB_PENDING_FRAME_NO_NEXT_FRAME
Definition: node.h:463
u32 n_packets_to_capture
Number of packets to capture.
Definition: pcap.h:165
static u32 vlib_buffer_get_trace_index(vlib_buffer_t *b)
Extract the trace (pool) index from a trace handle.
Definition: buffer.h:388
vlib_frame_t * vlib_get_next_frame_internal(vlib_main_t *vm, vlib_node_runtime_t *node, u32 next_index, u32 allocate_new_next_frame)
Definition: main.c:379
uword * processing_rpc_requests
Definition: main.h:276
static u32 clib_get_current_numa_node()
Definition: cpu.h:175
u8 * format_vnet_buffer_opaque2(u8 *s, va_list *args)
Definition: main.c:1020
#define VLIB_PCAP_MAJOR_VERSION
Definition: main.h:415
vlib_trace_node_t * nodes
Definition: trace.h:100
static void vlib_elog_main_loop_event(vlib_main_t *vm, u32 node_index, u64 time, u32 n_vectors, u32 is_return)
Definition: main.c:938
static clib_error_t * elog_save_buffer(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:740
int elog_post_mortem_dump
Definition: main.h:246
uword dispatch_pcap_enable
Definition: main.h:168
#define PREDICT_TRUE(x)
Definition: clib.h:119
uword random_seed
Definition: main.h:209
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
unsigned long u64
Definition: types.h:89
static void vlib_increment_main_loop_counter(vlib_main_t *vm)
Definition: main.h:383
f64 loop_interval_start
Definition: main.h:257
elog_time_stamp_t init_time
Timestamps.
Definition: elog.h:172
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
static void vlib_process_sync_stats(vlib_main_t *vm, vlib_process_t *p, uword n_calls, uword n_vectors, uword n_clocks, uword n_ticks0, uword n_ticks1)
Definition: main.c:597
void clib_random_buffer_init(clib_random_buffer_t *b, uword seed)
Definition: random_buffer.c:62
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
u32 index
Definition: node.h:282
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:291
static void dummy_queue_signal_callback(vlib_main_t *vm)
Definition: main.c:2046
f64 buffer_alloc_success_rate
Definition: main.h:281
u32 current_process_index
Definition: node.h:745
void vlib_register_all_static_nodes(vlib_main_t *vm)
Definition: node.c:548
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:110
static u32 clib_get_current_cpu_id()
Definition: cpu.h:167
u32 thread_index
Definition: main.h:218
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
static_always_inline uword vlib_process_startup(vlib_main_t *vm, vlib_process_t *p, vlib_frame_t *f)
Definition: main.c:1513
u32 main_loop_exit_set
Definition: main.h:127
static u64 dispatch_pending_node(vlib_main_t *vm, uword pending_frame_index, u64 last_time_stamp)
Definition: main.c:1352
static vlib_frame_t * vlib_get_frame(vlib_main_t *vm, vlib_frame_t *f)
Definition: node_funcs.h:216
static void vlib_validate_frame_indices(vlib_frame_t *f)
Definition: main.c:191
f64 loops_per_second
Definition: main.h:258
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:590
static u64 clib_cpu_time_now(void)
Definition: time.h:81
u8 * format_vnet_buffer_opaque(u8 *s, va_list *args)
Definition: main.c:1011
elog_track_t elog_track
Definition: threads.h:100
u32 clocks_since_last_overflow
Number of clock cycles.
Definition: node.h:478
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:628
uword unformat_user(unformat_input_t *input, unformat_function_t *func,...)
Definition: unformat.c:989
static void vlib_process_update_stats(vlib_main_t *vm, vlib_process_t *p, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:714
static void elog_show_buffer_internal(vlib_main_t *vm, u32 n_events_to_show)
Definition: main.c:875
#define VLIB_MAIN_LOOP_EXIT_CLI
Definition: main.h:134
void clib_longjmp(clib_longjmp_t *save, uword return_value)
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
static vlib_frame_size_t * get_frame_size_info(vlib_node_main_t *nm, u32 n_scalar_bytes, u32 n_vector_bytes)
Definition: main.c:90
static clib_error_t * elog_show_buffer(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:903
u8 * format_elog_event(u8 *s, va_list *va)
Definition: elog.c:296
clib_time_t clib_time
Definition: main.h:87
u32 numa_node
Definition: main.h:220
u8 * format_vnet_buffer_flags(u8 *s, va_list *args)
Definition: main.c:1003
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:518
clib_spinlock_t pending_interrupt_lock
Definition: node.h:715
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:402
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:252
uword clib_setjmp(clib_longjmp_t *save, uword return_value_not_taken)
#define vec_alloc(V, N)
Allocate space for N more elements (no header, unspecified alignment)
Definition: vec.h:319
unsigned char u8
Definition: types.h:56
clib_error_t * vlib_sort_init_exit_functions(_vlib_init_function_list_elt_t **head)
Topological sorter for init function chains.
Definition: init.c:76
static uword vlib_process_stack_is_valid(vlib_process_t *p)
Definition: main.c:1472
u8 buffer_pool_index
index of buffer pool this buffer belongs.
Definition: buffer.h:133
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
u64 perf_counter0_ticks
Definition: node.h:238
double f64
Definition: types.h:142
u16 scalar_size
Definition: node.h:323
u8 state
Definition: node.h:311
u16 thread_index
thread this node runs on
Definition: node.h:523
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:204
u64 internal_node_calls
Definition: main.h:107
volatile uword check_frame_queues
Definition: main.h:272
static void vlib_worker_thread_barrier_check(void)
Definition: threads.h:400
static clib_error_t * dispatch_trace_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:2370
u32 * pending_interrupt_node_runtime_indices
Definition: node.h:714
u32 input_main_loops_per_call
For input nodes: decremented on each main loop interation until it reaches zero and function is calle...
Definition: node.h:500
static void vlib_main_loop(vlib_main_t *vm)
Definition: main.c:1988
#define foreach_weak_reference_stub
Definition: main.c:2050
void(** vlib_node_runtime_perf_counter_cbs)(struct vlib_main_t *, u64 *, u64 *, vlib_node_runtime_t *, vlib_frame_t *, int)
Definition: main.h:115
vlib_trace_header_t ** trace_buffer_pool
Definition: trace.h:86
#define static_always_inline
Definition: clib.h:106
uword unformat_vlib_tmpfile(unformat_input_t *input, va_list *args)
Definition: format.c:192
PCAP main state data structure.
Definition: pcap.h:156
u32 trace_enable
Definition: trace.h:97
clib_spinlock_t lock
spinlock to protect e.g.
Definition: pcap.h:159
#define VLIB_FRAME_NO_APPEND
Definition: node.h:421
vlib_node_function_t * function
Node function to call.
Definition: node.h:470
void elog_init(elog_main_t *em, u32 n_events)
Definition: elog.c:493
#define VLIB_FRAME_MAGIC
void vl_api_set_elog_main(elog_main_t *m)
Definition: main.c:2064
u32 main_loop_vectors_processed
Definition: main.h:102
#define VLIB_INVALID_NODE_INDEX
Definition: node.h:377
u16 log2_n_stack_bytes
Definition: node.h:579
vlib_node_t ** nodes
Definition: node.h:701
u32 vectors_since_last_overflow
Number of vector elements processed by this node.
Definition: node.h:488
vlib_frame_t * frame
Definition: main.c:1481
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:182
#define clib_error_return(e, args...)
Definition: error.h:99
static void dispatch_pcap_trace(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: main.c:1051
#define VLIB_FRAME_ALIGN
Definition: node.h:381
u64 perf_counter_vectors
Definition: node.h:240
u32 cpu_id
Definition: main.h:219
static uword vlib_timing_wheel_data_is_timed_event(u32 d)
Definition: node.h:675
#define VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE
Definition: node.h:306
#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
Definition: node.h:570
#define vec_resize(V, N)
Resize a vector (no header, unspecified alignment) Add N elements to end of given vector V...
Definition: vec.h:281
unsigned int u32
Definition: types.h:88
u64 cpu_time_main_loop_start
Definition: main.h:96
vlib_node_runtime_t * nodes_by_type[VLIB_N_NODE_TYPE]
Definition: node.h:711
static u32 vlib_frame_bytes(u32 n_scalar_bytes, u32 n_vector_bytes)
Definition: main.c:53
#define VLIB_MAIN_LOOP_EXIT_NONE
Definition: main.h:131
#define VLIB_FRAME_SIZE
Definition: node.h:380
static vlib_next_frame_t * vlib_node_runtime_get_next_frame(vlib_main_t *vm, vlib_node_runtime_t *n, u32 next_index)
Definition: node_funcs.h:264
u32 perf_counter0_ticks_since_last_overflow
Perf counter 0 ticks.
Definition: node.h:491
u8 * pcap_buffer
Definition: main.h:170
#define VLIB_PROCESS_IS_RUNNING
Definition: node.h:576
#define VLIB_PROCESS_RETURN_LONGJMP_RETURN
Definition: node.h:561
f64 seconds_per_loop
Definition: main.h:259
u64 max_clock_n
Definition: node.h:237
unformat_function_t unformat_line_input
Definition: format.h:283
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:63
vl_api_fib_path_type_t type
Definition: fib_types.api:123
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:136
u32 calls_since_last_overflow
Number of calls.
Definition: node.h:486
char * name
Definition: main.h:140
clib_error_t * pcap_close(pcap_main_t *pm)
Close PCAP file.
Definition: pcap.c:74
vlib_worker_thread_t * vlib_worker_threads
Definition: threads.c:34
#define hash_get(h, key)
Definition: hash.h:249
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:534
static clib_error_t * elog_write_file(elog_main_t *em, char *clib_file, int flush_ring)
Definition: elog.h:532
u32 next_frame_index
Definition: node.h:460
static void * vlib_process_signal_event_helper(vlib_node_main_t *nm, vlib_node_t *n, vlib_process_t *p, uword t, uword n_data_elts, uword n_data_elt_bytes)
Definition: node_funcs.h:743
vlib_node_stats_t stats_total
Definition: node.h:272
static void vlib_put_next_frame_validate(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 next_index, u32 n_vectors_left)
Definition: main.c:444
u16 state
Input node state.
Definition: node.h:513
static u8 * format_buffer_metadata(u8 *s, va_list *args)
Definition: main.c:1027
format_function_t format_vlib_trace
Definition: trace.h:113
vlib_signal_timed_event_data_t * signal_timed_event_data_pool
Definition: node.h:732
u16 frame_flags
Definition: node.h:387
#define VLIB_FRAME_IS_ALLOCATED
Definition: node.h:428
#define VLIB_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:417
u8 * pcap_data
Vector of pcap data.
Definition: pcap.h:184
struct _unformat_input_t unformat_input_t
static_always_inline u64 dispatch_node(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_node_type_t type, vlib_node_state_t dispatch_state, vlib_frame_t *frame, u64 last_time_stamp)
Definition: main.c:1159
u32 n_alloc_frames
Definition: node.h:541
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:216
vec_header_t h
Definition: buffer.c:322
static uword vlib_timing_wheel_data_get_index(u32 d)
Definition: node.h:693
u32 *TW() tw_timer_expire_timers_vec(TWT(tw_timer_wheel) *tw, f64 now, u32 *vec)
void di(unformat_input_t *i)
Definition: unformat.c:163
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:302
elog_event_type_t * node_return_elog_event_types
Definition: main.h:204
#define TWT(a)
void vlib_frame_free(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_frame_t *f)
Definition: main.c:238
u32 polling_threshold_vector_length
Definition: node.h:720
#define ELOG_DATA(em, f)
Definition: elog.h:484
u64 * n_vectors_by_next_node
Definition: node.h:346
u32 trace_handle
Specifies trace buffer handle if VLIB_PACKET_IS_TRACED flag is set.
Definition: buffer.h:163
#define PREDICT_FALSE(x)
Definition: clib.h:118
#define always_inline
Definition: ipsec.h:28
f64 seconds_per_clock
Definition: time.h:58
void elog_post_mortem_dump(void)
Definition: main.c:777
u8 protocol_hint
Definition: node.h:317
vlib_frame_t * frame
Definition: node.h:457
u32 node_index
Node index.
Definition: node.h:498
#define foreach_vlib_main(body)
Definition: threads.h:241
uword * init_functions_called
Definition: main.h:215
static void * pcap_add_packet(pcap_main_t *pm, f64 time_now, u32 n_bytes_in_trace, u32 n_bytes_in_packet)
Add packet.
Definition: pcap_funcs.h:41
#define VLIB_PROCESS_STACK_MAGIC
Definition: node.h:627
#define VLIB_PROCESS_RESUME_LONGJMP_RESUME
Definition: node.h:567
uword * frame_size_hash
Definition: node.h:757
void elog_alloc(elog_main_t *em, u32 n_events)
Definition: elog.c:479
vlib_main_t * vm
Definition: in2out_ed.c:1599
clib_error_t * vlibmemory_init(vlib_main_t *vm)
Definition: memory_api.c:933
volatile u32 queue_signal_pending
Definition: main.h:231
static vlib_frame_t * vlib_frame_alloc_to_node(vlib_main_t *vm, u32 to_node_index, u32 frame_flags)
Definition: main.c:116
clib_error_t * map_api_segment_init(vlib_main_t *vm)
Definition: memory_api.c:476
f64 damping_constant
Definition: main.h:260
u8 * name
Definition: node.h:266
static u64 dispatch_process(vlib_main_t *vm, vlib_process_t *p, vlib_frame_t *f, u64 last_time_stamp)
Definition: main.c:1544
static u32 vlib_node_runtime_update_stats(vlib_main_t *vm, vlib_node_runtime_t *node, uword n_calls, uword n_vectors, uword n_clocks, uword n_ticks0, uword n_ticks1)
Definition: main.c:648
static_always_inline void os_set_numa_index(uword numa_index)
Definition: os.h:81
u32 owner_node_index
Definition: node.h:357
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
Definition: init.h:226
static u32 vlib_node_runtime_update_main_loop_vector_stats(vlib_main_t *vm, vlib_node_runtime_t *node, uword n_vectors)
Definition: node_funcs.h:1029
#define UNFORMAT_END_OF_INPUT
Definition: format.h:145
u32 flags
Definition: vhost_user.h:248
u8 inline_event_data[64 - 3 *sizeof(u32) - 2 *sizeof(u16)]
Definition: node.h:666
svmdb_client_t * c
u16 n_vectors
Definition: node.h:399
u32 runtime_index
Definition: node.h:285
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:80
int elog_trace_graph_dispatch
Definition: main.h:198
static uword elog_n_events_in_buffer(elog_main_t *em)
Return number of events in the event-log buffer.
Definition: elog.h:191
static_always_inline void vlib_main_or_worker_loop(vlib_main_t *vm, int is_main)
Definition: main.c:1707
u32 node_runtime_index
Definition: node.h:454
vlib_pending_frame_t * pending_frames
Definition: node.h:727
u32 flow_id
Generic flow identifier.
Definition: buffer.h:127
int vlib_frame_queue_dequeue(vlib_main_t *vm, vlib_frame_queue_main_t *fqm)
Definition: threads.c:1686
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:317
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:380
clib_error_t * vpe_api_init(vlib_main_t *vm)
Definition: api.c:656
#define VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:295
#define clib_warning(format, args...)
Definition: error.h:59
#define VLIB_PROCESS_RESUME_PENDING
Definition: node.h:573
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
Definition: node_funcs.h:89
u8 data[]
Packet data.
Definition: buffer.h:181
u32 perf_counter_vectors_since_last_overflow
Perf counter vectors.
Definition: node.h:493
#define VLIB_BUFFER_TRACE_TRAJECTORY
Compile time buffer trajectory tracing option Turn this on if you run into "bad monkey" contexts...
Definition: buffer.h:483
elog_main_t elog_main
Definition: main.h:193
clib_error_t * vlib_buffer_main_init(struct vlib_main_t *vm)
Definition: buffer.c:812
#define VLIB_FRAME_PENDING
Definition: node.h:431
static u32 vlib_frame_vector_byte_offset(u32 scalar_size)
Definition: node_funcs.h:233
u32 current_config_index
Used by feature subgraph arcs to visit enabled feature nodes.
Definition: buffer.h:147
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:483
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:442
clib_error_t * pcap_write(pcap_main_t *pm)
Write PCAP file.
Definition: pcap.c:89
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:256
u32 n_total_events_disable_limit
When count reaches limit logging is disabled.
Definition: elog.h:139
vlib_frame_t ** free_frames
Definition: node.h:544
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1599
u8 exp
Definition: fib_types.api:27
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:152
#define VLIB_PCAP_MINOR_VERSION
Definition: main.h:416
#define never_inline
Definition: clib.h:102
signed int i32
Definition: types.h:77
#define hash_create(elts, value_bytes)
Definition: hash.h:696
#define uword_to_pointer(u, type)
Definition: types.h:136
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
Definition: node.h:517
clib_error_t * vlib_node_main_init(vlib_main_t *vm)
Definition: node.c:639
#define PCAP_MAIN_INIT_DONE
Definition: pcap.h:175
#define pool_put_index(p, i)
Free pool element with given index.
Definition: pool.h:331
#define ASSERT(truth)
u64 last_cpu_time
Definition: time.h:51
never_inline void vlib_node_runtime_sync_stats(vlib_main_t *vm, vlib_node_runtime_t *r, uword n_calls, uword n_vectors, uword n_clocks, uword n_ticks0, uword n_ticks1)
Definition: main.c:569
static void vlib_node_runtime_perf_counter(vlib_main_t *vm, u64 *pmc0, u64 *pmc1, vlib_node_runtime_t *node, vlib_frame_t *frame, int before_or_after)
Definition: main.c:702
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:689
static void elog_reset_buffer(elog_main_t *em)
Reset the event buffer.
Definition: elog.h:210
u64 perf_counter1_ticks
Definition: node.h:239
vlib_frame_queue_main_t * frame_queue_mains
Definition: threads.h:332
u8 data[128]
Definition: ipsec_types.api:89
#define ELOG_TRACK_DATA(em, f, track)
Definition: elog.h:478
vlib_frame_t * frame
Definition: node.h:408
uword event_ring_size
Power of 2 number of elements in ring.
Definition: elog.h:145
u16 flags
Definition: node.h:390
u32 main_loop_nodes_processed
Definition: main.h:103
u64 cpu
CPU cycle counter.
Definition: elog.h:126
u64 loops_this_reporting_interval
Definition: main.h:255
void vlib_worker_thread_initial_barrier_sync_and_release(vlib_main_t *vm)
Definition: threads.c:1413
#define clib_error_report(e)
Definition: error.h:113
u32 elog_trace_graph_circuit_node_index
Definition: main.h:200
vlib_trace_main_t trace_main
Definition: main.h:164
int vl_api_set_elog_trace_api_messages(int enable)
Definition: main.c:2071
u8 * vnet_trace_dummy
Definition: trace.c:43
static uword pointer_to_uword(const void *p)
Definition: types.h:131
void(**volatile worker_thread_main_loop_callbacks)(struct vlib_main_t *)
Definition: main.h:237
#define A(x)
Definition: main.c:1048
void(* queue_signal_callback)(struct vlib_main_t *)
Definition: main.h:233
void vlib_gdb_show_event_log(void)
Definition: main.c:932
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
static_always_inline uword vlib_process_resume(vlib_process_t *p)
Definition: main.c:1531
#define VLIB_FRAME_OWNER
Definition: node.h:425
void vlib_node_sync_stats(vlib_main_t *vm, vlib_node_t *n)
Definition: main.c:611
#define vec_elt(v, i)
Get vector value at index i.
int vlib_main(vlib_main_t *volatile vm, unformat_input_t *input)
Definition: main.c:2087
#define unformat_parse_error(input)
Definition: format.h:269
typedef key
Definition: ipsec_types.api:85
pcap_packet_type_t packet_type
Packet type.
Definition: pcap.h:168
void vlib_worker_loop(vlib_main_t *vm)
Definition: main.c:1994
static vlib_node_t * vlib_get_next_node(vlib_main_t *vm, u32 node_index, u32 next_index)
Get vlib node by graph arc (next) index.
Definition: node_funcs.h:72
void vl_api_send_pending_rpc_requests(vlib_main_t *)
Definition: main.c:1701
#define clib_mem_alloc_aligned_no_fail(size, align)
Definition: mem.h:191
u32 buffer_alloc_success_seed
Definition: main.h:280
u32 * data_from_advancing_timing_wheel
Definition: node.h:735
clib_error_t * vlib_physmem_init(vlib_main_t *vm)
Definition: physmem.c:94
u32 perf_counter1_ticks_since_last_overflow
Perf counter 1 ticks.
Definition: node.h:492
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u32 input_node_counts_by_state[VLIB_N_NODE_STATE]
Definition: node.h:754
vlib_pending_frame_t * suspended_process_frames
Definition: node.h:748
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:140
vlib_node_main_t node_main
Definition: main.h:158
static u32 * vlib_frame_find_magic(vlib_frame_t *f, vlib_node_t *node)
Definition: main.c:78
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
u32 owner_next_index
Definition: node.h:357
u8 vector_size
Definition: node.h:396
vlib_next_frame_t * next_frames
Definition: node.h:724
static clib_error_t * elog_stop(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:803
static void unformat_free(unformat_input_t *i)
Definition: format.h:163
clib_error_t * vlib_call_all_config_functions(vlib_main_t *vm, unformat_input_t *input, int is_early)
Definition: init.c:405
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
static uword elog_is_enabled(elog_main_t *em)
event logging enabled predicate
Definition: elog.h:276
static f64 clib_time_now_internal(clib_time_t *c, u64 n)
Definition: time.h:210
#define VLIB_FRAME_TRACE
Definition: node.h:437
u32 stop_timer_handle
Definition: node.h:609
clib_error_t * vlib_call_all_init_functions(vlib_main_t *vm)
Definition: init.c:378
#define VLIB_FRAME_SIZE_EXTRA
unformat_function_t unformat_vlib_node
Definition: node_funcs.h:1147
static void elog_track(elog_main_t *em, elog_event_type_t *type, elog_track_t *track, u32 data)
Log a single-datum event to a specific track, non-inline version.
Definition: elog.h:398
static u64 dispatch_suspended_process(vlib_main_t *vm, uword process_index, u64 last_time_stamp)
Definition: main.c:1625
vlib_frame_size_t * frame_sizes
Definition: node.h:760
u64 resume_clock_interval
Definition: node.h:606
static void add_trajectory_trace(vlib_buffer_t *b, u32 node_index)
Definition: main.c:991
u16 flags
Definition: node.h:569
elog_event_type_t * node_call_elog_event_types
Definition: main.h:203
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:165
vlib_node_type_t type
Definition: node.h:279
static vlib_process_t * vlib_get_process_from_node(vlib_main_t *vm, vlib_node_t *node)
Definition: node_funcs.h:208
uword clib_calljmp(uword(*func)(uword func_arg), uword func_arg, void *stack)
u8 * format_unformat_error(u8 *s, va_list *va)
Definition: unformat.c:91
u32 name_elog_string
Definition: node.h:269
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1540
#define clib_call_callbacks(h,...)
call the specified callback set
Definition: callback.h:67
vlib_node_state_t
Definition: node.h:252
pcap_main_t dispatch_pcap_main
Definition: main.h:167
u64 init_cpu_time
Definition: time.h:61
u32 TW() tw_timer_start(TWT(tw_timer_wheel) *tw, u32 user_id, u32 timer_id, u64 interval)
Start a Tw Timer.
u32 suspended_process_frame_index
Definition: node.h:581
static void validate_frame_magic(vlib_main_t *vm, vlib_frame_t *f, vlib_node_t *n, uword next_index)
Definition: main.c:370
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
f64 loop_interval_end
Definition: main.h:256
static clib_error_t * vlib_main_configure(vlib_main_t *vm, unformat_input_t *input)
Definition: main.c:2002
static vlib_cli_command_t pcap_dispatch_trace_command
(constructor) VLIB_CLI_COMMAND (pcap_dispatch_trace_command)
Definition: main.c:2508
u32 internal_node_last_vectors_per_main_loop
Definition: main.h:112
static vlib_node_t * vlib_get_node(vlib_main_t *vm, u32 i)
Get vlib node by index.
Definition: node_funcs.h:59
static clib_error_t * vlib_cli_elog_clear(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:723
#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT
Definition: node.h:571
#define vec_foreach(var, vec)
Vector iterator.
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
Definition: in2out_ed.c:1600
elog_track_t default_track
Default track.
Definition: elog.h:166
clib_longjmp_t return_longjmp
Definition: node.h:559
u16 flags
Copy of main node flags.
Definition: node.h:511
u32 node_runtime_index
Definition: node.h:411
u32 n_total_events
Total number of events in buffer.
Definition: elog.h:135
u8 scalar_size
Definition: node.h:393
clib_longjmp_t main_loop_exit
Definition: main.h:130
clib_longjmp_t resume_longjmp
Definition: node.h:565
void clib_mem_trace(int enable)
Definition: mem_dlmalloc.c:454
static clib_error_t * elog_resize(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:843
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:280
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:304
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:167
#define TW(a)
clib_error_t * vlib_map_stat_segment_init(void)
Definition: stat_segment.c:366
static u32 vlib_timing_wheel_data_set_suspended_process(u32 i)
Definition: node.h:681
void vlib_start_process(vlib_main_t *vm, uword process_index)
Definition: main.c:1617
int vlib_pcap_dispatch_trace_configure(vlib_pcap_dispatch_trace_args_t *a)
Definition: main.c:2254
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:95
void * timing_wheel
Definition: node.h:730
volatile u8 ref_count
Reference count for this buffer.
Definition: buffer.h:130
int elog_trace_graph_circuit
Definition: main.h:199
u32 n_packets_captured
Number of packets currently captured.
Definition: pcap.h:171
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
clib_random_buffer_t random_buffer
Definition: main.h:212
static clib_error_t * elog_restart(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:823
#define VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE
Definition: node.h:307
u8 *(* validate_frame)(struct vlib_main_t *vm, struct vlib_node_runtime_t *, struct vlib_frame_t *f)
Definition: node.h:367
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:978
u32 * dispatch_buffer_trace_nodes
Definition: main.h:169
clib_error_t * main_loop_error
Definition: main.h:137
vlib_process_t * process
Definition: main.c:1480
elog_event_t * elog_peek_events(elog_main_t *em)
convert event ring events to events, and return them as a vector.
Definition: elog.c:537
static uword elog_buffer_capacity(elog_main_t *em)
Return number of events which can fit in the event buffer.
Definition: elog.h:201
u64 cpu_time_last_node_dispatch
Definition: main.h:93
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:171
#define VLIB_FRAME_FREE_AFTER_DISPATCH
Definition: node.h:434
clib_error_t * vlib_thread_init(vlib_main_t *vm)
Definition: threads.c:207
u32 vectors_since_last_overflow
Definition: node.h:440
u32 max_clock_n
Number of vectors in the recorded max_clock.
Definition: node.h:483
#define VLIB_PROCESS_RETURN_LONGJMP_SUSPEND
Definition: node.h:562