53 .format =
"bar-trace-%s-#%d",
54 .format_args =
"T4i4",
59 u32 caller,
count, t_entry, t_open, t_closed;
63 ed->count = (int) vlib_worker_threads[0].barrier_sync_count;
66 ed->t_entry = (int) (1000000.0 * t_entry);
67 ed->t_open = (int) (1000000.0 * t_open);
68 ed->t_closed = (int) (1000000.0 * t_closed);
80 .format =
"bar-syncrec-%s-#%d",
81 .format_args =
"T4i4",
90 ed->depth = (int) vlib_worker_threads[0].recursion_level - 1;
104 .format =
"bar-relrrec-#%d",
114 ed->depth = (int) vlib_worker_threads[0].recursion_level;
126 .format =
"bar-rel-#%d-e%d-u%d-t%d",
127 .format_args =
"i4i4i4i4",
132 u32 count, t_entry, t_update_main, t_closed_total;
136 ed->t_entry = (int) (1000000.0 * t_entry);
137 ed->t_update_main = (int) (1000000.0 * t_update_main);
138 ed->t_closed_total = (int) (1000000.0 * t_closed_total);
139 ed->count = (int) vlib_worker_threads[0].barrier_sync_count;
154 int pthread_setname_np (pthread_t __target_thread,
const char *__name);
156 pthread_t
thread = pthread_self ();
160 rv = pthread_setname_np (thread, name);
172 return ((
i32) ((*tr0)->no_data_structure_clone)
173 - ((
i32) ((*tr1)->no_data_structure_clone)));
182 fp = fopen (filename,
"r");
188 if (fgets ((
char *) buffer, 256, fp))
192 strlen ((
char *) buffer));
193 if (
unformat (&in,
"%U", unformat_bitmap_list, &r) != 1)
212 u32 n_vlib_mains = 1;
269 pthread_setaffinity_np (pthread_self (),
sizeof (cpu_set_t), &cpuset);
274 _vec_len (vlib_worker_threads) = 1;
287 struct sched_param sched_param;
288 if (!sched_getparam (w->
lwp, &sched_param))
312 first_index += tr->
count;
324 if (clib_bitmap_get(avail_cpu, c) == 0)
325 return clib_error_return (0,
"cpu %u is not available to be used" 326 " for the '%s' thread",c, tr->name);
328 avail_cpu = clib_bitmap_set(avail_cpu, c, 0);
334 for (j = 0; j < tr->
count; j++)
342 if (c == ~0 && avail_c0)
350 "no available cpus to be used for" 351 " the '%s' thread", tr->
name);
389 fformat (stderr,
"WARNING: fq->tail unaligned\n");
391 fformat (stderr,
"WARNING: fq->head unaligned\n");
393 fformat (stderr,
"WARNING: fq->elts unaligned\n");
396 fformat (stderr,
"WARNING: fq->elts[0] size %d\n",
397 sizeof (fq->
elts[0]));
398 if (nelts & (nelts - 1))
400 fformat (stderr,
"FATAL: nelts MUST be a power of 2\n");
417 vlib_frame_queue_dequeue_internal (
int thread_id,
425 u32 node_runtime_index;
445 node_runtime_index = elt->node_runtime_index;
450 case VLIB_FRAME_QUEUE_ELT_FREE_BUFFERS:
453 case VLIB_FRAME_QUEUE_ELT_FREE_FRAME:
462 p->frame_index = vlib_frame_index (vm, f);
466 case VLIB_FRAME_QUEUE_ELT_API_MSG:
470 clib_warning (
"bogus frame queue message, type %d", msg_type);
488 return vlib_frame_queue_dequeue_internal (thread_id, vm, nm);
515 elt = fq->
elts + (new_tail & (fq->
nelts - 1));
526 elt->node_runtime_index = node_runtime_index;
594 const char *sys_cpu_path =
"/sys/devices/system/cpu/cpu";
595 const char *sys_node_path =
"/sys/devices/system/node/node";
599 int core_id = -1, numa_id = -1;
601 p =
format (p,
"%s%u/topology/core_id%c", sys_cpu_path, cpu_id, 0);
607 unformat_bitmap_list, &nbmp);
609 p =
format (p,
"%s%u/cpulist%c", sys_node_path, node, 0);
629 void *(*fp_arg) (
void *) = fp;
659 CPU_SET (cpu_id, &cpuset);
661 if (pthread_create (&worker, NULL , fp_arg, (
void *) w))
664 if (pthread_setaffinity_np (worker,
sizeof (cpu_set_t), &cpuset))
682 u32 worker_thread_index;
702 if (n_vlib_mains > 1)
735 worker_thread_index = 1;
747 for (k = 0; k < tr->
count; k++)
751 vec_add2 (vlib_worker_threads, w, 1);
767 (
char *)
format (0,
"%s %d", tr->
name, k + 1);
801 u32 save_node_runtime_index;
808 nf->
flags = save_flags;
882 #ifdef VLIB_SUPPORTS_ARBITRARY_SCALAR_SIZES 897 worker_thread_index, oldheap, 1);
900 (
vlib_mains[0]->error_main.counters_last_clear,
903 worker_thread_index++;
914 for (j = 0; j < tr->
count; j++)
916 vec_add2 (vlib_worker_threads, w, 1);
930 (
char *)
format (0,
"%s %d", tr->
name, j + 1);
938 worker_thread_index = 1;
949 for (j = 0; j < tr->
count; j++)
951 w = vlib_worker_threads + worker_thread_index++;
963 w = vlib_worker_threads + worker_thread_index++;
964 err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn,
967 clib_error_report (err);
1021 n = nm_clone->
nodes[j];
1075 u32 save_node_runtime_index;
1082 nf->
flags = save_flags;
1085 old_nodes_clone = nm_clone->
nodes;
1086 nm_clone->
nodes = 0;
1101 if (j >= vec_len (old_nodes_clone))
1149 for (j = 0; j <
vec_len (old_rt); j++)
1176 for (j = 0; j <
vec_len (old_rt); j++)
1203 for (j = 0; j <
vec_len (old_rt); j++)
1231 u32 *r = va_arg (*args,
u32 *);
1234 #define _(v,f,s) else if (unformat (input, s)) *r = SCHED_POLICY_##f; 1269 if (
unformat (input,
"use-pthreads"))
1277 else if (
unformat (input,
"numa-heap-size %U",
1280 else if (
unformat (input,
"coremask-%s %U", &name,
1281 unformat_bitmap_mask, &bitmap) ||
1282 unformat (input,
"corelist-%s %U", &name,
1283 unformat_bitmap_list, &bitmap))
1293 "corelist cannot be set for '%s' threads",
1297 (0,
"core placement of '%s' threads is already configured",
1310 else if (
unformat (input,
"%s %u", &name, &count))
1320 (0,
"number of '%s' threads not configurable", name);
1323 (0,
"number of '%s' threads is already configured", name);
1346 "scheduling priority (%d) is not allowed for `normal` scheduling policy",
1404 #ifndef BARRIER_MINIMUM_OPEN_LIMIT 1405 #define BARRIER_MINIMUM_OPEN_LIMIT 0.001 1408 #ifndef BARRIER_MINIMUM_OPEN_FACTOR 1409 #define BARRIER_MINIMUM_OPEN_FACTOR 3 1429 fformat (stderr,
"%s: worker thread deadlock\n", __FUNCTION__);
1457 f64 max_vector_rate;
1474 if (++vlib_worker_threads[0].recursion_level > 1)
1484 max_vector_rate = 0.0;
1499 if (max_vector_rate > 10.0)
1511 (
"clock change: would have waited for %.4f seconds",
1528 fformat (stderr,
"%s: worker thread deadlock\n", __FUNCTION__);
1547 f64 t_update_main = 0.0;
1548 int refork_needed = 0;
1559 if (--vlib_worker_threads[0].recursion_level > 0)
1603 fformat (stderr,
"%s: worker thread deadlock\n", __FUNCTION__);
1619 fformat (stderr,
"%s: worker thread refork deadlock\n",
1670 for (ii = 1; ii <
vec_len (counts); ii++)
1672 while (counts[ii] ==
vlib_mains[ii]->main_loop_count)
1731 for (elix = 0; elix < fqt->
nelts; elix++)
1734 if (1 || elt->
valid)
1769 if (b->
flags & VLIB_BUFFER_IS_TRACED)
1776 while (n_left_to_node >= 4)
1784 n_left_to_node -= 4;
1787 while (n_left_to_node > 0)
1862 if (frame_queue_nelts == 0)
1866 ASSERT (frame_queue_nelts >= 8 + num_threads);
1924 (*fp) (callback, args, arg_size);
1927 clib_warning (
"BUG: rpc_call_main_thread_cb_fn NULL!");
1947 (void)
unformat (input,
"verbose %=", &verbose, 1);
1981 .path =
"show clock",
1982 .short_help =
"show clock",
_vlib_init_function_list_elt_t * worker_init_function_registrations
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
volatile u32 main_loop_count
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
static void barrier_trace_release_rec(f64 t_entry)
static void barrier_trace_sync(f64 t_entry, f64 t_open, f64 t_closed)
uword * pending_rpc_requests
vlib_main_t vlib_global_main
#define vec_foreach_index(var, v)
Iterate over vector indices.
void vlib_worker_wait_one_loop(void)
Wait until each of the workers has been once around the track.
void vlib_stats_pop_heap2(u64 *error_vector, u32 thread_index, void *oldheap, int lock)
clib_spinlock_t pending_rpc_lock
vlib_process_t ** processes
#define VLIB_PENDING_FRAME_NO_NEXT_FRAME
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
u8 runtime_data[0]
Function dependent node-runtime data.
int vlib_frame_queue_enqueue(vlib_main_t *vm, u32 node_runtime_index, u32 frame_queue_index, vlib_frame_t *frame, vlib_frame_queue_msg_type_t type)
#define clib_atomic_add_fetch(a, b)
#define VLIB_MAIN_LOOP_ENTER_FUNCTION(x)
word elog_track_register(elog_main_t *em, elog_track_t *t)
register an event track
#define CLIB_MEMORY_STORE_BARRIER()
#define clib_memcpy_fast(a, b, c)
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
static f64 vlib_time_now(vlib_main_t *vm)
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
clib_error_t * threads_init(vlib_main_t *vm)
format_function_t format_clib_time
u32 vlib_frame_queue_main_init(u32 node_index, u32 frame_queue_nelts)
void * thread_function_arg
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
static int sort_registrations_by_no_clone(void *a0, void *a1)
static u64 clib_cpu_time_now(void)
frame_queue_trace_t * frame_queue_traces
void vlib_process_signal_event_mt_helper(vlib_process_signal_event_mt_args_t *args)
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
void vnet_main_fixup(vlib_fork_fixup_t which)
void * vlib_stats_push_heap(void *old)
static uword * clib_bitmap_set(uword *ai, uword i, uword value)
Sets the ith bit of a bitmap to new_value Removes trailing zeros from the bitmap. ...
#define hash_set_mem(h, key, value)
void vlib_worker_thread_fn(void *arg)
u32 unformat_sched_policy(unformat_input_t *input, va_list *args)
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
struct vlib_thread_registration_ * next
u32 buffer_index[VLIB_FRAME_SIZE]
void vlib_worker_thread_barrier_sync_int(vlib_main_t *vm, const char *func_name)
static u32 vlib_last_vectors_per_main_loop(vlib_main_t *vm)
vlib_main_t ** vlib_mains
#define clib_bitmap_dup(v)
Duplicate a bitmap.
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
u16 thread_index
thread this node runs on
#define vlib_worker_thread_barrier_sync(X)
#define clib_memcpy(d, s, n)
u64 * counters_last_clear
static void vlib_worker_thread_barrier_check(void)
vlib_thread_registration_t * next
#define vec_add1_aligned(V, E, A)
Add 1 element to end of vector (alignment specified).
#define VLIB_NODE_RUNTIME_DATA_SIZE
vlib_node_stats_t stats_last_clear
void vlib_worker_thread_node_runtime_update(void)
u64 count[FRAME_QUEUE_MAX_NELTS]
#define VLIB_INIT_FUNCTION(x)
#define VLIB_INVALID_NODE_INDEX
#define vec_new(T, N)
Create new vector of given type and length (unspecified alignment, no header).
static f64 clib_timebase_now(clib_timebase_t *tb)
vlib_frame_queue_msg_type_t
vlib_frame_queue_elt_t ** handoff_queue_elt_by_thread_index
clib_error_t * vlib_call_init_exit_functions_no_sort(vlib_main_t *vm, _vlib_init_function_list_elt_t **headp, int call_once)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
#define clib_error_return(e, args...)
uword * lock
SMP lock, non-zero means locking required.
#define BARRIER_MINIMUM_OPEN_FACTOR
vlib_frame_queue_elt_t * elts
pthread_t thread[MAX_CONNS]
f64 time_last_barrier_release
void * clib_per_numa_mheaps[CLIB_MAX_NUMAS]
vlib_node_runtime_t * nodes_by_type[VLIB_N_NODE_TYPE]
void vlib_set_thread_name(char *name)
void vl_msg_api_handler_no_free(void *)
format_function_t format_clib_timebase_time
#define hash_create_string(elts, value_bytes)
#define BARRIER_SYNC_TIMEOUT
static void clib_spinlock_init(clib_spinlock_t *p)
void * rpc_call_main_thread_cb_fn
vl_api_fib_path_type_t type
VLIB_REGISTER_THREAD(worker_thread_reg, static)
vlib_worker_thread_t * vlib_worker_threads
#define clib_bitmap_foreach(i, ai, body)
Macro to iterate across set bits in a bitmap.
vlib_node_stats_t stats_total
u16 state
Input node state.
static void vlib_process_signal_event(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
static uword clib_bitmap_first_set(uword *ai)
Return the lowest numbered set bit in a bitmap.
u8 * vlib_thread_stack_init(uword thread_index)
static void vlib_next_frame_init(vlib_next_frame_t *nf)
vlib_error_main_t error_main
vlib_thread_callbacks_t cb
#define VLIB_FRAME_NO_FREE_AFTER_DISPATCH
int vlib_thread_cb_register(struct vlib_main_t *vm, vlib_thread_callbacks_t *cb)
void vlib_stat_segment_unlock(void)
u8 vlib_worker_thread_barrier_held(void)
Return true if the wroker thread barrier is held.
const char * barrier_context
char * name
Track name vector.
#define clib_error_return_unix(e, args...)
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
static void * clib_mem_get_per_cpu_heap(void)
void vlib_frame_free(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_frame_t *f)
static clib_error_t * vlib_launch_thread_int(void *fp, vlib_worker_thread_t *w, unsigned cpu_id)
void vlib_worker_thread_node_refork(void)
clib_error_t *(* vlib_thread_set_lcore_cb)(u32 thread, u16 cpu)
u32 node_index
Node index.
uword * init_functions_called
void clib_time_init(clib_time_t *c)
vlib_thread_main_t vlib_thread_main
clib_error_t * clib_sysfs_read(char *file_name, char *fmt,...)
void(* thread_function)(void *)
static clib_error_t * cpu_config(vlib_main_t *vm, unformat_input_t *input)
i32 n_vectors[FRAME_QUEUE_MAX_NELTS]
vlib_frame_queue_t * vlib_frame_queue_alloc(int nelts)
#define clib_mem_alloc_no_fail(size)
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
vlib_frame_queue_t ** vlib_frame_queues
DLMALLOC_EXPORT mspace create_mspace(size_t capacity, int locked)
static_always_inline uword vlib_get_thread_index(void)
vlib_pending_frame_t * pending_frames
vlib_thread_function_t * function
int vlib_frame_queue_dequeue(vlib_main_t *vm, vlib_frame_queue_main_t *fqm)
sll srl srl sll sra u16x4 i
#define vec_free(V)
Free vector's memory (no header).
static void * clib_mem_set_heap(void *heap)
#define clib_warning(format, args...)
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
void vlib_stat_segment_lock(void)
frame_queue_nelt_counter_t * frame_queue_histogram
#define VLIB_FRAME_PENDING
static uword clib_bitmap_get(uword *ai, uword i)
Gets the ith bit value from a bitmap.
#define ELOG_TYPE_DECLARE(f)
vlib_main_t vlib_node_runtime_t * node
void vlib_worker_thread_init(vlib_worker_thread_t *w)
uword os_get_nthreads(void)
static void * clib_mem_get_heap(void)
volatile u32 * wait_at_barrier
#define VLIB_CLI_COMMAND(x,...)
vlib_frame_queue_per_thread_data_t * per_thread_data
vlib_frame_queue_t ** congested_handoff_queue_by_thread_index
#define hash_create(elts, value_bytes)
never_inline void vlib_node_runtime_sync_stats(vlib_main_t *vm, vlib_node_runtime_t *r, uword n_calls, uword n_vectors, uword n_clocks, uword n_ticks0, uword n_ticks1)
static clib_error_t * show_clock_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
static void barrier_trace_sync_rec(f64 t_entry)
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
vlib_frame_queue_main_t * frame_queue_mains
void clib_timebase_init(clib_timebase_t *tb, i32 timezone_offset_in_hours, clib_timebase_daylight_time_t daylight_type, clib_time_t *clib_time)
static void clib_mem_free(void *p)
void vlib_worker_thread_initial_barrier_sync_and_release(vlib_main_t *vm)
#define clib_error_report(e)
#define clib_bitmap_free(v)
Free a bitmap.
int need_vlib_worker_thread_node_runtime_update
uword * thread_registrations_by_name
#define BARRIER_MINIMUM_OPEN_LIMIT
clib_error_t *(* vlib_launch_thread_cb)(void *fp, vlib_worker_thread_t *w, unsigned cpu_id)
volatile u32 * node_reforks_required
const char * barrier_caller
static vlib_main_t * vlib_get_main(void)
#define clib_atomic_fetch_add(a, b)
void vlib_node_sync_stats(vlib_main_t *vm, vlib_node_t *n)
void vlib_get_thread_core_numa(vlib_worker_thread_t *w, unsigned cpu_id)
static uword clib_bitmap_count_set_bits(uword *ai)
Return the number of set bits in a bitmap.
static void barrier_trace_release(f64 t_entry, f64 t_closed_total, f64 t_update_main)
void vlib_worker_loop(vlib_main_t *vm)
#define vec_dup_aligned(V, A)
Return copy of vector (no header, alignment specified).
u32 elog_string(elog_main_t *em, char *fmt,...)
add a string to the event-log string table
f64 barrier_no_close_before
static clib_error_t * start_workers(vlib_main_t *vm)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
void vlib_rpc_call_main_thread(void *callback, u8 *args, u32 arg_size)
vlib_node_main_t node_main
VLIB buffer representation.
vlib_next_frame_t * next_frames
#define vec_sort_with_function(vec, f)
Sort a vector using the supplied element comparison function.
int no_data_structure_clone
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
void * clib_mem_init_thread_safe_numa(void *memory, uword memory_size, u8 numa)
#define VLIB_THREAD_STACK_SIZE
vlib_frame_size_t * frame_sizes
#define hash_get_mem(h, key)
static void worker_thread_node_runtime_update_internal(void)
static void * clib_mem_alloc_aligned(uword size, uword align)
volatile u32 * workers_at_barrier
uword clib_calljmp(uword(*func)(uword func_arg), uword func_arg, void *stack)
static uword * clib_sysfs_list_to_bitmap(char *filename)
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
static vlib_thread_main_t * vlib_get_thread_main()
static f64 vlib_time_now_ticks(vlib_main_t *vm, u64 n)
static vlib_node_t * vlib_get_node(vlib_main_t *vm, u32 i)
Get vlib node by index.
#define vec_foreach(var, vec)
Vector iterator.
void * vlib_worker_thread_bootstrap_fn(void *arg)
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
#define CLIB_MEMORY_BARRIER()
uword * cpu_socket_bitmap
#define foreach_sched_policy
vlib_thread_registration_t ** registrations
#define VLIB_NODE_FLAG_TRACE
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
#define CLIB_CACHE_LINE_BYTES
vlib_thread_registration_t * registration
volatile u32 worker_thread_release
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
void vlib_worker_thread_fork_fixup(vlib_fork_fixup_t which)
clib_random_buffer_t random_buffer
#define FRAME_QUEUE_MAX_NELTS
#define VLIB_FRAME_FREE_AFTER_DISPATCH
clib_error_t * vlib_thread_init(vlib_main_t *vm)