15 #ifndef included_vlib_threads_h 16 #define included_vlib_threads_h 20 #include <linux/sched.h> 56 #define VLIB_MAX_CPUS 256 59 #if VLIB_MAX_CPUS > CLIB_MAX_MHEAPS 60 #error Please increase number of per-cpu mheaps 63 #define VLIB_CPU_MASK (VLIB_MAX_CPUS - 1) 64 #define VLIB_OFFSET_MASK (~VLIB_CPU_MASK) 66 #define VLIB_LOG2_THREAD_STACK_SIZE (21) 67 #define VLIB_THREAD_STACK_SIZE (1<<VLIB_LOG2_THREAD_STACK_SIZE) 98 void (*thread_function) (
void *);
182 vlib_frame_queue_msg_type_t
type);
190 void (*thread_function) (
void *));
196 #define BARRIER_SYNC_DELAY (0.030000) 200 #define BARRIER_SYNC_TIMEOUT (600.1) 202 #define BARRIER_SYNC_TIMEOUT (1.0) 205 #define vlib_worker_thread_barrier_sync(X) {vlib_worker_thread_barrier_sync_int(X, __FUNCTION__);} 208 const char *func_name);
221 return __os_thread_index;
230 fformat (stderr,
"%s: SMP unsafe warning...\n", __FUNCTION__);
242 #define foreach_vlib_main(body) \ 244 vlib_main_t ** __vlib_mains = 0, *this_vlib_main; \ 247 for (ii = 0; ii < vec_len (vlib_mains); ii++) \ 249 this_vlib_main = vlib_mains[ii]; \ 251 this_vlib_main->parked_at_barrier == 1); \ 252 if (this_vlib_main) \ 253 vec_add1 (__vlib_mains, this_vlib_main); \ 256 for (ii = 0; ii < vec_len (__vlib_mains); ii++) \ 258 this_vlib_main = __vlib_mains[ii]; \ 262 vec_free (__vlib_mains); \ 265 #define foreach_sched_policy \ 266 _(SCHED_OTHER, OTHER, "other") \ 267 _(SCHED_BATCH, BATCH, "batch") \ 268 _(SCHED_IDLE, IDLE, "idle") \ 269 _(SCHED_FIFO, FIFO, "fifo") \ 270 _(SCHED_RR, RR, "rr") 274 #define _(v,f,s) SCHED_POLICY_##f = v, 285 } vlib_thread_callbacks_t;
345 vlib_thread_callbacks_t
cb;
357 #define VLIB_REGISTER_THREAD(x,...) \ 358 __VA_ARGS__ vlib_thread_registration_t x; \ 359 static void __vlib_add_thread_registration_##x (void) \ 360 __attribute__((__constructor__)) ; \ 361 static void __vlib_add_thread_registration_##x (void) \ 363 vlib_thread_main_t * tm = &vlib_thread_main; \ 367 static void __vlib_rm_thread_registration_##x (void) \ 368 __attribute__((__destructor__)) ; \ 369 static void __vlib_rm_thread_registration_##x (void) \ 371 vlib_thread_main_t * tm = &vlib_thread_main; \ 372 VLIB_REMOVE_FROM_LINKED_LIST (tm->next, &x, next); \ 374 __VA_ARGS__ vlib_thread_registration_t x 385 return worker_index + 1;
391 return thread_index - 1;
418 .format =
"barrier-wait-thread-%d",
430 ed->thread_index = thread_index;
468 .format =
"barrier-refork-thread-%d",
480 ed->thread_index = thread_index;
495 .format =
"barrier-released-thread-%d: %dus",
496 .format_args =
"i4i4",
508 ed->thread_index = thread_index;
509 ed->duration = (int) (1000000.0 * t);
523 ASSERT (worker_index < tm->n_vlib_mains - 1);
524 vm = vlib_mains[worker_index + 1];
563 elt = fq->
elts + (new_tail & (fq->
nelts - 1));
580 handoff_queue_by_worker_index)
587 fq = handoff_queue_by_worker_index[
index];
599 handoff_queue_by_worker_index[
index] = fq;
609 u32 vlib_worker_index,
611 handoff_queue_elt_by_worker_index)
615 if (handoff_queue_elt_by_worker_index[vlib_worker_index])
616 return handoff_queue_elt_by_worker_index[vlib_worker_index];
620 handoff_queue_elt_by_worker_index[vlib_worker_index] = elt;
627 vlib_thread_callbacks_t * cb);
vlib_main_t vlib_global_main
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
void vlib_worker_thread_init(vlib_worker_thread_t *w)
void() vlib_thread_function_t(void *arg)
int vlib_frame_queue_enqueue(vlib_main_t *vm, u32 node_runtime_index, u32 frame_queue_index, vlib_frame_t *frame, vlib_frame_queue_msg_type_t type)
#define clib_atomic_add_fetch(a, b)
void vlib_worker_thread_initial_barrier_sync_and_release(vlib_main_t *vm)
static f64 vlib_time_now(vlib_main_t *vm)
void vlib_set_thread_name(char *name)
void * thread_function_arg
frame_queue_trace_t * frame_queue_traces
vlib_worker_thread_t * vlib_worker_threads
void vlib_rpc_call_main_thread(void *function, u8 *args, u32 size)
struct vlib_thread_registration_ * next
int vlib_thread_cb_register(struct vlib_main_t *vm, vlib_thread_callbacks_t *cb)
static vlib_frame_queue_t * is_vlib_frame_queue_congested(u32 frame_queue_index, u32 index, u32 queue_hi_thresh, vlib_frame_queue_t **handoff_queue_by_worker_index)
static void vlib_smp_unsafe_warning(void)
volatile int parked_at_barrier
static void vlib_worker_thread_barrier_check(void)
vlib_thread_registration_t * next
static u32 vlib_get_worker_index(u32 thread_index)
#define static_always_inline
Callback multiplex scheme For a fully worked-out example, see .../src/vlib/main.
vlib_frame_queue_msg_type_t
vlib_frame_queue_elt_t ** handoff_queue_elt_by_thread_index
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
int vlib_frame_queue_dequeue(vlib_main_t *vm, vlib_frame_queue_main_t *fqm)
vlib_frame_queue_elt_t * elts
pthread_t thread[MAX_CONNS]
f64 time_last_barrier_release
static u32 vlib_get_current_worker_index()
void vlib_process_signal_event_mt_helper(vlib_process_signal_event_mt_args_t *args)
vl_api_fib_path_type_t type
u8 * vlib_thread_stack_init(uword thread_index)
vlib_thread_callbacks_t cb
const char * barrier_context
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
static vlib_frame_queue_elt_t * vlib_get_worker_handoff_queue_elt(u32 frame_queue_index, u32 vlib_worker_index, vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index)
void vlib_worker_thread_node_runtime_update(void)
void vlib_worker_thread_fork_fixup(vlib_fork_fixup_t which)
vlib_frame_queue_t ** vlib_frame_queues
static_always_inline uword vlib_get_thread_index(void)
frame_queue_nelt_counter_t * frame_queue_histogram
void * rpc_call_main_thread_cb_fn
#define ELOG_TYPE_DECLARE(f)
void vlib_worker_wait_one_loop(void)
Wait until each of the workers has been once around the track.
vlib_worker_thread_t * worker_threads
static u8 vlib_thread_is_main_w_barrier(void)
static vlib_frame_queue_elt_t * vlib_get_frame_queue_elt(u32 frame_queue_index, u32 index)
volatile u32 * wait_at_barrier
vlib_frame_queue_per_thread_data_t * per_thread_data
vlib_frame_queue_t ** congested_handoff_queue_by_thread_index
u8 vlib_worker_thread_barrier_held(void)
Return true if the wroker thread barrier is held.
vlib_frame_queue_main_t * frame_queue_mains
#define ELOG_TRACK_DATA(em, f, track)
struct vlib_thread_registration_ vlib_thread_registration_t
uword * thread_registrations_by_name
u32 vlib_frame_queue_main_init(u32 node_index, u32 frame_queue_nelts)
volatile u32 * node_reforks_required
const char * barrier_caller
static vlib_main_t * vlib_get_main(void)
clib_error_t * vlib_thread_init(vlib_main_t *vm)
#define clib_atomic_fetch_add(a, b)
void(**volatile barrier_perf_callbacks)(struct vlib_main_t *, u64 t, int leave)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
int no_data_structure_clone
vlib_main_t ** vlib_mains
void vlib_create_worker_threads(vlib_main_t *vm, int n, void(*thread_function)(void *))
static void vlib_put_frame_queue_elt(vlib_frame_queue_elt_t *hf)
volatile u32 * workers_at_barrier
#define clib_call_callbacks(h,...)
call the specified callback set
static u32 vlib_num_workers()
void vlib_worker_thread_node_refork(void)
#define CLIB_MEMORY_BARRIER()
uword * cpu_socket_bitmap
#define foreach_sched_policy
vlib_thread_registration_t ** registrations
vlib_thread_main_t vlib_thread_main
vlib_thread_registration_t * registration
volatile u32 worker_thread_release
static vlib_main_t * vlib_get_worker_vlib_main(u32 worker_index)
static u32 vlib_get_worker_thread_index(u32 worker_index)
void vlib_get_thread_core_numa(vlib_worker_thread_t *w, unsigned cpu_id)
void vlib_worker_thread_barrier_sync_int(vlib_main_t *vm, const char *func_name)