21 #include <sys/ioctl.h> 33 #define foreach_memif_input_error \ 34 _(NOT_IP, "not ip packet") 38 #define _(f,s) MEMIF_INPUT_ERROR_##f, 57 static __clib_unused
u8 *
65 s =
format (s,
"memif: hw_if_index %d next-index %d",
118 u16 ring_size,
u32 n_buffer_bytes,
120 u32 * first_bi,
u32 * bi,
u16 * num_slots)
129 u16 mask = ring_size - 1;
136 while (data_len && (*n_free_bufs))
146 _vec_len (nm->
rx_buffers[thread_index]) = last_buf;
153 [*n_free_bufs], ring_size);
154 _vec_len (nm->
rx_buffers[thread_index]) = *n_free_bufs;
165 data_len > n_buffer_bytes ? n_buffer_bytes : data_len;
178 if (total_bytes == 0)
192 total_bytes += bytes_to_copy;
193 data_len -= bytes_to_copy;
204 return (total_bytes);
236 u32 n_rx_packets = 0;
240 u32 b0_total, b1_total;
242 u16 ring_size, mask, num_slots;
249 mask = ring_size - 1;
264 ring_size + n_free_bufs - 1);
268 _vec_len (nm->
rx_buffers[thread_index]) = n_free_bufs;
281 u32 next0 = next_index;
282 u32 next1 = next_index;
285 while (num_slots > 11 && n_left_to_next > 2)
299 u32 bi0 = 0, first_bi0 = 0;
303 &n_free_bufs, &first_b0,
308 u32 bi1 = 0, first_bi1 = 0;
312 &n_free_bufs, &first_b1,
321 to_next[0] = first_bi0;
322 to_next[1] = first_bi1;
381 n_left_to_next, first_bi0,
382 first_bi1, next0, next1);
386 n_rx_bytes += b0_total + b1_total;
390 if (!first_bi0 && !first_bi1)
396 first_bi0 = first_bi1;
436 to_next[0] = first_bi0;
442 n_left_to_next, first_bi0, next0);
446 n_rx_bytes += b0_total;
448 while (num_slots && n_left_to_next)
451 u32 bi0 = 0, first_bi0 = 0;
455 &n_free_bufs, &first_b0,
497 to_next[0] = first_bi0;
503 n_left_to_next, first_bi0, next0);
507 n_rx_bytes += b0_total;
542 if ((mif->
flags & MEMIF_IF_FLAG_ADMIN_UP) &&
543 (mif->
flags & MEMIF_IF_FLAG_CONNECTED))
545 if (mif->
flags & MEMIF_IF_FLAG_IS_SLAVE)
573 #ifndef CLIB_MULTIARCH_VARIANT 577 .name =
"memif-input",
578 .sibling_of =
"device-input",
581 .state = VLIB_NODE_STATE_INTERRUPT,
590 static void __clib_constructor
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
uword( vlib_node_function_t)(struct vlib_main_t *vm, struct vlib_node_runtime_t *node, struct vlib_frame_t *frame)
static_always_inline uword memif_device_input_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, memif_if_t *mif, memif_ring_type_t type, u16 qid, memif_interface_mode_t mode)
static u32 vlib_get_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt)
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
vnet_main_t * vnet_get_main(void)
vnet_interface_main_t interface_main
#define CLIB_MEMORY_STORE_BARRIER()
vlib_error_t * errors
Vector of errors for this node.
static void vlib_trace_buffer(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, vlib_buffer_t *b, int follow_chain)
static_always_inline uword memif_copy_buffer_from_rx_ring(vlib_main_t *vm, memif_if_t *mif, memif_ring_t *ring, memif_queue_t *mq, u16 ring_size, u32 n_buffer_bytes, u32 *n_free_bufs, vlib_buffer_t **first_b, u32 *first_bi, u32 *bi, u16 *num_slots)
Copy buffer from rx ring.
#define VLIB_BUFFER_NEXT_PRESENT
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
#define static_always_inline
static_always_inline void memif_buffer_add_to_chain(vlib_main_t *vm, u32 bi, u32 first_bi, u32 prev_bi)
static_always_inline u32 memif_next_from_ip_hdr(vlib_node_runtime_t *node, vlib_buffer_t *b)
vlib_combined_counter_main_t * combined_sw_if_counters
#define VLIB_BUFFER_TOTAL_LENGTH_VALID
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
u32 per_interface_next_index
static __clib_unused char * memif_input_error_strings[]
vlib_node_registration_t memif_input_node
(constructor) VLIB_REGISTER_NODE (memif_input_node)
u16 current_length
Nbytes between current data and the end of this buffer.
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
static u32 vlib_buffer_free_list_buffer_size(vlib_main_t *vm, u32 free_list_index)
#define foreach_memif_input_error
#define vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1)
Finish enqueueing two buffers forward in the graph.
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
vlib_error_t error
Error code for buffers to be enqueued to error handler.
static_always_inline void vnet_feature_start_device_input_x2(u32 sw_if_index, u32 *next0, u32 *next1, vlib_buffer_t *b0, vlib_buffer_t *b1)
vlib_node_function_t __clib_weak memif_input_fn_avx512
static_always_inline uword vlib_get_thread_index(void)
#define CLIB_PREFETCH(addr, size, type)
static void __clib_constructor memif_input_multiarch_select(void)
#define clib_memcpy(a, b, c)
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
static_always_inline void memif_prefetch(vlib_main_t *vm, u32 bi)
#define MEMIF_DESC_FLAG_NEXT
static_always_inline void * memif_get_buffer(memif_if_t *mif, memif_ring_t *ring, u16 slot)
u32 next_buffer
Next buffer for this linked-list of buffers.
uword CLIB_MULTIARCH_FN() memif_input_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
template key/value backing page structure
#define foreach_device_and_queue(var, vec)
vlib_node_function_t __clib_weak memif_input_fn_avx2
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
memif_log2_ring_size_t log2_ring_size
static_always_inline void vnet_feature_start_device_input_x1(u32 sw_if_index, u32 *next0, vlib_buffer_t *b0)
#define VLIB_REGISTER_NODE(x,...)
memif_queue_t * rx_queues
static __clib_unused u8 * format_memif_input_trace(u8 *s, va_list *args)
static void vlib_set_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 count)
#define CLIB_CACHE_LINE_BYTES
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
memif_interface_mode_t mode
#define CLIB_MULTIARCH_FN(fn)