48 u32 data_offset,
u32 n_bytes)
61 for (i = 0; i < n_bytes; i++)
62 if ((bd[i] & pm[i]) != pd[
i])
86 ASSERT (v0 >= v_min && v0 <= v_max);
93 if (is_net_byte_order)
94 v0 = clib_host_to_net_u16 (v0);
99 if (is_net_byte_order)
100 v0 = clib_host_to_net_u32 (v0);
105 if (is_net_byte_order)
106 v0 = clib_host_to_net_u64 (v0);
115 u32 n_bits,
u32 is_net_byte_order,
u32 is_increment)
117 ASSERT (v0 >= v_min && v0 <= v_max);
118 ASSERT (v1 >= v_min && v1 <= (v_max + is_increment));
126 if (is_net_byte_order)
128 v0 = clib_host_to_net_u16 (v0);
129 v1 = clib_host_to_net_u16 (v1);
136 if (is_net_byte_order)
138 v0 = clib_host_to_net_u32 (v0);
139 v1 = clib_host_to_net_u32 (v1);
146 if (is_net_byte_order)
148 v0 = clib_host_to_net_u64 (v0);
149 v1 = clib_host_to_net_u64 (v1);
162 u32 byte_offset,
u32 is_net_byte_order,
u64 v_min,
u64 v_max)
166 while (n_buffers >= 4)
178 a0 = (
void *) b0 + byte_offset;
179 a1 = (
void *) b1 + byte_offset;
180 CLIB_PREFETCH ((
void *) b2 + byte_offset,
sizeof (v_min), WRITE);
181 CLIB_PREFETCH ((
void *) b3 + byte_offset,
sizeof (v_min), WRITE);
183 set_2 (a0, a1, v_min, v_min, v_min, v_max, n_bits, is_net_byte_order,
190 while (n_buffers > 0)
199 a0 = (
void *) b0 + byte_offset;
201 set_1 (a0, v_min, v_min, v_max, n_bits, is_net_byte_order);
214 u32 is_net_byte_order,
220 ASSERT (v >= v_min && v <= v_max);
222 while (n_buffers >= 4)
235 a0 = (
void *) b0 + byte_offset;
236 a1 = (
void *) b1 + byte_offset;
237 CLIB_PREFETCH ((
void *) b2 + byte_offset,
sizeof (v_min), WRITE);
238 CLIB_PREFETCH ((
void *) b3 + byte_offset,
sizeof (v_min), WRITE);
242 v = v > v_max ? v_min :
v;
244 v_old + 0, v_old + 1, v_min, v_max, n_bits, is_net_byte_order,
248 sum += 2 * v_old + 1;
253 sum -= 2 * v_old + 1;
256 set_1 (a0, v + 0, v_min, v_max, n_bits, is_net_byte_order);
261 v = v > v_max ? v_min :
v;
262 set_1 (a1, v + 0, v_min, v_max, n_bits, is_net_byte_order);
272 while (n_buffers > 0)
282 a0 = (
void *) b0 + byte_offset;
288 v = v > v_max ? v_min :
v;
290 ASSERT (v_old >= v_min && v_old <= v_max);
291 set_1 (a0, v_old, v_min, v_max, n_bits, is_net_byte_order);
309 u32 is_net_byte_order,
313 u64 v_diff = v_max - v_min + 1;
324 while (n_buffers >= 4)
337 a0 = (
void *) b0 + byte_offset;
338 a1 = (
void *) b1 + byte_offset;
339 CLIB_PREFETCH ((
void *) b2 + byte_offset,
sizeof (v_min), WRITE);
340 CLIB_PREFETCH ((
void *) b3 + byte_offset,
sizeof (v_min), WRITE);
347 u##n * r = random_data; \ 350 random_data = r + 2; \ 367 v0 = v0 > v_max ? v0 - v_diff : v0;
368 v1 = v1 > v_max ? v1 - v_diff : v1;
369 v0 = v0 > v_max ? v0 - v_diff : v0;
370 v1 = v1 > v_max ? v1 - v_diff : v1;
375 set_2 (a0, a1, v0, v1, v_min, v_max, n_bits, is_net_byte_order,
382 while (n_buffers > 0)
392 a0 = (
void *) b0 + byte_offset;
399 u##n * r = random_data; \ 401 random_data = r + 1; \ 417 v0 = v0 > v_max ? v0 - v_diff : v0;
418 v0 = v0 > v_max ? v0 - v_diff : v0;
423 set_1 (a0, v0, v_min, v_max, n_bits, is_net_byte_order);
433 clib_mem_unaligned (a##i, t) = \ 434 clib_host_to_net_##t ((clib_net_to_host_mem_##t (a##i) &~ mask) \ 443 ASSERT (v0 >= v_min && v0 <= v_max);
444 if (max_bits ==
BITS (
u8))
445 ((
u8 *) a0)[0] = (((
u8 *) a0)[0] & ~mask) | (v0 << shift);
447 else if (max_bits ==
BITS (
u16))
451 else if (max_bits ==
BITS (
u32))
455 else if (max_bits ==
BITS (
u64))
467 ASSERT (v0 >= v_min && v0 <= v_max);
468 ASSERT (v1 >= v_min && v1 <= v_max + is_increment);
469 if (max_bits ==
BITS (
u8))
471 ((
u8 *) a0)[0] = (((
u8 *) a0)[0] & ~mask) | (v0 << shift);
472 ((
u8 *) a1)[0] = (((
u8 *) a1)[0] & ~mask) | (v1 << shift);
475 else if (max_bits ==
BITS (
u16))
480 else if (max_bits ==
BITS (
u32))
485 else if (max_bits ==
BITS (
u64))
505 while (n_buffers >= 4)
517 a0 = (
void *) b0 + byte_offset;
518 a1 = (
void *) b1 + byte_offset;
519 CLIB_PREFETCH ((
void *) b2 + byte_offset,
sizeof (v_min), WRITE);
520 CLIB_PREFETCH ((
void *) b3 + byte_offset,
sizeof (v_min), WRITE);
523 v_min, v_min, v_min, v_max, max_bits, n_bits, mask, shift,
530 while (n_buffers > 0)
539 a0 = (
void *) b0 + byte_offset;
541 setbits_1 (a0, v_min, v_min, v_max, max_bits, n_bits, mask, shift);
558 ASSERT (v >= v_min && v <= v_max);
560 while (n_buffers >= 4)
573 a0 = (
void *) b0 + byte_offset;
574 a1 = (
void *) b1 + byte_offset;
575 CLIB_PREFETCH ((
void *) b2 + byte_offset,
sizeof (v_min), WRITE);
576 CLIB_PREFETCH ((
void *) b3 + byte_offset,
sizeof (v_min), WRITE);
580 v = v > v_max ? v_min :
v;
582 v_old + 0, v_old + 1,
583 v_min, v_max, max_bits, n_bits, mask, shift,
589 setbits_1 (a0, v + 0, v_min, v_max, max_bits, n_bits, mask, shift);
592 v = v > v_max ? v_min :
v;
593 setbits_1 (a1, v + 0, v_min, v_max, max_bits, n_bits, mask, shift);
600 while (n_buffers > 0)
610 a0 = (
void *) b0 + byte_offset;
614 v = v > v_max ? v_min :
v;
616 ASSERT (v_old >= v_min && v_old <= v_max);
617 setbits_1 (a0, v_old, v_min, v_max, max_bits, n_bits, mask, shift);
635 u64 v_diff = v_max - v_min + 1;
644 while (n_buffers >= 4)
657 a0 = (
void *) b0 + byte_offset;
658 a1 = (
void *) b1 + byte_offset;
659 CLIB_PREFETCH ((
void *) b2 + byte_offset,
sizeof (v_min), WRITE);
660 CLIB_PREFETCH ((
void *) b3 + byte_offset,
sizeof (v_min), WRITE);
667 u##n * r = random_data; \ 670 random_data = r + 2; \ 687 v0 = v0 > v_max ? v0 - v_diff : v0;
688 v1 = v1 > v_max ? v1 - v_diff : v1;
689 v0 = v0 > v_max ? v0 - v_diff : v0;
690 v1 = v1 > v_max ? v1 - v_diff : v1;
692 setbits_2 (a0, a1, v0, v1, v_min, v_max, max_bits, n_bits, mask, shift,
699 while (n_buffers > 0)
709 a0 = (
void *) b0 + byte_offset;
716 u##n * r = random_data; \ 718 random_data = r + 1; \ 734 v0 = v0 > v_max ? v0 - v_diff : v0;
735 v0 = v0 > v_max ? v0 - v_diff : v0;
737 setbits_1 (a0, v0, v_min, v_max, max_bits, n_bits, mask, shift);
751 u32 max_bits, l0, l1, h1, start_bit;
760 start_bit = l0 *
BITS (
u8);
762 max_bits = hi_bit - start_bit;
767 if (edit_type == PG_EDIT_INCREMENT) \ 768 v = do_set_increment (pg, s, buffers, n_buffers, \ 775 else if (edit_type == PG_EDIT_RANDOM) \ 776 do_set_random (pg, s, buffers, n_buffers, \ 783 do_set_fixed (pg, s, buffers, n_buffers, \ 790 if (l1 == 0 && h1 == 0)
806 u32 n_bits = max_bits;
810 mask = ((
u64) 1 << (
u64) n_bits) - 1;
811 mask &= ~(((
u64) 1 << (
u64) shift) - 1);
813 mask <<= max_bits - n_bits;
814 shift += max_bits - n_bits;
820 if (edit_type == PG_EDIT_INCREMENT) \ 821 v = do_setbits_increment (pg, s, buffers, n_buffers, \ 822 BITS (u##n), n_bits, \ 823 l0, v_min, v_max, v, \ 825 else if (edit_type == PG_EDIT_RANDOM) \ 826 do_setbits_random (pg, s, buffers, n_buffers, \ 827 BITS (u##n), n_bits, \ 831 do_setbits_fixed (pg, s, buffers, n_buffers, \ 832 BITS (u##n), n_bits, \ 854 u64 v_min, v_max, length_sum;
885 length_sum = v_min * n_buffers;
905 u32 * buffers,
u32 n_buffers)
910 static u32 *unused_buffers = 0;
912 while (n_buffers > 0)
930 if (n_bytes_left > 0)
946 ASSERT (n_bytes_left == 0);
952 if (
vec_len (unused_buffers) > 0)
955 _vec_len (unused_buffers) = 0;
980 lo_bit = hi_bit - e->
n_bits;
983 =
do_it (pg, s, buffers, n_buffers, lo_bit, hi_bit, v_min, v_max,
1013 u32 * buffers,
u32 * next_buffers,
u32 n_buffers)
1017 while (n_buffers >= 4)
1024 ni0 = next_buffers[0];
1025 ni1 = next_buffers[1];
1040 while (n_buffers > 0)
1046 ni0 = next_buffers[0];
1060 u32 n_buffers,
u32 data_offset,
u32 n_data)
1062 u32 n_left, *b,
i, l;
1088 if (data_offset + n_data >=
vec_len (d0))
1089 n0 =
vec_len (d0) > data_offset ?
vec_len (d0) - data_offset : 0;
1094 i = i + 1 == l ? 0 : i + 1;
1102 u32 n_buffers,
u32 data_offset,
u32 n_data,
u32 set_data)
1204 u32 * buffers,
u32 * next_buffers,
u32 n_alloc)
1223 if (is_start_of_packet)
1231 if (n_allocated == 0)
1238 n_alloc = n_allocated;
1255 if (is_start_of_packet)
1265 for (i = 0; i < n_alloc; i++)
1292 word i, n_in_fifo, n_alloc, n_free, n_added;
1293 u32 *tail, *start, *end, *last_tail, *last_start;
1298 if (n_in_fifo >= n_buffers)
1301 n_alloc = n_buffers - n_in_fifo;
1327 last_tail = last_start = 0;
1335 if (n_free < n_alloc)
1342 if (tail + n_alloc <= end)
1352 if (n_added == n && n_alloc > n_added)
1355 (pg, s, bi, start, last_start, n_alloc - n_added);
1368 return n_in_fifo + n_added;
1430 u32 *b, n_left, stream_index, next_index;
1434 stream_index = s - pg->
streams;
1505 u32 *to_next, n_this_frame, n_left, n_trace, n_packets_in_fifo;
1506 uword n_packets_generated;
1513 u32 current_config_index = ~(
u32) 0;
1518 n_packets_in_fifo =
pg_stream_fill (pg, s, n_packets_to_generate);
1519 n_packets_to_generate =
clib_min (n_packets_in_fifo, n_packets_to_generate);
1520 n_packets_generated = 0;
1525 current_config_index =
1531 while (n_packets_to_generate > 0)
1533 u32 *head, *start, *end;
1537 n_this_frame = n_packets_to_generate;
1538 if (n_this_frame > n_left)
1539 n_this_frame = n_left;
1545 if (head + n_this_frame <= end)
1557 if (current_config_index != ~(
u32) 0)
1558 for (i = 0; i < n_this_frame; i++)
1562 vnet_buffer (b)->device_input_feat.saved_next_index =
1564 vnet_buffer (b)->device_input_feat.buffer_advance = 0;
1576 n_packets_to_generate -= n_this_frame;
1577 n_packets_generated += n_this_frame;
1578 n_left -= n_this_frame;
1582 return n_packets_generated;
1638 uword n_packets = 0;
1639 u32 worker_index = 0;
1646 pg_stream_t *s = vec_elt_at_index (pg->streams, i);
1647 n_packets += pg_input_stream (node, pg, s);
1658 .sibling_of =
"device-input",
1664 .state = VLIB_NODE_STATE_DISABLED,
vnet_config_main_t config_main
sll srl srl sll sra u16x4 i
#define clib_fifo_head(v)
static u32 vlib_get_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt)
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
u32 current_config_index
Used by feature subgraph arcs to visit enabled feature nodes.
static uword clib_fifo_elts(void *v)
vnet_main_t * vnet_get_main(void)
vnet_interface_main_t interface_main
pg_edit_group_t * edit_groups
#define clib_fifo_advance_tail(f, n_elts)
static f64 vlib_time_now(vlib_main_t *vm)
#define clib_fifo_resize(f, n_elts)
static void * clib_random_buffer_get_data(clib_random_buffer_t *b, uword n_bytes)
vlib_buffer_t buffer_init_template
void(* edit_function)(struct pg_main_t *pg, struct pg_stream_t *s, struct pg_edit_group_t *g, u32 *buffers, u32 n_buffers)
u32 * config_index_by_sw_if_index
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
vlib_buffer_main_t * buffer_main
#define STRUCT_OFFSET_OF(t, f)
static vnet_sw_interface_t * vnet_get_sw_interface(vnet_main_t *vnm, u32 sw_if_index)
static_always_inline int vnet_have_features(u8 arc, u32 sw_if_index)
pg_buffer_index_t * buffer_indices
pg_edit_type_t packet_size_edit_type
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
u8 * fixed_packet_data_mask
static void vlib_trace_buffer(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, vlib_buffer_t *b, int follow_chain)
#define VLIB_BUFFER_NEXT_PRESENT
#define static_always_inline
void(* buffer_init_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl, u32 *buffers, u32 n_buffers)
#define vlib_prefetch_buffer_with_index(vm, bi, type)
Prefetch buffer metadata by buffer index The first 64 bytes of buffer contains most header informatio...
u32 local_interface_sw_if_index
static uword pow2_mask(uword x)
vlib_combined_counter_main_t * combined_sw_if_counters
static uword clib_fifo_free_elts(void *v)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
u8 feature_arc_index
Used to identify feature arcs by intermediate feature node.
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
static u32 vlib_get_current_worker_index()
#define vec_end(v)
End (last data address) of vector.
static void vlib_copy_buffers(u32 *dst, u32 *src, u32 n)
u32 last_increment_packet_size
#define clib_bitmap_foreach(i, ai, body)
Macro to iterate across set bits in a bitmap.
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
format_function_t format_vnet_buffer
u16 current_length
Nbytes between current data and the end of this buffer.
u32 current_replay_packet_index
static u64 pg_edit_get_value(pg_edit_t *e, int hi_or_lo)
static void * vnet_get_config_data(vnet_config_main_t *cm, u32 *config_index, u32 *next_index, u32 n_data_bytes)
#define PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE
format_function_t * format_buffer
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
static void vlib_buffer_free_no_next(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers, does not free the buffer chain for each buffer.
static_always_inline uword vlib_get_thread_index(void)
#define CLIB_PREFETCH(addr, size, type)
#define clib_warning(format, args...)
#define clib_memcpy(a, b, c)
static u32 vlib_buffer_alloc_from_free_list(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 free_list_index)
Allocate buffers from specific freelist into supplied array.
static uword max_pow2(uword x)
#define pool_is_free_index(P, I)
Use free bitmap to query whether given index is free.
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
pg_edit_t * non_fixed_edits
u8 ** replay_packet_templates
static uword clib_fifo_advance_head(void *v, uword n_elts)
u32 next_buffer
Next buffer for this linked-list of buffers.
void pg_stream_enable_disable(pg_main_t *pg, pg_stream_t *s, int is_enable)
static uword vlib_buffer_index_length_in_chain(vlib_main_t *vm, u32 bi)
Get length in bytes of the buffer index buffer chain.
static vlib_main_t * vlib_get_main(void)
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
#define vec_elt(v, i)
Get vector value at index i.
u8 device_input_feature_arc_index
Feature arc index for device-input.
u32 sw_if_index[VLIB_N_RX_TX]
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define clib_mem_unaligned(pointer, type)
f64 rate_packets_per_second
#define VLIB_REGISTER_NODE(x,...)
static u32 vlib_num_workers()
#define STRUCT_SIZE_OF(t, f)
static vlib_node_t * vlib_get_node(vlib_main_t *vm, u32 i)
Get vlib node by index.
#define vec_foreach(var, vec)
Vector iterator.
uword buffer_init_function_opaque
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, u32 free_list_index)
static void vlib_set_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 count)
vnet_feature_config_main_t * feature_config_mains
feature config main objects
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
vnet_feature_main_t feature_main
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
clib_random_buffer_t random_buffer