19 #include <sys/mount.h> 42 #define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) 70 .pktfield0_slabpos = 0,
71 .pktfield0_slabmask = 0,
78 .pktfield1_slabpos = 40,
79 .pktfield1_slabmask = 0x0000000FFF000000LLU,
86 .pktfield2_slabpos = 8,
87 .pktfield2_slabmask = 0x00000000000000FCLLU,
89 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
90 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
91 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
92 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
101 .frame_overhead = RTE_SCHED_FRAME_OVERHEAD_DEFAULT,
102 .n_subports_per_port = 1,
103 .n_pipes_per_subport = 4096,
104 .qsize = {64, 64, 64, 64},
105 .pipe_profiles =
NULL,
106 .n_pipe_profiles = 1,
111 [0][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
113 [0][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
115 [0][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
119 [1][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
121 [1][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
123 [1][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
127 [2][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
129 [2][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
131 [2][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
135 [3][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
137 [3][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
139 [3][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
147 .tb_rate = 1250000000,
149 .tc_rate = {1250000000, 1250000000, 1250000000, 1250000000},
156 .tc_rate = {305175, 305175, 305175, 305175},
158 #ifdef RTE_SCHED_SUBPORT_TC_OV 161 .wrr_weights = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
173 int count = __builtin_popcountll (mask);
174 int pos_lead =
sizeof (
u64) * 8 - __builtin_clzll (mask);
175 int pos_trail = __builtin_ctzll (mask);
176 int count_expected = __builtin_popcount (n - 1);
182 if ((mask == 0) && (n == 1))
185 if (((mask == 0) && (n != 1)) || ((mask != 0) && (n == 1)))
189 if ((pos_lead - pos_trail) != count)
193 if (count != count_expected)
201 hqos,
u32 pipe_profile_id)
210 struct rte_sched_subport_params *subport_params;
211 struct rte_sched_pipe_params *pipe_params;
215 memcpy (hqos, &hqos_params_default,
sizeof (hqos_params_default));
221 memcpy (&pipe_params[i],
224 hqos->
port.pipe_profiles = hqos->
pipe;
230 memcpy (&subport_params[i],
237 hqos->
port.n_subports_per_port * hqos->
port.n_pipes_per_subport);
258 int worker_thread_first = 0;
259 int worker_thread_count = 0;
265 if (tr && tr->
count > 0)
268 worker_thread_count = tr->
count;
283 for (i = 0; i < worker_thread_count + 1; i++)
285 u32 swq_flags = RING_F_SP_ENQ | RING_F_SC_DEQ;
287 snprintf (name,
sizeof (name),
"SWQ-worker%u-to-device%u", i,
293 "SWQ-worker%u-to-device%u: rte_ring_create err",
302 snprintf (name,
sizeof (name),
"HQoS%u", xd->
device_index);
303 hqos->
port.name = strdup (name);
308 if (hqos->
port.socket == SOCKET_ID_ANY)
309 hqos->
port.socket = 0;
317 for (subport_id = 0; subport_id < hqos->
port.n_subports_per_port;
323 rte_sched_subport_config (xd->
hqos_ht->
hqos, subport_id,
327 "HQoS%u subport %u: rte_sched_subport_config err (%d)",
331 for (pipe_id = 0; pipe_id < hqos->
port.n_pipes_per_subport; pipe_id++)
333 u32 pos = subport_id * hqos->
port.n_pipes_per_subport + pipe_id;
337 rte_sched_pipe_config (xd->
hqos_ht->
hqos, subport_id, pipe_id,
341 "HQoS%u subport %u pipe %u: rte_sched_pipe_config err (%d)",
357 for (i = 0; i < worker_thread_count + 1; i++)
361 tid = worker_thread_first + (i - 1);
409 if (dev_pos >= n_devs)
420 struct rte_mbuf **pkts_enq = hqos->
pkts_enq;
426 for (
i = 0;
i < n_swq;
i++)
429 struct rte_ring *swq = hqos->
swq[swq_pos];
432 pkts_enq_len += rte_ring_sc_dequeue_burst (swq,
434 &pkts_enq[pkts_enq_len],
439 if (swq_pos >= n_swq)
446 u32 n_pkts = rte_eth_tx_burst (device_index,
449 (uint16_t) pkts_enq_len);
451 for (; n_pkts < pkts_enq_len; n_pkts++)
452 rte_pktmbuf_free (pkts_enq[n_pkts]);
464 rte_sched_port_enqueue (hqos->
hqos, pkts_enq, pkts_enq_len);
496 if (dev_pos >= n_devs)
507 struct rte_mbuf **pkts_enq = hqos->
pkts_enq;
508 struct rte_mbuf **pkts_deq = hqos->
pkts_deq;
517 for (
i = 0;
i < n_swq;
i++)
520 struct rte_ring *swq = hqos->
swq[swq_pos];
523 pkts_enq_len += rte_ring_sc_dequeue_burst (swq,
525 &pkts_enq[pkts_enq_len],
530 if (swq_pos >= n_swq)
537 rte_sched_port_enqueue (hqos->
hqos, pkts_enq, pkts_enq_len);
549 rte_sched_port_enqueue (hqos->
hqos, pkts_enq, pkts_enq_len);
562 u32 pkts_deq_len, n_pkts;
564 pkts_deq_len = rte_sched_port_dequeue (hqos->
hqos,
568 for (n_pkts = 0; n_pkts < pkts_deq_len;)
569 n_pkts += rte_eth_tx_burst (device_index,
572 (uint16_t) (pkts_deq_len - n_pkts));
601 (
"current I/O TX thread does not have any devices assigned to it");
620 .name =
"hqos-threads",
621 .short_name =
"hqos-threads",
629 #define BITFIELD(byte_array, slab_pos, slab_mask, slab_shr) \ 631 u64 slab = *((u64 *) &byte_array[slab_pos]); \ 632 u64 val = (rte_be_to_cpu_64(slab) & slab_mask) >> slab_shr; \ 636 #define RTE_SCHED_PORT_HIERARCHY(subport, pipe, traffic_class, queue, color) \ 637 ((((u64) (queue)) & 0x3) | \ 638 ((((u64) (traffic_class)) & 0x3) << 2) | \ 639 ((((u64) (color)) & 0x3) << 4) | \ 640 ((((u64) (subport)) & 0xFFFF) << 16) | \ 641 ((((u64) (pipe)) & 0xFFFFFFFF) << 32)) 645 struct rte_mbuf **pkts,
u32 n_pkts)
649 for (i = 0; i < (n_pkts & (~0x3)); i += 4)
651 struct rte_mbuf *pkt0 = pkts[
i];
652 struct rte_mbuf *pkt1 = pkts[i + 1];
653 struct rte_mbuf *pkt2 = pkts[i + 2];
654 struct rte_mbuf *pkt3 = pkts[i + 3];
656 u8 *pkt0_data = rte_pktmbuf_mtod (pkt0,
u8 *);
657 u8 *pkt1_data = rte_pktmbuf_mtod (pkt1,
u8 *);
658 u8 *pkt2_data = rte_pktmbuf_mtod (pkt2,
u8 *);
659 u8 *pkt3_data = rte_pktmbuf_mtod (pkt3,
u8 *);
730 pkt0->hash.sched.lo = pkt0_sched & 0xFFFFFFFF;
731 pkt0->hash.sched.hi = pkt0_sched >> 32;
732 pkt1->hash.sched.lo = pkt1_sched & 0xFFFFFFFF;
733 pkt1->hash.sched.hi = pkt1_sched >> 32;
734 pkt2->hash.sched.lo = pkt2_sched & 0xFFFFFFFF;
735 pkt2->hash.sched.hi = pkt2_sched >> 32;
736 pkt3->hash.sched.lo = pkt3_sched & 0xFFFFFFFF;
737 pkt3->hash.sched.hi = pkt3_sched >> 32;
740 for (; i < n_pkts; i++)
742 struct rte_mbuf *pkt = pkts[
i];
744 u8 *pkt_data = rte_pktmbuf_mtod (pkt,
u8 *);
764 pkt->hash.sched.lo = pkt_sched & 0xFFFFFFFF;
765 pkt->hash.sched.hi = pkt_sched >> 32;
VLIB_REGISTER_THREAD(hqos_thread_reg, static)
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
sll srl srl sll sra u16x4 i
static struct rte_sched_subport_params hqos_subport_params_default
#define clib_error(format, args...)
dpdk_device_and_queue_t ** devices_by_hqos_cpu
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
struct rte_sched_port_params port
struct rte_mbuf ** pkts_enq
struct rte_sched_port * hqos
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
#define BITFIELD(byte_array, slab_pos, slab_mask, slab_shr)
static void vlib_worker_thread_barrier_check(void)
struct rte_sched_pipe_params * pipe
#define static_always_inline
struct rte_mbuf ** pkts_deq
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
dpdk_device_hqos_per_worker_thread_t * hqos_wt
uword os_get_cpu_number(void)
static_always_inline void dpdk_hqos_thread_internal_hqos_dbg_bypass(vlib_main_t *vm)
void clib_time_init(clib_time_t *c)
int dpdk_hqos_validate_mask(u64 mask, u32 n)
#define HQOS_FLUSH_COUNT_THRESHOLD
static void * clib_mem_set_heap(void *heap)
void vlib_worker_thread_init(vlib_worker_thread_t *w)
#define DPDK_HQOS_DBG_BYPASS
void dpdk_hqos_thread_fn(void *arg)
clib_error_t * dpdk_port_setup_hqos(dpdk_device_t *xd, dpdk_device_config_hqos_t *hqos)
Bitmaps built as vectors of machine words.
dpdk_device_hqos_per_hqos_thread_t * hqos_ht
static_always_inline void dpdk_hqos_thread_internal(vlib_main_t *vm)
uword * thread_registrations_by_name
static vlib_main_t * vlib_get_main(void)
void dpdk_hqos_metadata_set(dpdk_device_hqos_per_worker_thread_t *hqos, struct rte_mbuf **pkts, u32 n_pkts)
void dpdk_device_config_hqos_default(dpdk_device_config_hqos_t *hqos)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
void dpdk_hqos_thread(vlib_worker_thread_t *w)
#define hash_get_mem(h, key)
static vlib_thread_main_t * vlib_get_thread_main()
struct rte_sched_subport_params * subport
#define RTE_SCHED_PORT_HIERARCHY(subport, pipe, traffic_class, queue, color)
#define CLIB_CACHE_LINE_BYTES
void dpdk_device_config_hqos_pipe_profile_default(dpdk_device_config_hqos_t *hqos, u32 pipe_profile_id)
volatile u32 worker_thread_release
static struct rte_sched_pipe_params hqos_pipe_params_default
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".