19 #include <sys/mount.h> 41 #define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) 55 .hqos_thread_valid = 0,
67 .pktfield0_slabpos = 0,
68 .pktfield0_slabmask = 0,
75 .pktfield1_slabpos = 40,
76 .pktfield1_slabmask = 0x0000000FFF000000LLU,
83 .pktfield2_slabpos = 8,
84 .pktfield2_slabmask = 0x00000000000000FCLLU,
86 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
87 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
88 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
89 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
98 .frame_overhead = RTE_SCHED_FRAME_OVERHEAD_DEFAULT,
99 .n_subports_per_port = 1,
100 .n_pipes_per_subport = 4096,
101 .qsize = {64, 64, 64, 64},
102 .pipe_profiles = NULL,
103 .n_pipe_profiles = 1,
108 [0][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
110 [0][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
112 [0][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
116 [1][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
118 [1][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
120 [1][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
124 [2][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
126 [2][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
128 [2][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
132 [3][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
134 [3][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
136 [3][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
144 .tb_rate = 1250000000,
146 .tc_rate = {1250000000, 1250000000, 1250000000, 1250000000},
153 .tc_rate = {305175, 305175, 305175, 305175},
155 #ifdef RTE_SCHED_SUBPORT_TC_OV 158 .wrr_weights = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
170 int count = __builtin_popcountll (mask);
173 int count_expected = __builtin_popcount (n - 1);
179 if ((mask == 0) && (n == 1))
182 if (((mask == 0) && (n != 1)) || ((mask != 0) && (n == 1)))
186 if ((pos_lead - pos_trail) != count)
190 if (count != count_expected)
198 hqos,
u32 pipe_profile_id)
207 struct rte_sched_subport_params *subport_params;
208 struct rte_sched_pipe_params *pipe_params;
215 vec_add2 (hqos->pipe, pipe_params, hqos->port.n_pipe_profiles);
217 for (i = 0; i <
vec_len (hqos->pipe); i++)
218 memcpy (&pipe_params[i],
221 hqos->port.pipe_profiles = hqos->pipe;
224 vec_add2 (hqos->subport, subport_params, hqos->port.n_subports_per_port);
226 for (i = 0; i <
vec_len (hqos->subport); i++)
227 memcpy (&subport_params[i],
234 hqos->port.n_subports_per_port * hqos->port.n_pipes_per_subport);
236 for (i = 0; i <
vec_len (hqos->pipe_map); i++)
255 int worker_thread_first = 0;
256 int worker_thread_count = 0;
262 if (tr && tr->
count > 0)
265 worker_thread_count = tr->
count;
274 clib_memset (xd->hqos_ht, 0, sizeof (xd->hqos_ht[0]));
280 for (i = 0; i < worker_thread_count + 1; i++)
282 u32 swq_flags = RING_F_SP_ENQ | RING_F_SC_DEQ;
284 snprintf (name,
sizeof (name),
"SWQ-worker%u-to-device%u", i,
286 xd->hqos_ht->swq[
i] =
287 rte_ring_create (name, hqos->swq_size, xd->
cpu_socket, swq_flags);
288 if (xd->hqos_ht->swq[i] == NULL)
290 "SWQ-worker%u-to-device%u: rte_ring_create err",
299 snprintf (name,
sizeof (name),
"HQoS%u", xd->
port_id);
300 hqos->port.name = strdup (name);
301 if (hqos->port.name == NULL)
304 hqos->port.socket = rte_eth_dev_socket_id (xd->
port_id);
305 if (hqos->port.socket == SOCKET_ID_ANY)
306 hqos->port.socket = 0;
308 xd->hqos_ht->hqos = rte_sched_port_config (&hqos->port);
309 if (xd->hqos_ht->hqos == NULL)
314 for (subport_id = 0; subport_id < hqos->port.n_subports_per_port;
320 rte_sched_subport_config (xd->hqos_ht->hqos, subport_id,
321 &hqos->subport[subport_id]);
324 "HQoS%u subport %u: rte_sched_subport_config err (%d)",
328 for (pipe_id = 0; pipe_id < hqos->port.n_pipes_per_subport; pipe_id++)
330 u32 pos = subport_id * hqos->port.n_pipes_per_subport + pipe_id;
331 u32 profile_id = hqos->pipe_map[pos];
334 rte_sched_pipe_config (xd->hqos_ht->hqos, subport_id, pipe_id,
338 "HQoS%u subport %u pipe %u: rte_sched_pipe_config err (%d)",
339 xd->
port_id, subport_id, pipe_id, rv);
344 xd->hqos_ht->hqos_burst_enq = hqos->burst_enq;
345 xd->hqos_ht->hqos_burst_deq = hqos->burst_deq;
346 vec_validate (xd->hqos_ht->pkts_enq, 2 * hqos->burst_enq - 1);
347 vec_validate (xd->hqos_ht->pkts_deq, hqos->burst_deq - 1);
348 xd->hqos_ht->pkts_enq_len = 0;
349 xd->hqos_ht->swq_pos = 0;
350 xd->hqos_ht->flush_count = 0;
353 for (i = 0; i < worker_thread_count + 1; i++)
357 tid = worker_thread_first + (i - 1);
361 xd->hqos_wt[tid].swq = xd->hqos_ht->swq[
i];
362 xd->hqos_wt[tid].hqos_field0_slabpos = hqos->pktfield0_slabpos;
363 xd->hqos_wt[tid].hqos_field0_slabmask = hqos->pktfield0_slabmask;
364 xd->hqos_wt[tid].hqos_field0_slabshr =
366 xd->hqos_wt[tid].hqos_field1_slabpos = hqos->pktfield1_slabpos;
367 xd->hqos_wt[tid].hqos_field1_slabmask = hqos->pktfield1_slabmask;
368 xd->hqos_wt[tid].hqos_field1_slabshr =
370 xd->hqos_wt[tid].hqos_field2_slabpos = hqos->pktfield2_slabpos;
371 xd->hqos_wt[tid].hqos_field2_slabmask = hqos->pktfield2_slabmask;
372 xd->hqos_wt[tid].hqos_field2_slabshr =
374 memcpy (xd->hqos_wt[tid].hqos_tc_table, hqos->tc_table,
375 sizeof (hqos->tc_table));
404 u32 n_devs =
vec_len (dm->devices_by_hqos_cpu[thread_index]);
405 if (dev_pos >= n_devs)
408 dpdk_device_and_queue_t *dq =
412 dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht;
414 u16 queue_id = dq->queue_id;
416 struct rte_mbuf **pkts_enq = hqos->pkts_enq;
417 u32 pkts_enq_len = hqos->pkts_enq_len;
418 u32 swq_pos = hqos->swq_pos;
420 u32 flush_count = hqos->flush_count;
422 for (
i = 0;
i < n_swq;
i++)
425 struct rte_ring *swq = hqos->swq[swq_pos];
428 pkts_enq_len += rte_ring_sc_dequeue_burst (swq,
430 &pkts_enq[pkts_enq_len],
431 hqos->hqos_burst_enq, 0);
435 if (swq_pos >= n_swq)
437 hqos->swq_pos = swq_pos;
440 if (pkts_enq_len >= hqos->hqos_burst_enq)
442 u32 n_pkts = rte_eth_tx_burst (device_index,
445 (uint16_t) pkts_enq_len);
447 for (; n_pkts < pkts_enq_len; n_pkts++)
448 rte_pktmbuf_free (pkts_enq[n_pkts]);
458 if (
PREDICT_FALSE (flush_count == HQOS_FLUSH_COUNT_THRESHOLD))
460 rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
466 hqos->pkts_enq_len = pkts_enq_len;
467 hqos->flush_count = flush_count;
486 u32 n_devs =
vec_len (dm->devices_by_hqos_cpu[thread_index]);
492 if (dev_pos >= n_devs)
495 dpdk_device_and_queue_t *dq =
499 dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht;
501 u16 queue_id = dq->queue_id;
503 struct rte_mbuf **pkts_enq = hqos->pkts_enq;
504 struct rte_mbuf **pkts_deq = hqos->pkts_deq;
505 u32 pkts_enq_len = hqos->pkts_enq_len;
506 u32 swq_pos = hqos->swq_pos;
508 u32 flush_count = hqos->flush_count;
513 for (
i = 0;
i < n_swq;
i++)
516 struct rte_ring *swq = hqos->swq[swq_pos];
519 pkts_enq_len += rte_ring_sc_dequeue_burst (swq,
521 &pkts_enq[pkts_enq_len],
522 hqos->hqos_burst_enq, 0);
526 if (swq_pos >= n_swq)
528 hqos->swq_pos = swq_pos;
531 if (pkts_enq_len >= hqos->hqos_burst_enq)
533 rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
543 if (
PREDICT_FALSE (flush_count == HQOS_FLUSH_COUNT_THRESHOLD))
545 rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
551 hqos->pkts_enq_len = pkts_enq_len;
552 hqos->flush_count = flush_count;
558 u32 pkts_deq_len, n_pkts;
560 pkts_deq_len = rte_sched_port_dequeue (hqos->hqos,
562 hqos->hqos_burst_deq);
564 for (n_pkts = 0; n_pkts < pkts_deq_len;)
565 n_pkts += rte_eth_tx_burst (device_index,
568 (uint16_t) (pkts_deq_len - n_pkts));
597 (
"current I/O TX thread does not have any devices assigned to it");
599 if (DPDK_HQOS_DBG_BYPASS)
616 .name =
"hqos-threads",
617 .short_name =
"hqos-threads",
625 #define BITFIELD(byte_array, slab_pos, slab_mask, slab_shr) \ 627 u64 slab = *((u64 *) &byte_array[slab_pos]); \ 628 u64 val = (rte_be_to_cpu_64(slab) & slab_mask) >> slab_shr; \ 632 #define RTE_SCHED_PORT_HIERARCHY(subport, pipe, traffic_class, queue, color) \ 633 ((((u64) (queue)) & 0x3) | \ 634 ((((u64) (traffic_class)) & 0x3) << 2) | \ 635 ((((u64) (color)) & 0x3) << 4) | \ 636 ((((u64) (subport)) & 0xFFFF) << 16) | \ 637 ((((u64) (pipe)) & 0xFFFFFFFF) << 32)) 641 struct rte_mbuf **pkts,
u32 n_pkts)
645 for (i = 0; i < (n_pkts & (~0x3)); i += 4)
647 struct rte_mbuf *pkt0 = pkts[
i];
648 struct rte_mbuf *pkt1 = pkts[i + 1];
649 struct rte_mbuf *pkt2 = pkts[i + 2];
650 struct rte_mbuf *pkt3 = pkts[i + 3];
652 u8 *pkt0_data = rte_pktmbuf_mtod (pkt0,
u8 *);
653 u8 *pkt1_data = rte_pktmbuf_mtod (pkt1,
u8 *);
654 u8 *pkt2_data = rte_pktmbuf_mtod (pkt2,
u8 *);
655 u8 *pkt3_data = rte_pktmbuf_mtod (pkt3,
u8 *);
657 u64 pkt0_subport =
BITFIELD (pkt0_data, hqos->hqos_field0_slabpos,
658 hqos->hqos_field0_slabmask,
659 hqos->hqos_field0_slabshr);
660 u64 pkt0_pipe =
BITFIELD (pkt0_data, hqos->hqos_field1_slabpos,
661 hqos->hqos_field1_slabmask,
662 hqos->hqos_field1_slabshr);
663 u64 pkt0_dscp =
BITFIELD (pkt0_data, hqos->hqos_field2_slabpos,
664 hqos->hqos_field2_slabmask,
665 hqos->hqos_field2_slabshr);
666 u32 pkt0_tc = hqos->hqos_tc_table[pkt0_dscp & 0x3F] >> 2;
667 u32 pkt0_tc_q = hqos->hqos_tc_table[pkt0_dscp & 0x3F] & 0x3;
669 u64 pkt1_subport =
BITFIELD (pkt1_data, hqos->hqos_field0_slabpos,
670 hqos->hqos_field0_slabmask,
671 hqos->hqos_field0_slabshr);
672 u64 pkt1_pipe =
BITFIELD (pkt1_data, hqos->hqos_field1_slabpos,
673 hqos->hqos_field1_slabmask,
674 hqos->hqos_field1_slabshr);
675 u64 pkt1_dscp =
BITFIELD (pkt1_data, hqos->hqos_field2_slabpos,
676 hqos->hqos_field2_slabmask,
677 hqos->hqos_field2_slabshr);
678 u32 pkt1_tc = hqos->hqos_tc_table[pkt1_dscp & 0x3F] >> 2;
679 u32 pkt1_tc_q = hqos->hqos_tc_table[pkt1_dscp & 0x3F] & 0x3;
681 u64 pkt2_subport =
BITFIELD (pkt2_data, hqos->hqos_field0_slabpos,
682 hqos->hqos_field0_slabmask,
683 hqos->hqos_field0_slabshr);
684 u64 pkt2_pipe =
BITFIELD (pkt2_data, hqos->hqos_field1_slabpos,
685 hqos->hqos_field1_slabmask,
686 hqos->hqos_field1_slabshr);
687 u64 pkt2_dscp =
BITFIELD (pkt2_data, hqos->hqos_field2_slabpos,
688 hqos->hqos_field2_slabmask,
689 hqos->hqos_field2_slabshr);
690 u32 pkt2_tc = hqos->hqos_tc_table[pkt2_dscp & 0x3F] >> 2;
691 u32 pkt2_tc_q = hqos->hqos_tc_table[pkt2_dscp & 0x3F] & 0x3;
693 u64 pkt3_subport =
BITFIELD (pkt3_data, hqos->hqos_field0_slabpos,
694 hqos->hqos_field0_slabmask,
695 hqos->hqos_field0_slabshr);
696 u64 pkt3_pipe =
BITFIELD (pkt3_data, hqos->hqos_field1_slabpos,
697 hqos->hqos_field1_slabmask,
698 hqos->hqos_field1_slabshr);
699 u64 pkt3_dscp =
BITFIELD (pkt3_data, hqos->hqos_field2_slabpos,
700 hqos->hqos_field2_slabmask,
701 hqos->hqos_field2_slabshr);
702 u32 pkt3_tc = hqos->hqos_tc_table[pkt3_dscp & 0x3F] >> 2;
703 u32 pkt3_tc_q = hqos->hqos_tc_table[pkt3_dscp & 0x3F] & 0x3;
726 pkt0->hash.sched.lo = pkt0_sched & 0xFFFFFFFF;
727 pkt0->hash.sched.hi = pkt0_sched >> 32;
728 pkt1->hash.sched.lo = pkt1_sched & 0xFFFFFFFF;
729 pkt1->hash.sched.hi = pkt1_sched >> 32;
730 pkt2->hash.sched.lo = pkt2_sched & 0xFFFFFFFF;
731 pkt2->hash.sched.hi = pkt2_sched >> 32;
732 pkt3->hash.sched.lo = pkt3_sched & 0xFFFFFFFF;
733 pkt3->hash.sched.hi = pkt3_sched >> 32;
736 for (; i < n_pkts; i++)
738 struct rte_mbuf *pkt = pkts[
i];
740 u8 *pkt_data = rte_pktmbuf_mtod (pkt,
u8 *);
742 u64 pkt_subport =
BITFIELD (pkt_data, hqos->hqos_field0_slabpos,
743 hqos->hqos_field0_slabmask,
744 hqos->hqos_field0_slabshr);
745 u64 pkt_pipe =
BITFIELD (pkt_data, hqos->hqos_field1_slabpos,
746 hqos->hqos_field1_slabmask,
747 hqos->hqos_field1_slabshr);
748 u64 pkt_dscp =
BITFIELD (pkt_data, hqos->hqos_field2_slabpos,
749 hqos->hqos_field2_slabmask,
750 hqos->hqos_field2_slabshr);
751 u32 pkt_tc = hqos->hqos_tc_table[pkt_dscp & 0x3F] >> 2;
752 u32 pkt_tc_q = hqos->hqos_tc_table[pkt_dscp & 0x3F] & 0x3;
760 pkt->hash.sched.lo = pkt_sched & 0xFFFFFFFF;
761 pkt->hash.sched.hi = pkt_sched >> 32;
VLIB_REGISTER_THREAD(hqos_thread_reg, static)
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
static struct rte_sched_subport_params hqos_subport_params_default
#define count_leading_zeros(x)
Optimized string handling code, including c11-compliant "safe C library" variants.
#define clib_error(format, args...)
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
#define BITFIELD(byte_array, slab_pos, slab_mask, slab_shr)
#define count_trailing_zeros(x)
static void vlib_worker_thread_barrier_check(void)
#define static_always_inline
static dpdk_device_config_hqos_t hqos_params_default
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
static_always_inline void dpdk_hqos_thread_internal_hqos_dbg_bypass(vlib_main_t *vm)
void clib_time_init(clib_time_t *c)
int dpdk_hqos_validate_mask(u64 mask, u32 n)
static_always_inline uword vlib_get_thread_index(void)
sll srl srl sll sra u16x4 i
static void * clib_mem_set_heap(void *heap)
void vlib_worker_thread_init(vlib_worker_thread_t *w)
void dpdk_hqos_thread_fn(void *arg)
clib_error_t * dpdk_port_setup_hqos(dpdk_device_t *xd, dpdk_device_config_hqos_t *hqos)
Bitmaps built as vectors of machine words.
static_always_inline void dpdk_hqos_thread_internal(vlib_main_t *vm)
uword * thread_registrations_by_name
static vlib_main_t * vlib_get_main(void)
void dpdk_hqos_metadata_set(dpdk_device_hqos_per_worker_thread_t *hqos, struct rte_mbuf **pkts, u32 n_pkts)
void dpdk_device_config_hqos_default(dpdk_device_config_hqos_t *hqos)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
void dpdk_hqos_thread(vlib_worker_thread_t *w)
#define hash_get_mem(h, key)
static vlib_thread_main_t * vlib_get_thread_main()
#define RTE_SCHED_PORT_HIERARCHY(subport, pipe, traffic_class, queue, color)
#define CLIB_CACHE_LINE_BYTES
void dpdk_device_config_hqos_pipe_profile_default(dpdk_device_config_hqos_t *hqos, u32 pipe_profile_id)
volatile u32 worker_thread_release
static struct rte_sched_pipe_params hqos_pipe_params_default
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".