FD.io VPP  v17.04.2-2-ga8f93f8
Vector Packet Processing
hqos.c
Go to the documentation of this file.
1 /*
2  * Copyright(c) 2016 Intel Corporation. All rights reserved.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include <stdio.h>
16 #include <stdlib.h>
17 #include <unistd.h>
18 #include <sys/stat.h>
19 #include <sys/mount.h>
20 #include <string.h>
21 #include <fcntl.h>
22 
23 #include <vppinfra/vec.h>
24 #include <vppinfra/error.h>
25 #include <vppinfra/format.h>
26 #include <vppinfra/bitmap.h>
27 
28 #include <vnet/vnet.h>
29 #include <vnet/ethernet/ethernet.h>
30 #include <dpdk/device/dpdk.h>
31 
32 #include <vlib/unix/physmem.h>
33 #include <vlib/pci/pci.h>
34 #include <vlibmemory/api.h>
35 #include <vlibmemory/vl_memory_msg_enum.h> /* enumerate all vlib messages */
36 
37 #define vl_typedefs /* define message structures */
39 #undef vl_typedefs
40 
41 /* instantiate all the print functions we know about */
42 #define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
43 #define vl_printfun
45 #undef vl_printfun
46 
47 #include <dpdk/device/dpdk_priv.h>
48 
50 
51 /***
52  *
53  * HQoS default configuration values
54  *
55  ***/
56 
57 static dpdk_device_config_hqos_t hqos_params_default = {
58  .hqos_thread_valid = 0,
59 
60  .swq_size = 4096,
61  .burst_enq = 256,
62  .burst_deq = 220,
63 
64  /*
65  * Packet field to identify the subport.
66  *
67  * Default value: Since only one subport is defined by default (see below:
68  * n_subports_per_port = 1), the subport ID is hardcoded to 0.
69  */
70  .pktfield0_slabpos = 0,
71  .pktfield0_slabmask = 0,
72 
73  /*
74  * Packet field to identify the pipe.
75  *
76  * Default value: Assuming Ethernet/IPv4/UDP packets, UDP payload bits 12 .. 23
77  */
78  .pktfield1_slabpos = 40,
79  .pktfield1_slabmask = 0x0000000FFF000000LLU,
80 
81  /* Packet field used as index into TC translation table to identify the traffic
82  * class and queue.
83  *
84  * Default value: Assuming Ethernet/IPv4 packets, IPv4 DSCP field
85  */
86  .pktfield2_slabpos = 8,
87  .pktfield2_slabmask = 0x00000000000000FCLLU,
88  .tc_table = {
89  0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
90  0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
91  0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
92  0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
93  },
94 
95  /* port */
96  .port = {
97  .name = NULL, /* Set at init */
98  .socket = 0, /* Set at init */
99  .rate = 1250000000, /* Assuming 10GbE port */
100  .mtu = 14 + 1500, /* Assuming Ethernet/IPv4 pkt (Ethernet FCS not included) */
101  .frame_overhead = RTE_SCHED_FRAME_OVERHEAD_DEFAULT,
102  .n_subports_per_port = 1,
103  .n_pipes_per_subport = 4096,
104  .qsize = {64, 64, 64, 64},
105  .pipe_profiles = NULL, /* Set at config */
106  .n_pipe_profiles = 1,
107 
108 #ifdef RTE_SCHED_RED
109  .red_params = {
110  /* Traffic Class 0 Colors Green / Yellow / Red */
111  [0][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
112  10,.wq_log2 = 9},
113  [0][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
114  10,.wq_log2 = 9},
115  [0][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
116  10,.wq_log2 = 9},
117 
118  /* Traffic Class 1 - Colors Green / Yellow / Red */
119  [1][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
120  10,.wq_log2 = 9},
121  [1][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
122  10,.wq_log2 = 9},
123  [1][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
124  10,.wq_log2 = 9},
125 
126  /* Traffic Class 2 - Colors Green / Yellow / Red */
127  [2][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
128  10,.wq_log2 = 9},
129  [2][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
130  10,.wq_log2 = 9},
131  [2][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
132  10,.wq_log2 = 9},
133 
134  /* Traffic Class 3 - Colors Green / Yellow / Red */
135  [3][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
136  10,.wq_log2 = 9},
137  [3][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
138  10,.wq_log2 = 9},
139  [3][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
140  10,.wq_log2 = 9}
141  },
142 #endif /* RTE_SCHED_RED */
143  },
144 };
145 
146 static struct rte_sched_subport_params hqos_subport_params_default = {
147  .tb_rate = 1250000000, /* 10GbE line rate (measured in bytes/second) */
148  .tb_size = 1000000,
149  .tc_rate = {1250000000, 1250000000, 1250000000, 1250000000},
150  .tc_period = 10,
151 };
152 
153 static struct rte_sched_pipe_params hqos_pipe_params_default = {
154  .tb_rate = 305175, /* 10GbE line rate divided by 4K pipes */
155  .tb_size = 1000000,
156  .tc_rate = {305175, 305175, 305175, 305175},
157  .tc_period = 40,
158 #ifdef RTE_SCHED_SUBPORT_TC_OV
159  .tc_ov_weight = 1,
160 #endif
161  .wrr_weights = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
162 };
163 
164 /***
165  *
166  * HQoS configuration
167  *
168  ***/
169 
170 int
172 {
173  int count = __builtin_popcountll (mask);
174  int pos_lead = sizeof (u64) * 8 - __builtin_clzll (mask);
175  int pos_trail = __builtin_ctzll (mask);
176  int count_expected = __builtin_popcount (n - 1);
177 
178  /* Handle the exceptions */
179  if (n == 0)
180  return -1; /* Error */
181 
182  if ((mask == 0) && (n == 1))
183  return 0; /* OK */
184 
185  if (((mask == 0) && (n != 1)) || ((mask != 0) && (n == 1)))
186  return -2; /* Error */
187 
188  /* Check that mask is contiguous */
189  if ((pos_lead - pos_trail) != count)
190  return -3; /* Error */
191 
192  /* Check that mask contains the expected number of bits set */
193  if (count != count_expected)
194  return -4; /* Error */
195 
196  return 0; /* OK */
197 }
198 
199 void
201  hqos, u32 pipe_profile_id)
202 {
203  memcpy (&hqos->pipe[pipe_profile_id], &hqos_pipe_params_default,
204  sizeof (hqos_pipe_params_default));
205 }
206 
207 void
209 {
210  struct rte_sched_subport_params *subport_params;
211  struct rte_sched_pipe_params *pipe_params;
212  u32 *pipe_map;
213  u32 i;
214 
215  memcpy (hqos, &hqos_params_default, sizeof (hqos_params_default));
216 
217  /* pipe */
218  vec_add2 (hqos->pipe, pipe_params, hqos->port.n_pipe_profiles);
219 
220  for (i = 0; i < vec_len (hqos->pipe); i++)
221  memcpy (&pipe_params[i],
223 
224  hqos->port.pipe_profiles = hqos->pipe;
225 
226  /* subport */
227  vec_add2 (hqos->subport, subport_params, hqos->port.n_subports_per_port);
228 
229  for (i = 0; i < vec_len (hqos->subport); i++)
230  memcpy (&subport_params[i],
232  sizeof (hqos_subport_params_default));
233 
234  /* pipe profile */
235  vec_add2 (hqos->pipe_map,
236  pipe_map,
237  hqos->port.n_subports_per_port * hqos->port.n_pipes_per_subport);
238 
239  for (i = 0; i < vec_len (hqos->pipe_map); i++)
240  pipe_map[i] = 0;
241 }
242 
243 /***
244  *
245  * HQoS init
246  *
247  ***/
248 
249 clib_error_t *
251 {
253  char name[32];
254  u32 subport_id, i;
255  int rv;
256 
257  /* Detect the set of worker threads */
258  int worker_thread_first = 0;
259  int worker_thread_count = 0;
260 
261  uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers");
263  p ? (vlib_thread_registration_t *) p[0] : 0;
264 
265  if (tr && tr->count > 0)
266  {
267  worker_thread_first = tr->first_index;
268  worker_thread_count = tr->count;
269  }
270 
271  /* Allocate the per-thread device data array */
274  memset (xd->hqos_wt, 0, tm->n_vlib_mains * sizeof (xd->hqos_wt[0]));
275 
277  memset (xd->hqos_ht, 0, sizeof (xd->hqos_ht[0]));
278 
279  /* Allocate space for one SWQ per worker thread in the I/O TX thread data structure */
280  vec_validate (xd->hqos_ht->swq, worker_thread_count);
281 
282  /* SWQ */
283  for (i = 0; i < worker_thread_count + 1; i++)
284  {
285  u32 swq_flags = RING_F_SP_ENQ | RING_F_SC_DEQ;
286 
287  snprintf (name, sizeof (name), "SWQ-worker%u-to-device%u", i,
288  xd->device_index);
289  xd->hqos_ht->swq[i] =
290  rte_ring_create (name, hqos->swq_size, xd->cpu_socket, swq_flags);
291  if (xd->hqos_ht->swq[i] == NULL)
292  return clib_error_return (0,
293  "SWQ-worker%u-to-device%u: rte_ring_create err",
294  i, xd->device_index);
295  }
296 
297  /*
298  * HQoS
299  */
300 
301  /* HQoS port */
302  snprintf (name, sizeof (name), "HQoS%u", xd->device_index);
303  hqos->port.name = strdup (name);
304  if (hqos->port.name == NULL)
305  return clib_error_return (0, "HQoS%u: strdup err", xd->device_index);
306 
307  hqos->port.socket = rte_eth_dev_socket_id (xd->device_index);
308  if (hqos->port.socket == SOCKET_ID_ANY)
309  hqos->port.socket = 0;
310 
311  xd->hqos_ht->hqos = rte_sched_port_config (&hqos->port);
312  if (xd->hqos_ht->hqos == NULL)
313  return clib_error_return (0, "HQoS%u: rte_sched_port_config err",
314  xd->device_index);
315 
316  /* HQoS subport */
317  for (subport_id = 0; subport_id < hqos->port.n_subports_per_port;
318  subport_id++)
319  {
320  u32 pipe_id;
321 
322  rv =
323  rte_sched_subport_config (xd->hqos_ht->hqos, subport_id,
324  &hqos->subport[subport_id]);
325  if (rv)
326  return clib_error_return (0,
327  "HQoS%u subport %u: rte_sched_subport_config err (%d)",
328  xd->device_index, subport_id, rv);
329 
330  /* HQoS pipe */
331  for (pipe_id = 0; pipe_id < hqos->port.n_pipes_per_subport; pipe_id++)
332  {
333  u32 pos = subport_id * hqos->port.n_pipes_per_subport + pipe_id;
334  u32 profile_id = hqos->pipe_map[pos];
335 
336  rv =
337  rte_sched_pipe_config (xd->hqos_ht->hqos, subport_id, pipe_id,
338  profile_id);
339  if (rv)
340  return clib_error_return (0,
341  "HQoS%u subport %u pipe %u: rte_sched_pipe_config err (%d)",
342  xd->device_index, subport_id, pipe_id,
343  rv);
344  }
345  }
346 
347  /* Set up per-thread device data for the I/O TX thread */
348  xd->hqos_ht->hqos_burst_enq = hqos->burst_enq;
349  xd->hqos_ht->hqos_burst_deq = hqos->burst_deq;
350  vec_validate (xd->hqos_ht->pkts_enq, 2 * hqos->burst_enq - 1);
351  vec_validate (xd->hqos_ht->pkts_deq, hqos->burst_deq - 1);
352  xd->hqos_ht->pkts_enq_len = 0;
353  xd->hqos_ht->swq_pos = 0;
354  xd->hqos_ht->flush_count = 0;
355 
356  /* Set up per-thread device data for each worker thread */
357  for (i = 0; i < worker_thread_count + 1; i++)
358  {
359  u32 tid;
360  if (i)
361  tid = worker_thread_first + (i - 1);
362  else
363  tid = i;
364 
365  xd->hqos_wt[tid].swq = xd->hqos_ht->swq[i];
368  xd->hqos_wt[tid].hqos_field0_slabshr =
369  __builtin_ctzll (hqos->pktfield0_slabmask);
372  xd->hqos_wt[tid].hqos_field1_slabshr =
373  __builtin_ctzll (hqos->pktfield1_slabmask);
376  xd->hqos_wt[tid].hqos_field2_slabshr =
377  __builtin_ctzll (hqos->pktfield2_slabmask);
378  memcpy (xd->hqos_wt[tid].hqos_tc_table, hqos->tc_table,
379  sizeof (hqos->tc_table));
380  }
381 
382  return 0;
383 }
384 
385 /***
386  *
387  * HQoS run-time
388  *
389  ***/
390 /*
391  * dpdk_hqos_thread - Contains the main loop of an HQoS thread.
392  *
393  * w
394  * Information for the current thread
395  */
398 {
399  dpdk_main_t *dm = &dpdk_main;
400  u32 cpu_index = vm->cpu_index;
401  u32 dev_pos;
402 
403  dev_pos = 0;
404  while (1)
405  {
407 
408  u32 n_devs = vec_len (dm->devices_by_hqos_cpu[cpu_index]);
409  if (dev_pos >= n_devs)
410  dev_pos = 0;
411 
413  vec_elt_at_index (dm->devices_by_hqos_cpu[cpu_index], dev_pos);
414  dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device);
415 
417  u32 device_index = xd->device_index;
418  u16 queue_id = dq->queue_id;
419 
420  struct rte_mbuf **pkts_enq = hqos->pkts_enq;
421  u32 pkts_enq_len = hqos->pkts_enq_len;
422  u32 swq_pos = hqos->swq_pos;
423  u32 n_swq = vec_len (hqos->swq), i;
424  u32 flush_count = hqos->flush_count;
425 
426  for (i = 0; i < n_swq; i++)
427  {
428  /* Get current SWQ for this device */
429  struct rte_ring *swq = hqos->swq[swq_pos];
430 
431  /* Read SWQ burst to packet buffer of this device */
432  pkts_enq_len += rte_ring_sc_dequeue_burst (swq,
433  (void **)
434  &pkts_enq[pkts_enq_len],
435  hqos->hqos_burst_enq);
436 
437  /* Get next SWQ for this device */
438  swq_pos++;
439  if (swq_pos >= n_swq)
440  swq_pos = 0;
441  hqos->swq_pos = swq_pos;
442 
443  /* HWQ TX enqueue when burst available */
444  if (pkts_enq_len >= hqos->hqos_burst_enq)
445  {
446  u32 n_pkts = rte_eth_tx_burst (device_index,
447  (uint16_t) queue_id,
448  pkts_enq,
449  (uint16_t) pkts_enq_len);
450 
451  for (; n_pkts < pkts_enq_len; n_pkts++)
452  rte_pktmbuf_free (pkts_enq[n_pkts]);
453 
454  pkts_enq_len = 0;
455  flush_count = 0;
456  break;
457  }
458  }
459  if (pkts_enq_len)
460  {
461  flush_count++;
462  if (PREDICT_FALSE (flush_count == HQOS_FLUSH_COUNT_THRESHOLD))
463  {
464  rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
465 
466  pkts_enq_len = 0;
467  flush_count = 0;
468  }
469  }
470  hqos->pkts_enq_len = pkts_enq_len;
471  hqos->flush_count = flush_count;
472 
473  /* Advance to next device */
474  dev_pos++;
475  }
476 }
477 
480 {
481  dpdk_main_t *dm = &dpdk_main;
482  u32 cpu_index = vm->cpu_index;
483  u32 dev_pos;
484 
485  dev_pos = 0;
486  while (1)
487  {
489 
490  u32 n_devs = vec_len (dm->devices_by_hqos_cpu[cpu_index]);
491  if (PREDICT_FALSE (n_devs == 0))
492  {
493  dev_pos = 0;
494  continue;
495  }
496  if (dev_pos >= n_devs)
497  dev_pos = 0;
498 
500  vec_elt_at_index (dm->devices_by_hqos_cpu[cpu_index], dev_pos);
501  dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device);
502 
504  u32 device_index = xd->device_index;
505  u16 queue_id = dq->queue_id;
506 
507  struct rte_mbuf **pkts_enq = hqos->pkts_enq;
508  struct rte_mbuf **pkts_deq = hqos->pkts_deq;
509  u32 pkts_enq_len = hqos->pkts_enq_len;
510  u32 swq_pos = hqos->swq_pos;
511  u32 n_swq = vec_len (hqos->swq), i;
512  u32 flush_count = hqos->flush_count;
513 
514  /*
515  * SWQ dequeue and HQoS enqueue for current device
516  */
517  for (i = 0; i < n_swq; i++)
518  {
519  /* Get current SWQ for this device */
520  struct rte_ring *swq = hqos->swq[swq_pos];
521 
522  /* Read SWQ burst to packet buffer of this device */
523  pkts_enq_len += rte_ring_sc_dequeue_burst (swq,
524  (void **)
525  &pkts_enq[pkts_enq_len],
526  hqos->hqos_burst_enq);
527 
528  /* Get next SWQ for this device */
529  swq_pos++;
530  if (swq_pos >= n_swq)
531  swq_pos = 0;
532  hqos->swq_pos = swq_pos;
533 
534  /* HQoS enqueue when burst available */
535  if (pkts_enq_len >= hqos->hqos_burst_enq)
536  {
537  rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
538 
539  pkts_enq_len = 0;
540  flush_count = 0;
541  break;
542  }
543  }
544  if (pkts_enq_len)
545  {
546  flush_count++;
547  if (PREDICT_FALSE (flush_count == HQOS_FLUSH_COUNT_THRESHOLD))
548  {
549  rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
550 
551  pkts_enq_len = 0;
552  flush_count = 0;
553  }
554  }
555  hqos->pkts_enq_len = pkts_enq_len;
556  hqos->flush_count = flush_count;
557 
558  /*
559  * HQoS dequeue and HWQ TX enqueue for current device
560  */
561  {
562  u32 pkts_deq_len, n_pkts;
563 
564  pkts_deq_len = rte_sched_port_dequeue (hqos->hqos,
565  pkts_deq,
566  hqos->hqos_burst_deq);
567 
568  for (n_pkts = 0; n_pkts < pkts_deq_len;)
569  n_pkts += rte_eth_tx_burst (device_index,
570  (uint16_t) queue_id,
571  &pkts_deq[n_pkts],
572  (uint16_t) (pkts_deq_len - n_pkts));
573  }
574 
575  /* Advance to next device */
576  dev_pos++;
577  }
578 }
579 
580 void
582 {
583  vlib_main_t *vm;
585  dpdk_main_t *dm = &dpdk_main;
586 
587  vm = vlib_get_main ();
588 
589  ASSERT (vm->cpu_index == os_get_cpu_number ());
590 
591  clib_time_init (&vm->clib_time);
593 
594  /* Wait until the dpdk init sequence is complete */
595  while (tm->worker_thread_release == 0)
597 
598  if (vec_len (dm->devices_by_hqos_cpu[vm->cpu_index]) == 0)
599  return
600  clib_error
601  ("current I/O TX thread does not have any devices assigned to it");
602 
605  else
607 }
608 
609 void
611 {
614  dpdk_hqos_thread (w);
615 }
616 
617 /* *INDENT-OFF* */
618 VLIB_REGISTER_THREAD (hqos_thread_reg, static) =
619 {
620  .name = "hqos-threads",
621  .short_name = "hqos-threads",
622  .function = dpdk_hqos_thread_fn,
623 };
624 /* *INDENT-ON* */
625 
626 /*
627  * HQoS run-time code to be called by the worker threads
628  */
629 #define BITFIELD(byte_array, slab_pos, slab_mask, slab_shr) \
630 ({ \
631  u64 slab = *((u64 *) &byte_array[slab_pos]); \
632  u64 val = (rte_be_to_cpu_64(slab) & slab_mask) >> slab_shr; \
633  val; \
634 })
635 
636 #define RTE_SCHED_PORT_HIERARCHY(subport, pipe, traffic_class, queue, color) \
637  ((((u64) (queue)) & 0x3) | \
638  ((((u64) (traffic_class)) & 0x3) << 2) | \
639  ((((u64) (color)) & 0x3) << 4) | \
640  ((((u64) (subport)) & 0xFFFF) << 16) | \
641  ((((u64) (pipe)) & 0xFFFFFFFF) << 32))
642 
643 void
645  struct rte_mbuf **pkts, u32 n_pkts)
646 {
647  u32 i;
648 
649  for (i = 0; i < (n_pkts & (~0x3)); i += 4)
650  {
651  struct rte_mbuf *pkt0 = pkts[i];
652  struct rte_mbuf *pkt1 = pkts[i + 1];
653  struct rte_mbuf *pkt2 = pkts[i + 2];
654  struct rte_mbuf *pkt3 = pkts[i + 3];
655 
656  u8 *pkt0_data = rte_pktmbuf_mtod (pkt0, u8 *);
657  u8 *pkt1_data = rte_pktmbuf_mtod (pkt1, u8 *);
658  u8 *pkt2_data = rte_pktmbuf_mtod (pkt2, u8 *);
659  u8 *pkt3_data = rte_pktmbuf_mtod (pkt3, u8 *);
660 
661  u64 pkt0_subport = BITFIELD (pkt0_data, hqos->hqos_field0_slabpos,
662  hqos->hqos_field0_slabmask,
663  hqos->hqos_field0_slabshr);
664  u64 pkt0_pipe = BITFIELD (pkt0_data, hqos->hqos_field1_slabpos,
665  hqos->hqos_field1_slabmask,
666  hqos->hqos_field1_slabshr);
667  u64 pkt0_dscp = BITFIELD (pkt0_data, hqos->hqos_field2_slabpos,
668  hqos->hqos_field2_slabmask,
669  hqos->hqos_field2_slabshr);
670  u32 pkt0_tc = hqos->hqos_tc_table[pkt0_dscp & 0x3F] >> 2;
671  u32 pkt0_tc_q = hqos->hqos_tc_table[pkt0_dscp & 0x3F] & 0x3;
672 
673  u64 pkt1_subport = BITFIELD (pkt1_data, hqos->hqos_field0_slabpos,
674  hqos->hqos_field0_slabmask,
675  hqos->hqos_field0_slabshr);
676  u64 pkt1_pipe = BITFIELD (pkt1_data, hqos->hqos_field1_slabpos,
677  hqos->hqos_field1_slabmask,
678  hqos->hqos_field1_slabshr);
679  u64 pkt1_dscp = BITFIELD (pkt1_data, hqos->hqos_field2_slabpos,
680  hqos->hqos_field2_slabmask,
681  hqos->hqos_field2_slabshr);
682  u32 pkt1_tc = hqos->hqos_tc_table[pkt1_dscp & 0x3F] >> 2;
683  u32 pkt1_tc_q = hqos->hqos_tc_table[pkt1_dscp & 0x3F] & 0x3;
684 
685  u64 pkt2_subport = BITFIELD (pkt2_data, hqos->hqos_field0_slabpos,
686  hqos->hqos_field0_slabmask,
687  hqos->hqos_field0_slabshr);
688  u64 pkt2_pipe = BITFIELD (pkt2_data, hqos->hqos_field1_slabpos,
689  hqos->hqos_field1_slabmask,
690  hqos->hqos_field1_slabshr);
691  u64 pkt2_dscp = BITFIELD (pkt2_data, hqos->hqos_field2_slabpos,
692  hqos->hqos_field2_slabmask,
693  hqos->hqos_field2_slabshr);
694  u32 pkt2_tc = hqos->hqos_tc_table[pkt2_dscp & 0x3F] >> 2;
695  u32 pkt2_tc_q = hqos->hqos_tc_table[pkt2_dscp & 0x3F] & 0x3;
696 
697  u64 pkt3_subport = BITFIELD (pkt3_data, hqos->hqos_field0_slabpos,
698  hqos->hqos_field0_slabmask,
699  hqos->hqos_field0_slabshr);
700  u64 pkt3_pipe = BITFIELD (pkt3_data, hqos->hqos_field1_slabpos,
701  hqos->hqos_field1_slabmask,
702  hqos->hqos_field1_slabshr);
703  u64 pkt3_dscp = BITFIELD (pkt3_data, hqos->hqos_field2_slabpos,
704  hqos->hqos_field2_slabmask,
705  hqos->hqos_field2_slabshr);
706  u32 pkt3_tc = hqos->hqos_tc_table[pkt3_dscp & 0x3F] >> 2;
707  u32 pkt3_tc_q = hqos->hqos_tc_table[pkt3_dscp & 0x3F] & 0x3;
708 
709  u64 pkt0_sched = RTE_SCHED_PORT_HIERARCHY (pkt0_subport,
710  pkt0_pipe,
711  pkt0_tc,
712  pkt0_tc_q,
713  0);
714  u64 pkt1_sched = RTE_SCHED_PORT_HIERARCHY (pkt1_subport,
715  pkt1_pipe,
716  pkt1_tc,
717  pkt1_tc_q,
718  0);
719  u64 pkt2_sched = RTE_SCHED_PORT_HIERARCHY (pkt2_subport,
720  pkt2_pipe,
721  pkt2_tc,
722  pkt2_tc_q,
723  0);
724  u64 pkt3_sched = RTE_SCHED_PORT_HIERARCHY (pkt3_subport,
725  pkt3_pipe,
726  pkt3_tc,
727  pkt3_tc_q,
728  0);
729 
730  pkt0->hash.sched.lo = pkt0_sched & 0xFFFFFFFF;
731  pkt0->hash.sched.hi = pkt0_sched >> 32;
732  pkt1->hash.sched.lo = pkt1_sched & 0xFFFFFFFF;
733  pkt1->hash.sched.hi = pkt1_sched >> 32;
734  pkt2->hash.sched.lo = pkt2_sched & 0xFFFFFFFF;
735  pkt2->hash.sched.hi = pkt2_sched >> 32;
736  pkt3->hash.sched.lo = pkt3_sched & 0xFFFFFFFF;
737  pkt3->hash.sched.hi = pkt3_sched >> 32;
738  }
739 
740  for (; i < n_pkts; i++)
741  {
742  struct rte_mbuf *pkt = pkts[i];
743 
744  u8 *pkt_data = rte_pktmbuf_mtod (pkt, u8 *);
745 
746  u64 pkt_subport = BITFIELD (pkt_data, hqos->hqos_field0_slabpos,
747  hqos->hqos_field0_slabmask,
748  hqos->hqos_field0_slabshr);
749  u64 pkt_pipe = BITFIELD (pkt_data, hqos->hqos_field1_slabpos,
750  hqos->hqos_field1_slabmask,
751  hqos->hqos_field1_slabshr);
752  u64 pkt_dscp = BITFIELD (pkt_data, hqos->hqos_field2_slabpos,
753  hqos->hqos_field2_slabmask,
754  hqos->hqos_field2_slabshr);
755  u32 pkt_tc = hqos->hqos_tc_table[pkt_dscp & 0x3F] >> 2;
756  u32 pkt_tc_q = hqos->hqos_tc_table[pkt_dscp & 0x3F] & 0x3;
757 
758  u64 pkt_sched = RTE_SCHED_PORT_HIERARCHY (pkt_subport,
759  pkt_pipe,
760  pkt_tc,
761  pkt_tc_q,
762  0);
763 
764  pkt->hash.sched.lo = pkt_sched & 0xFFFFFFFF;
765  pkt->hash.sched.hi = pkt_sched >> 32;
766  }
767 }
768 
769 /*
770  * fd.io coding-style-patch-verification: ON
771  *
772  * Local Variables:
773  * eval: (c-set-style "gnu")
774  * End:
775  */
VLIB_REGISTER_THREAD(hqos_thread_reg, static)
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:436
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:343
static struct rte_sched_subport_params hqos_subport_params_default
Definition: hqos.c:146
dpdk_main_t dpdk_main
Definition: hqos.c:49
#define clib_error(format, args...)
Definition: error.h:62
#define NULL
Definition: clib.h:55
dpdk_device_and_queue_t ** devices_by_hqos_cpu
Definition: dpdk.h:357
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:561
struct rte_sched_port_params port
Definition: dpdk.h:278
struct rte_mbuf ** pkts_enq
Definition: dpdk.h:160
struct rte_ring ** swq
Definition: dpdk.h:159
struct rte_sched_port * hqos
Definition: dpdk.h:162
clib_time_t clib_time
Definition: main.h:62
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:447
#define BITFIELD(byte_array, slab_pos, slab_mask, slab_shr)
Definition: hqos.c:629
static void vlib_worker_thread_barrier_check(void)
Definition: threads.h:214
struct rte_sched_pipe_params * pipe
Definition: dpdk.h:280
#define static_always_inline
Definition: clib.h:85
struct rte_mbuf ** pkts_deq
Definition: dpdk.h:161
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:111
u32 cpu_index
Definition: main.h:159
unsigned long u64
Definition: types.h:89
u32 device_index
Definition: dpdk.h:176
dpdk_device_hqos_per_worker_thread_t * hqos_wt
Definition: dpdk.h:220
void * thread_mheap
Definition: threads.h:95
uword os_get_cpu_number(void)
Definition: unix-misc.c:224
static_always_inline void dpdk_hqos_thread_internal_hqos_dbg_bypass(vlib_main_t *vm)
Definition: hqos.c:397
#define PREDICT_FALSE(x)
Definition: clib.h:97
void clib_time_init(clib_time_t *c)
Definition: time.c:175
int dpdk_hqos_validate_mask(u64 mask, u32 n)
Definition: hqos.c:171
#define HQOS_FLUSH_COUNT_THRESHOLD
Definition: dpdk.h:258
dpdk_device_t * devices
Definition: dpdk.h:355
vlib_main_t * vm
Definition: buffer.c:276
static void * clib_mem_set_heap(void *heap)
Definition: mem.h:223
void vlib_worker_thread_init(vlib_worker_thread_t *w)
Definition: threads.c:455
#define ASSERT(truth)
unsigned int u32
Definition: types.h:88
#define DPDK_HQOS_DBG_BYPASS
Definition: dpdk.h:254
void dpdk_hqos_thread_fn(void *arg)
Definition: hqos.c:610
clib_error_t * dpdk_port_setup_hqos(dpdk_device_t *xd, dpdk_device_config_hqos_t *hqos)
Definition: hqos.c:250
Bitmaps built as vectors of machine words.
dpdk_device_hqos_per_hqos_thread_t * hqos_ht
Definition: dpdk.h:221
static_always_inline void dpdk_hqos_thread_internal(vlib_main_t *vm)
Definition: hqos.c:479
uword * thread_registrations_by_name
Definition: threads.h:276
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
void dpdk_hqos_metadata_set(dpdk_device_hqos_per_worker_thread_t *hqos, struct rte_mbuf **pkts, u32 n_pkts)
Definition: hqos.c:644
u64 uword
Definition: types.h:112
void dpdk_device_config_hqos_default(dpdk_device_config_hqos_t *hqos)
Definition: hqos.c:208
unsigned short u16
Definition: types.h:57
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
unsigned char u8
Definition: types.h:56
uint32_t * pipe_map
Definition: dpdk.h:281
void dpdk_hqos_thread(vlib_worker_thread_t *w)
Definition: hqos.c:581
#define hash_get_mem(h, key)
Definition: hash.h:268
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
struct rte_sched_subport_params * subport
Definition: dpdk.h:279
i8 cpu_socket
Definition: dpdk.h:192
#define RTE_SCHED_PORT_HIERARCHY(subport, pipe, traffic_class, queue, color)
Definition: hqos.c:636
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
void dpdk_device_config_hqos_pipe_profile_default(dpdk_device_config_hqos_t *hqos, u32 pipe_profile_id)
Definition: hqos.c:200
volatile u32 worker_thread_release
Definition: threads.h:317
static struct rte_sched_pipe_params hqos_pipe_params_default
Definition: hqos.c:153
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".