FD.io VPP  v18.01.2-1-g9b554f3
Vector Packet Processing
vnet_classify.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #ifndef __included_vnet_classify_h__
16 #define __included_vnet_classify_h__
17 
18 #include <stdarg.h>
19 
20 #include <vlib/vlib.h>
21 #include <vnet/vnet.h>
22 #include <vnet/pg/pg.h>
23 #include <vnet/ethernet/ethernet.h>
24 #include <vnet/ethernet/packet.h>
25 #include <vnet/ip/ip_packet.h>
26 #include <vnet/ip/ip4_packet.h>
27 #include <vnet/ip/ip6_packet.h>
28 #include <vlib/cli.h>
29 #include <vnet/l2/l2_input.h>
30 #include <vnet/l2/feat_bitmap.h>
31 #include <vnet/api_errno.h> /* for API error numbers */
32 
33 #include <vppinfra/error.h>
34 #include <vppinfra/hash.h>
35 #include <vppinfra/cache.h>
36 #include <vppinfra/xxhash.h>
37 
40 
41 #define CLASSIFY_TRACE 0
42 
43 #if !defined( __aarch64__) && !defined(__arm__)
44 #define CLASSIFY_USE_SSE //Allow usage of SSE operations
45 #endif
46 
47 #define U32X4_ALIGNED(p) PREDICT_TRUE((((intptr_t)p) & 0xf) == 0)
48 
49 /*
50  * Classify table option to process packets
51  * CLASSIFY_FLAG_USE_CURR_DATA:
52  * - classify packets starting from VPP node’s current data pointer
53  */
54 #define CLASSIFY_FLAG_USE_CURR_DATA 1
55 
56 /*
57  * Classify session action
58  * CLASSIFY_ACTION_SET_IP4_FIB_INDEX:
59  * - Classified IP packets will be looked up
60  * from the specified ipv4 fib table
61  * CLASSIFY_ACTION_SET_IP6_FIB_INDEX:
62  * - Classified IP packets will be looked up
63  * from the specified ipv6 fib table
64  */
66 {
70 } __attribute__ ((packed)) vnet_classify_action_t;
71 
72 struct _vnet_classify_main;
73 typedef struct _vnet_classify_main vnet_classify_main_t;
74 
75 #define foreach_size_in_u32x4 \
76 _(1) \
77 _(2) \
78 _(3) \
79 _(4) \
80 _(5)
81 
82 /* *INDENT-OFF* */
83 typedef CLIB_PACKED(struct _vnet_classify_entry {
84  /* Graph node next index */
85  u32 next_index;
86 
87  /* put into vnet_buffer(b)->l2_classfy.opaque_index */
88  union {
89  struct {
90  u32 opaque_index;
91  /* advance on hit, note it's a signed quantity... */
92  i32 advance;
93  };
94  u64 opaque_count;
95  };
96 
97  /* Really only need 1 bit */
98  u8 flags;
99 #define VNET_CLASSIFY_ENTRY_FREE (1<<0)
100 
101  vnet_classify_action_t action;
102  u16 metadata;
103 
104  /* Hit counter, last heard time */
105  union {
106  u64 hits;
107  struct _vnet_classify_entry * next_free;
108  };
109 
110  f64 last_heard;
111 
112  /* Must be aligned to a 16-octet boundary */
113  u32x4 key[0];
114 }) vnet_classify_entry_t;
115 /* *INDENT-ON* */
116 
117 static inline int
118 vnet_classify_entry_is_free (vnet_classify_entry_t * e)
119 {
120  return e->flags & VNET_CLASSIFY_ENTRY_FREE;
121 }
122 
123 static inline int
124 vnet_classify_entry_is_busy (vnet_classify_entry_t * e)
125 {
126  return ((e->flags & VNET_CLASSIFY_ENTRY_FREE) == 0);
127 }
128 
129 /* Need these to con the vector allocator */
130 /* *INDENT-OFF* */
131 #define _(size) \
132 typedef CLIB_PACKED(struct { \
133  u32 pad0[4]; \
134  u64 pad1[2]; \
135  u32x4 key[size]; \
136 }) vnet_classify_entry_##size##_t;
138 /* *INDENT-ON* */
139 #undef _
140 
141 typedef struct
142 {
143  union
144  {
145  struct
146  {
149  u8 pad[2];
151  };
153  };
155 
156 typedef struct
157 {
158  /* Mask to apply after skipping N vectors */
160  /* Buckets and entries */
162  vnet_classify_entry_t *entries;
163 
164  /* Config parameters */
175  /* Index of next table to try */
177 
178  /* Miss next index, return if next_table_index = 0 */
180 
181  /* Per-bucket working copies, one per thread */
182  vnet_classify_entry_t **working_copies;
185 
186  /* Free entry freelists */
187  vnet_classify_entry_t **freelists;
188 
190 
191  /* Private allocation arena, protected by the writer lock */
192  void *mheap;
193 
194  /* Writer (only) lock for this table */
195  volatile u32 *writer_lock;
196 
198 
199 struct _vnet_classify_main
200 {
201  /* Table pool */
202  vnet_classify_table_t *tables;
203 
204  /* Registered next-index, opaque unformat fcns */
205  unformat_function_t **unformat_l2_next_index_fns;
206  unformat_function_t **unformat_ip_next_index_fns;
207  unformat_function_t **unformat_acl_next_index_fns;
208  unformat_function_t **unformat_policer_next_index_fns;
209  unformat_function_t **unformat_opaque_index_fns;
210 
211  /* convenience variables */
214 };
215 
216 extern vnet_classify_main_t vnet_classify_main;
217 
218 u8 *format_classify_table (u8 * s, va_list * args);
219 
221 
222 static inline u64
224 {
225  u32x4 *mask;
226 
227  union
228  {
229  u32x4 as_u32x4;
230  u64 as_u64[2];
231  } xor_sum __attribute__ ((aligned (sizeof (u32x4))));
232 
233  ASSERT (t);
234  mask = t->mask;
235 #ifdef CLASSIFY_USE_SSE
236  if (U32X4_ALIGNED (h))
237  { //SSE can't handle unaligned data
238  u32x4 *data = (u32x4 *) h;
239  xor_sum.as_u32x4 = data[0 + t->skip_n_vectors] & mask[0];
240  switch (t->match_n_vectors)
241  {
242  case 5:
243  xor_sum.as_u32x4 ^= data[4 + t->skip_n_vectors] & mask[4];
244  /* FALLTHROUGH */
245  case 4:
246  xor_sum.as_u32x4 ^= data[3 + t->skip_n_vectors] & mask[3];
247  /* FALLTHROUGH */
248  case 3:
249  xor_sum.as_u32x4 ^= data[2 + t->skip_n_vectors] & mask[2];
250  /* FALLTHROUGH */
251  case 2:
252  xor_sum.as_u32x4 ^= data[1 + t->skip_n_vectors] & mask[1];
253  /* FALLTHROUGH */
254  case 1:
255  break;
256  default:
257  abort ();
258  }
259  }
260  else
261 #endif /* CLASSIFY_USE_SSE */
262  {
263  u32 skip_u64 = t->skip_n_vectors * 2;
264  u64 *data64 = (u64 *) h;
265  xor_sum.as_u64[0] = data64[0 + skip_u64] & ((u64 *) mask)[0];
266  xor_sum.as_u64[1] = data64[1 + skip_u64] & ((u64 *) mask)[1];
267  switch (t->match_n_vectors)
268  {
269  case 5:
270  xor_sum.as_u64[0] ^= data64[8 + skip_u64] & ((u64 *) mask)[8];
271  xor_sum.as_u64[1] ^= data64[9 + skip_u64] & ((u64 *) mask)[9];
272  /* FALLTHROUGH */
273  case 4:
274  xor_sum.as_u64[0] ^= data64[6 + skip_u64] & ((u64 *) mask)[6];
275  xor_sum.as_u64[1] ^= data64[7 + skip_u64] & ((u64 *) mask)[7];
276  /* FALLTHROUGH */
277  case 3:
278  xor_sum.as_u64[0] ^= data64[4 + skip_u64] & ((u64 *) mask)[4];
279  xor_sum.as_u64[1] ^= data64[5 + skip_u64] & ((u64 *) mask)[5];
280  /* FALLTHROUGH */
281  case 2:
282  xor_sum.as_u64[0] ^= data64[2 + skip_u64] & ((u64 *) mask)[2];
283  xor_sum.as_u64[1] ^= data64[3 + skip_u64] & ((u64 *) mask)[3];
284  /* FALLTHROUGH */
285  case 1:
286  break;
287 
288  default:
289  abort ();
290  }
291  }
292 
293  return clib_xxhash (xor_sum.as_u64[0] ^ xor_sum.as_u64[1]);
294 }
295 
296 static inline void
298 {
299  u32 bucket_index;
300 
301  ASSERT (is_pow2 (t->nbuckets));
302 
303  bucket_index = hash & (t->nbuckets - 1);
304 
305  CLIB_PREFETCH (&t->buckets[bucket_index], CLIB_CACHE_LINE_BYTES, LOAD);
306 }
307 
308 static inline vnet_classify_entry_t *
310 {
311  u8 *hp = t->mheap;
312  u8 *vp = hp + offset;
313 
314  return (void *) vp;
315 }
316 
317 static inline uword
319  vnet_classify_entry_t * v)
320 {
321  u8 *hp, *vp;
322 
323  hp = (u8 *) t->mheap;
324  vp = (u8 *) v;
325 
326  ASSERT ((vp - hp) < 0x100000000ULL);
327  return vp - hp;
328 }
329 
330 static inline vnet_classify_entry_t *
332  vnet_classify_entry_t * e, u32 index)
333 {
334  u8 *eu8;
335 
336  eu8 = (u8 *) e;
337 
338  eu8 += index * (sizeof (vnet_classify_entry_t) +
339  (t->match_n_vectors * sizeof (u32x4)));
340 
341  return (vnet_classify_entry_t *) eu8;
342 }
343 
344 static inline void
346 {
347  u32 bucket_index;
348  u32 value_index;
350  vnet_classify_entry_t *e;
351 
352  bucket_index = hash & (t->nbuckets - 1);
353 
354  b = &t->buckets[bucket_index];
355 
356  if (b->offset == 0)
357  return;
358 
359  hash >>= t->log2_nbuckets;
360 
361  e = vnet_classify_get_entry (t, b->offset);
362  value_index = hash & ((1 << b->log2_pages) - 1);
363 
364  e = vnet_classify_entry_at_index (t, e, value_index);
365 
367 }
368 
369 vnet_classify_entry_t *vnet_classify_find_entry (vnet_classify_table_t * t,
370  u8 * h, u64 hash, f64 now);
371 
372 static inline vnet_classify_entry_t *
374  u8 * h, u64 hash, f64 now)
375 {
376  vnet_classify_entry_t *v;
377  u32x4 *mask, *key;
378  union
379  {
380  u32x4 as_u32x4;
381  u64 as_u64[2];
382  } result __attribute__ ((aligned (sizeof (u32x4))));
384  u32 value_index;
385  u32 bucket_index;
386  u32 limit;
387  int i;
388 
389  bucket_index = hash & (t->nbuckets - 1);
390  b = &t->buckets[bucket_index];
391  mask = t->mask;
392 
393  if (b->offset == 0)
394  return 0;
395 
396  hash >>= t->log2_nbuckets;
397 
398  v = vnet_classify_get_entry (t, b->offset);
399  value_index = hash & ((1 << b->log2_pages) - 1);
400  limit = t->entries_per_page;
401  if (PREDICT_FALSE (b->linear_search))
402  {
403  value_index = 0;
404  limit *= (1 << b->log2_pages);
405  }
406 
407  v = vnet_classify_entry_at_index (t, v, value_index);
408 
409 #ifdef CLASSIFY_USE_SSE
410  if (U32X4_ALIGNED (h))
411  {
412  u32x4 *data = (u32x4 *) h;
413  for (i = 0; i < limit; i++)
414  {
415  key = v->key;
416  result.as_u32x4 = (data[0 + t->skip_n_vectors] & mask[0]) ^ key[0];
417  switch (t->match_n_vectors)
418  {
419  case 5:
420  result.as_u32x4 |=
421  (data[4 + t->skip_n_vectors] & mask[4]) ^ key[4];
422  /* FALLTHROUGH */
423  case 4:
424  result.as_u32x4 |=
425  (data[3 + t->skip_n_vectors] & mask[3]) ^ key[3];
426  /* FALLTHROUGH */
427  case 3:
428  result.as_u32x4 |=
429  (data[2 + t->skip_n_vectors] & mask[2]) ^ key[2];
430  /* FALLTHROUGH */
431  case 2:
432  result.as_u32x4 |=
433  (data[1 + t->skip_n_vectors] & mask[1]) ^ key[1];
434  /* FALLTHROUGH */
435  case 1:
436  break;
437  default:
438  abort ();
439  }
440 
441  if (u32x4_zero_byte_mask (result.as_u32x4) == 0xffff)
442  {
443  if (PREDICT_TRUE (now))
444  {
445  v->hits++;
446  v->last_heard = now;
447  }
448  return (v);
449  }
450  v = vnet_classify_entry_at_index (t, v, 1);
451  }
452  }
453  else
454 #endif /* CLASSIFY_USE_SSE */
455  {
456  u32 skip_u64 = t->skip_n_vectors * 2;
457  u64 *data64 = (u64 *) h;
458  for (i = 0; i < limit; i++)
459  {
460  key = v->key;
461 
462  result.as_u64[0] =
463  (data64[0 + skip_u64] & ((u64 *) mask)[0]) ^ ((u64 *) key)[0];
464  result.as_u64[1] =
465  (data64[1 + skip_u64] & ((u64 *) mask)[1]) ^ ((u64 *) key)[1];
466  switch (t->match_n_vectors)
467  {
468  case 5:
469  result.as_u64[0] |=
470  (data64[8 + skip_u64] & ((u64 *) mask)[8]) ^ ((u64 *) key)[8];
471  result.as_u64[1] |=
472  (data64[9 + skip_u64] & ((u64 *) mask)[9]) ^ ((u64 *) key)[9];
473  /* FALLTHROUGH */
474  case 4:
475  result.as_u64[0] |=
476  (data64[6 + skip_u64] & ((u64 *) mask)[6]) ^ ((u64 *) key)[6];
477  result.as_u64[1] |=
478  (data64[7 + skip_u64] & ((u64 *) mask)[7]) ^ ((u64 *) key)[7];
479  /* FALLTHROUGH */
480  case 3:
481  result.as_u64[0] |=
482  (data64[4 + skip_u64] & ((u64 *) mask)[4]) ^ ((u64 *) key)[4];
483  result.as_u64[1] |=
484  (data64[5 + skip_u64] & ((u64 *) mask)[5]) ^ ((u64 *) key)[5];
485  /* FALLTHROUGH */
486  case 2:
487  result.as_u64[0] |=
488  (data64[2 + skip_u64] & ((u64 *) mask)[2]) ^ ((u64 *) key)[2];
489  result.as_u64[1] |=
490  (data64[3 + skip_u64] & ((u64 *) mask)[3]) ^ ((u64 *) key)[3];
491  /* FALLTHROUGH */
492  case 1:
493  break;
494  default:
495  abort ();
496  }
497 
498  if (result.as_u64[0] == 0 && result.as_u64[1] == 0)
499  {
500  if (PREDICT_TRUE (now))
501  {
502  v->hits++;
503  v->last_heard = now;
504  }
505  return (v);
506  }
507 
508  v = vnet_classify_entry_at_index (t, v, 1);
509  }
510  }
511  return 0;
512 }
513 
514 vnet_classify_table_t *vnet_classify_new_table (vnet_classify_main_t * cm,
515  u8 * mask, u32 nbuckets,
516  u32 memory_size,
517  u32 skip_n_vectors,
518  u32 match_n_vectors);
519 
520 int vnet_classify_add_del_session (vnet_classify_main_t * cm,
521  u32 table_index,
522  u8 * match,
523  u32 hit_next_index,
524  u32 opaque_index,
525  i32 advance,
526  u8 action, u32 metadata, int is_add);
527 
528 int vnet_classify_add_del_table (vnet_classify_main_t * cm,
529  u8 * mask,
530  u32 nbuckets,
531  u32 memory_size,
532  u32 skip,
533  u32 match,
534  u32 next_table_index,
535  u32 miss_next_index,
536  u32 * table_index,
537  u8 current_data_flag,
538  i16 current_data_offset,
539  int is_add, int del_chain);
540 
555 
557  (unformat_function_t * fn);
558 
560  (unformat_function_t * fn);
561 
563  (unformat_function_t * fn);
564 
566  (unformat_function_t * fn);
567 
569  fn);
570 
571 #endif /* __included_vnet_classify_h__ */
572 
573 /*
574  * fd.io coding-style-patch-verification: ON
575  *
576  * Local Variables:
577  * eval: (c-set-style "gnu")
578  * End:
579  */
u64 vnet_classify_hash_packet(vnet_classify_table_t *t, u8 *h)
unformat_function_t unformat_ip4_match
vnet_classify_entry_t ** working_copies
uword( unformat_function_t)(unformat_input_t *input, va_list *args)
Definition: format.h:231
unformat_function_t unformat_vlan_tag
u8 pad[3]
log2 (size of the packing page block)
Definition: bihash_doc.h:61
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:337
unformat_function_t unformat_l2_mask
unformat_function_t unformat_ip_next_index
#define foreach_size_in_u32x4
Definition: vnet_classify.h:75
u64 as_u64
Definition: bihash_doc.h:63
#define PREDICT_TRUE(x)
Definition: clib.h:106
static vnet_classify_entry_t * vnet_classify_find_entry_inline(vnet_classify_table_t *t, u8 *h, u64 hash, f64 now)
#define VNET_CLASSIFY_ENTRY_FREE
unformat_function_t unformat_ip6_mask
#define U32X4_ALIGNED(p)
Definition: vnet_classify.h:47
static u64 clib_xxhash(u64 key)
Definition: xxhash.h:58
int vnet_classify_add_del_session(vnet_classify_main_t *cm, u32 table_index, u8 *match, u32 hit_next_index, u32 opaque_index, i32 advance, u8 action, u32 metadata, int is_add)
struct _vlib_node_registration vlib_node_registration_t
unformat_function_t unformat_classify_match
vnet_classify_table_t * vnet_classify_new_table(vnet_classify_main_t *cm, u8 *mask, u32 nbuckets, u32 memory_size, u32 skip_n_vectors, u32 match_n_vectors)
unformat_function_t unformat_l3_mask
unformat_function_t unformat_ip4_mask
unformat_function_t unformat_classify_mask
vnet_classify_action_t_
Definition: vnet_classify.h:65
typedef CLIB_PACKED(struct _vnet_classify_entry{u32 next_index;union{struct{u32 opaque_index;i32 advance;};u64 opaque_count;};u8 flags;#define VNET_CLASSIFY_ENTRY_FREE vnet_classify_action_t action;u16 metadata;union{u64 hits;struct _vnet_classify_entry *next_free;};f64 last_heard;u32x4 key[0];}) vnet_classify_entry_t
unsigned long long u32x4
Definition: ixge.c:28
int i32
Definition: types.h:81
void vnet_classify_register_unformat_opaque_index_fn(unformat_function_t *fn)
unsigned long u64
Definition: types.h:89
static void vnet_classify_prefetch_bucket(vnet_classify_table_t *t, u64 hash)
u8 * format_classify_table(u8 *s, va_list *args)
static void vnet_classify_prefetch_entry(vnet_classify_table_t *t, u64 hash)
static int vnet_classify_entry_is_free(vnet_classify_entry_t *e)
static u64 vnet_classify_hash_packet_inline(vnet_classify_table_t *t, u8 *h)
unformat_function_t unformat_l3_match
#define v
Definition: acl.c:341
vnet_classify_entry_t * entries
#define PREDICT_FALSE(x)
Definition: clib.h:105
unformat_function_t unformat_l2_next_index
vnet_main_t vnet_main
Definition: misc.c:43
void vnet_classify_register_unformat_policer_next_index_fn(unformat_function_t *fn)
static vnet_classify_entry_t * vnet_classify_entry_at_index(vnet_classify_table_t *t, vnet_classify_entry_t *e, u32 index)
static uword vnet_classify_get_offset(vnet_classify_table_t *t, vnet_classify_entry_t *v)
vnet_classify_bucket_t saved_bucket
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:82
void vnet_classify_register_unformat_l2_next_index_fn(unformat_function_t *fn)
Definition: vnet_classify.c:80
void vnet_classify_register_unformat_acl_next_index_fn(unformat_function_t *fn)
Definition: vnet_classify.c:96
#define ASSERT(truth)
unsigned int u32
Definition: types.h:88
unformat_function_t unformat_ip6_match
static uword is_pow2(uword x)
Definition: clib.h:280
u64 uword
Definition: types.h:112
int vlib_main(vlib_main_t *volatile vm, unformat_input_t *input)
Definition: main.c:1676
static int vnet_classify_entry_is_busy(vnet_classify_entry_t *e)
template key/value backing page structure
Definition: bihash_doc.h:44
unsigned short u16
Definition: types.h:57
vlib_node_registration_t ip6_classify_node
(constructor) VLIB_REGISTER_NODE (ip6_classify_node)
Definition: ip_classify.c:41
double f64
Definition: types.h:142
vnet_classify_bucket_t * buckets
unsigned char u8
Definition: types.h:56
vnet_classify_entry_t * vnet_classify_find_entry(vnet_classify_table_t *t, u8 *h, u64 hash, f64 now)
vlib_node_registration_t ip4_classify_node
(constructor) VLIB_REGISTER_NODE (ip4_classify_node)
Definition: ip_classify.c:40
unformat_function_t unformat_l4_match
volatile u32 * writer_lock
void vnet_classify_register_unformat_ip_next_index_fn(unformat_function_t *fn)
Definition: vnet_classify.c:88
struct clib_bihash_value offset
template key/value backing page structure
short i16
Definition: types.h:46
unformat_function_t unformat_l2_match
u32 flags
Definition: vhost-user.h:77
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
vnet_classify_entry_t ** freelists
vnet_classify_main_t vnet_classify_main
Definition: vnet_classify.c:22
static u32 u32x4_zero_byte_mask(u32x4 x)
static vnet_classify_entry_t * vnet_classify_get_entry(vnet_classify_table_t *t, uword offset)
int vnet_classify_add_del_table(vnet_classify_main_t *cm, u8 *mask, u32 nbuckets, u32 memory_size, u32 skip, u32 match, u32 next_table_index, u32 miss_next_index, u32 *table_index, u8 current_data_flag, i16 current_data_offset, int is_add, int del_chain)