FD.io VPP  v18.07-rc0-415-g6c78436
Vector Packet Processing
vnet_classify.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #ifndef __included_vnet_classify_h__
16 #define __included_vnet_classify_h__
17 
18 #include <stdarg.h>
19 
20 #include <vlib/vlib.h>
21 #include <vnet/vnet.h>
22 #include <vnet/pg/pg.h>
23 #include <vnet/ethernet/ethernet.h>
24 #include <vnet/ethernet/packet.h>
25 #include <vnet/ip/ip_packet.h>
26 #include <vnet/ip/ip4_packet.h>
27 #include <vnet/ip/ip6_packet.h>
28 #include <vlib/cli.h>
29 #include <vnet/l2/l2_input.h>
30 #include <vnet/l2/l2_output.h>
31 #include <vnet/l2/feat_bitmap.h>
32 #include <vnet/api_errno.h> /* for API error numbers */
33 
34 #include <vppinfra/error.h>
35 #include <vppinfra/hash.h>
36 #include <vppinfra/cache.h>
37 #include <vppinfra/xxhash.h>
38 
41 
42 #define CLASSIFY_TRACE 0
43 
44 #define U32X4_ALIGNED(p) PREDICT_TRUE((((intptr_t)p) & 0xf) == 0)
45 
46 /*
47  * Classify table option to process packets
48  * CLASSIFY_FLAG_USE_CURR_DATA:
49  * - classify packets starting from VPP node’s current data pointer
50  */
51 #define CLASSIFY_FLAG_USE_CURR_DATA 1
52 
53 /*
54  * Classify session action
55  * CLASSIFY_ACTION_SET_IP4_FIB_INDEX:
56  * - Classified IP packets will be looked up
57  * from the specified ipv4 fib table
58  * CLASSIFY_ACTION_SET_IP6_FIB_INDEX:
59  * - Classified IP packets will be looked up
60  * from the specified ipv6 fib table
61  */
63 {
67 } __attribute__ ((packed)) vnet_classify_action_t;
68 
69 struct _vnet_classify_main;
70 typedef struct _vnet_classify_main vnet_classify_main_t;
71 
72 #define foreach_size_in_u32x4 \
73 _(1) \
74 _(2) \
75 _(3) \
76 _(4) \
77 _(5)
78 
79 /* *INDENT-OFF* */
80 typedef CLIB_PACKED(struct _vnet_classify_entry {
81  /* Graph node next index */
82  u32 next_index;
83 
84  /* put into vnet_buffer(b)->l2_classfy.opaque_index */
85  union {
86  struct {
87  u32 opaque_index;
88  /* advance on hit, note it's a signed quantity... */
89  i32 advance;
90  };
91  u64 opaque_count;
92  };
93 
94  /* Really only need 1 bit */
95  u8 flags;
96 #define VNET_CLASSIFY_ENTRY_FREE (1<<0)
97 
98  vnet_classify_action_t action;
99  u16 metadata;
100 
101  /* Hit counter, last heard time */
102  union {
103  u64 hits;
104  struct _vnet_classify_entry * next_free;
105  };
106 
107  f64 last_heard;
108 
109  /* Must be aligned to a 16-octet boundary */
110  u32x4 key[0];
111 }) vnet_classify_entry_t;
112 /* *INDENT-ON* */
113 
114 static inline int
115 vnet_classify_entry_is_free (vnet_classify_entry_t * e)
116 {
117  return e->flags & VNET_CLASSIFY_ENTRY_FREE;
118 }
119 
120 static inline int
121 vnet_classify_entry_is_busy (vnet_classify_entry_t * e)
122 {
123  return ((e->flags & VNET_CLASSIFY_ENTRY_FREE) == 0);
124 }
125 
126 /* Need these to con the vector allocator */
127 /* *INDENT-OFF* */
128 #define _(size) \
129 typedef CLIB_PACKED(struct { \
130  u32 pad0[4]; \
131  u64 pad1[2]; \
132  u32x4 key[size]; \
133 }) vnet_classify_entry_##size##_t;
135 /* *INDENT-ON* */
136 #undef _
137 
138 typedef struct
139 {
140  union
141  {
142  struct
143  {
146  u8 pad[2];
148  };
150  };
152 
153 typedef struct
154 {
155  /* Mask to apply after skipping N vectors */
157  /* Buckets and entries */
159  vnet_classify_entry_t *entries;
160 
161  /* Config parameters */
172  /* Index of next table to try */
174 
175  /* Miss next index, return if next_table_index = 0 */
177 
178  /* Per-bucket working copies, one per thread */
179  vnet_classify_entry_t **working_copies;
182 
183  /* Free entry freelists */
184  vnet_classify_entry_t **freelists;
185 
187 
188  /* Private allocation arena, protected by the writer lock */
189  void *mheap;
190 
191  /* Writer (only) lock for this table */
192  volatile u32 *writer_lock;
193 
195 
196 struct _vnet_classify_main
197 {
198  /* Table pool */
199  vnet_classify_table_t *tables;
200 
201  /* Registered next-index, opaque unformat fcns */
202  unformat_function_t **unformat_l2_next_index_fns;
203  unformat_function_t **unformat_ip_next_index_fns;
204  unformat_function_t **unformat_acl_next_index_fns;
205  unformat_function_t **unformat_policer_next_index_fns;
206  unformat_function_t **unformat_opaque_index_fns;
207 
208  /* convenience variables */
211 };
212 
213 extern vnet_classify_main_t vnet_classify_main;
214 
215 u8 *format_classify_table (u8 * s, va_list * args);
216 
218 
219 static inline u64
221 {
222  u32x4 *mask;
223 
224  union
225  {
226  u32x4 as_u32x4;
227  u64 as_u64[2];
228  } xor_sum __attribute__ ((aligned (sizeof (u32x4))));
229 
230  ASSERT (t);
231  mask = t->mask;
232 #ifdef CLIB_HAVE_VEC128
233  if (U32X4_ALIGNED (h))
234  { //SSE can't handle unaligned data
235  u32x4 *data = (u32x4 *) h;
236  xor_sum.as_u32x4 = data[0 + t->skip_n_vectors] & mask[0];
237  switch (t->match_n_vectors)
238  {
239  case 5:
240  xor_sum.as_u32x4 ^= data[4 + t->skip_n_vectors] & mask[4];
241  /* FALLTHROUGH */
242  case 4:
243  xor_sum.as_u32x4 ^= data[3 + t->skip_n_vectors] & mask[3];
244  /* FALLTHROUGH */
245  case 3:
246  xor_sum.as_u32x4 ^= data[2 + t->skip_n_vectors] & mask[2];
247  /* FALLTHROUGH */
248  case 2:
249  xor_sum.as_u32x4 ^= data[1 + t->skip_n_vectors] & mask[1];
250  /* FALLTHROUGH */
251  case 1:
252  break;
253  default:
254  abort ();
255  }
256  }
257  else
258 #endif /* CLIB_HAVE_VEC128 */
259  {
260  u32 skip_u64 = t->skip_n_vectors * 2;
261  u64 *data64 = (u64 *) h;
262  xor_sum.as_u64[0] = data64[0 + skip_u64] & ((u64 *) mask)[0];
263  xor_sum.as_u64[1] = data64[1 + skip_u64] & ((u64 *) mask)[1];
264  switch (t->match_n_vectors)
265  {
266  case 5:
267  xor_sum.as_u64[0] ^= data64[8 + skip_u64] & ((u64 *) mask)[8];
268  xor_sum.as_u64[1] ^= data64[9 + skip_u64] & ((u64 *) mask)[9];
269  /* FALLTHROUGH */
270  case 4:
271  xor_sum.as_u64[0] ^= data64[6 + skip_u64] & ((u64 *) mask)[6];
272  xor_sum.as_u64[1] ^= data64[7 + skip_u64] & ((u64 *) mask)[7];
273  /* FALLTHROUGH */
274  case 3:
275  xor_sum.as_u64[0] ^= data64[4 + skip_u64] & ((u64 *) mask)[4];
276  xor_sum.as_u64[1] ^= data64[5 + skip_u64] & ((u64 *) mask)[5];
277  /* FALLTHROUGH */
278  case 2:
279  xor_sum.as_u64[0] ^= data64[2 + skip_u64] & ((u64 *) mask)[2];
280  xor_sum.as_u64[1] ^= data64[3 + skip_u64] & ((u64 *) mask)[3];
281  /* FALLTHROUGH */
282  case 1:
283  break;
284 
285  default:
286  abort ();
287  }
288  }
289 
290  return clib_xxhash (xor_sum.as_u64[0] ^ xor_sum.as_u64[1]);
291 }
292 
293 static inline void
295 {
296  u32 bucket_index;
297 
298  ASSERT (is_pow2 (t->nbuckets));
299 
300  bucket_index = hash & (t->nbuckets - 1);
301 
302  CLIB_PREFETCH (&t->buckets[bucket_index], CLIB_CACHE_LINE_BYTES, LOAD);
303 }
304 
305 static inline vnet_classify_entry_t *
307 {
308  u8 *hp = t->mheap;
309  u8 *vp = hp + offset;
310 
311  return (void *) vp;
312 }
313 
314 static inline uword
316  vnet_classify_entry_t * v)
317 {
318  u8 *hp, *vp;
319 
320  hp = (u8 *) t->mheap;
321  vp = (u8 *) v;
322 
323  ASSERT ((vp - hp) < 0x100000000ULL);
324  return vp - hp;
325 }
326 
327 static inline vnet_classify_entry_t *
329  vnet_classify_entry_t * e, u32 index)
330 {
331  u8 *eu8;
332 
333  eu8 = (u8 *) e;
334 
335  eu8 += index * (sizeof (vnet_classify_entry_t) +
336  (t->match_n_vectors * sizeof (u32x4)));
337 
338  return (vnet_classify_entry_t *) eu8;
339 }
340 
341 static inline void
343 {
344  u32 bucket_index;
345  u32 value_index;
347  vnet_classify_entry_t *e;
348 
349  bucket_index = hash & (t->nbuckets - 1);
350 
351  b = &t->buckets[bucket_index];
352 
353  if (b->offset == 0)
354  return;
355 
356  hash >>= t->log2_nbuckets;
357 
358  e = vnet_classify_get_entry (t, b->offset);
359  value_index = hash & ((1 << b->log2_pages) - 1);
360 
361  e = vnet_classify_entry_at_index (t, e, value_index);
362 
364 }
365 
366 vnet_classify_entry_t *vnet_classify_find_entry (vnet_classify_table_t * t,
367  u8 * h, u64 hash, f64 now);
368 
369 static inline vnet_classify_entry_t *
371  u8 * h, u64 hash, f64 now)
372 {
373  vnet_classify_entry_t *v;
374  u32x4 *mask, *key;
375  union
376  {
377  u32x4 as_u32x4;
378  u64 as_u64[2];
379  } result __attribute__ ((aligned (sizeof (u32x4))));
381  u32 value_index;
382  u32 bucket_index;
383  u32 limit;
384  int i;
385 
386  bucket_index = hash & (t->nbuckets - 1);
387  b = &t->buckets[bucket_index];
388  mask = t->mask;
389 
390  if (b->offset == 0)
391  return 0;
392 
393  hash >>= t->log2_nbuckets;
394 
395  v = vnet_classify_get_entry (t, b->offset);
396  value_index = hash & ((1 << b->log2_pages) - 1);
397  limit = t->entries_per_page;
398  if (PREDICT_FALSE (b->linear_search))
399  {
400  value_index = 0;
401  limit *= (1 << b->log2_pages);
402  }
403 
404  v = vnet_classify_entry_at_index (t, v, value_index);
405 
406 #ifdef CLIB_HAVE_VEC128
407  if (U32X4_ALIGNED (h))
408  {
409  u32x4 *data = (u32x4 *) h;
410  for (i = 0; i < limit; i++)
411  {
412  key = v->key;
413  result.as_u32x4 = (data[0 + t->skip_n_vectors] & mask[0]) ^ key[0];
414  switch (t->match_n_vectors)
415  {
416  case 5:
417  result.as_u32x4 |=
418  (data[4 + t->skip_n_vectors] & mask[4]) ^ key[4];
419  /* FALLTHROUGH */
420  case 4:
421  result.as_u32x4 |=
422  (data[3 + t->skip_n_vectors] & mask[3]) ^ key[3];
423  /* FALLTHROUGH */
424  case 3:
425  result.as_u32x4 |=
426  (data[2 + t->skip_n_vectors] & mask[2]) ^ key[2];
427  /* FALLTHROUGH */
428  case 2:
429  result.as_u32x4 |=
430  (data[1 + t->skip_n_vectors] & mask[1]) ^ key[1];
431  /* FALLTHROUGH */
432  case 1:
433  break;
434  default:
435  abort ();
436  }
437 
438  if (u32x4_zero_byte_mask (result.as_u32x4) == 0xffff)
439  {
440  if (PREDICT_TRUE (now))
441  {
442  v->hits++;
443  v->last_heard = now;
444  }
445  return (v);
446  }
447  v = vnet_classify_entry_at_index (t, v, 1);
448  }
449  }
450  else
451 #endif /* CLIB_HAVE_VEC128 */
452  {
453  u32 skip_u64 = t->skip_n_vectors * 2;
454  u64 *data64 = (u64 *) h;
455  for (i = 0; i < limit; i++)
456  {
457  key = v->key;
458 
459  result.as_u64[0] =
460  (data64[0 + skip_u64] & ((u64 *) mask)[0]) ^ ((u64 *) key)[0];
461  result.as_u64[1] =
462  (data64[1 + skip_u64] & ((u64 *) mask)[1]) ^ ((u64 *) key)[1];
463  switch (t->match_n_vectors)
464  {
465  case 5:
466  result.as_u64[0] |=
467  (data64[8 + skip_u64] & ((u64 *) mask)[8]) ^ ((u64 *) key)[8];
468  result.as_u64[1] |=
469  (data64[9 + skip_u64] & ((u64 *) mask)[9]) ^ ((u64 *) key)[9];
470  /* FALLTHROUGH */
471  case 4:
472  result.as_u64[0] |=
473  (data64[6 + skip_u64] & ((u64 *) mask)[6]) ^ ((u64 *) key)[6];
474  result.as_u64[1] |=
475  (data64[7 + skip_u64] & ((u64 *) mask)[7]) ^ ((u64 *) key)[7];
476  /* FALLTHROUGH */
477  case 3:
478  result.as_u64[0] |=
479  (data64[4 + skip_u64] & ((u64 *) mask)[4]) ^ ((u64 *) key)[4];
480  result.as_u64[1] |=
481  (data64[5 + skip_u64] & ((u64 *) mask)[5]) ^ ((u64 *) key)[5];
482  /* FALLTHROUGH */
483  case 2:
484  result.as_u64[0] |=
485  (data64[2 + skip_u64] & ((u64 *) mask)[2]) ^ ((u64 *) key)[2];
486  result.as_u64[1] |=
487  (data64[3 + skip_u64] & ((u64 *) mask)[3]) ^ ((u64 *) key)[3];
488  /* FALLTHROUGH */
489  case 1:
490  break;
491  default:
492  abort ();
493  }
494 
495  if (result.as_u64[0] == 0 && result.as_u64[1] == 0)
496  {
497  if (PREDICT_TRUE (now))
498  {
499  v->hits++;
500  v->last_heard = now;
501  }
502  return (v);
503  }
504 
505  v = vnet_classify_entry_at_index (t, v, 1);
506  }
507  }
508  return 0;
509 }
510 
511 vnet_classify_table_t *vnet_classify_new_table (vnet_classify_main_t * cm,
512  u8 * mask, u32 nbuckets,
513  u32 memory_size,
514  u32 skip_n_vectors,
515  u32 match_n_vectors);
516 
517 int vnet_classify_add_del_session (vnet_classify_main_t * cm,
518  u32 table_index,
519  u8 * match,
520  u32 hit_next_index,
521  u32 opaque_index,
522  i32 advance,
523  u8 action, u32 metadata, int is_add);
524 
525 int vnet_classify_add_del_table (vnet_classify_main_t * cm,
526  u8 * mask,
527  u32 nbuckets,
528  u32 memory_size,
529  u32 skip,
530  u32 match,
531  u32 next_table_index,
532  u32 miss_next_index,
533  u32 * table_index,
534  u8 current_data_flag,
535  i16 current_data_offset,
536  int is_add, int del_chain);
537 
552 
554  (unformat_function_t * fn);
555 
557  (unformat_function_t * fn);
558 
560  (unformat_function_t * fn);
561 
563  (unformat_function_t * fn);
564 
566  fn);
567 
568 #endif /* __included_vnet_classify_h__ */
569 
570 /*
571  * fd.io coding-style-patch-verification: ON
572  *
573  * Local Variables:
574  * eval: (c-set-style "gnu")
575  * End:
576  */
u64 vnet_classify_hash_packet(vnet_classify_table_t *t, u8 *h)
unformat_function_t unformat_ip4_match
vnet_classify_entry_t ** working_copies
uword( unformat_function_t)(unformat_input_t *input, va_list *args)
Definition: format.h:231
unformat_function_t unformat_vlan_tag
u8 pad[3]
log2 (size of the packing page block)
Definition: bihash_doc.h:61
unformat_function_t unformat_l2_mask
unformat_function_t unformat_ip_next_index
#define foreach_size_in_u32x4
Definition: vnet_classify.h:72
u64 as_u64
Definition: bihash_doc.h:63
#define PREDICT_TRUE(x)
Definition: clib.h:106
static vnet_classify_entry_t * vnet_classify_find_entry_inline(vnet_classify_table_t *t, u8 *h, u64 hash, f64 now)
unsigned long u64
Definition: types.h:89
#define VNET_CLASSIFY_ENTRY_FREE
unformat_function_t unformat_ip6_mask
#define U32X4_ALIGNED(p)
Definition: vnet_classify.h:44
static u64 clib_xxhash(u64 key)
Definition: xxhash.h:58
int vnet_classify_add_del_session(vnet_classify_main_t *cm, u32 table_index, u8 *match, u32 hit_next_index, u32 opaque_index, i32 advance, u8 action, u32 metadata, int is_add)
int i
unformat_function_t unformat_classify_match
vnet_classify_table_t * vnet_classify_new_table(vnet_classify_main_t *cm, u8 *mask, u32 nbuckets, u32 memory_size, u32 skip_n_vectors, u32 match_n_vectors)
unformat_function_t unformat_l3_mask
unsigned char u8
Definition: types.h:56
unformat_function_t unformat_ip4_mask
double f64
Definition: types.h:142
unformat_function_t unformat_classify_mask
vnet_classify_action_t_
Definition: vnet_classify.h:62
typedef CLIB_PACKED(struct _vnet_classify_entry{u32 next_index;union{struct{u32 opaque_index;i32 advance;};u64 opaque_count;};u8 flags;#define VNET_CLASSIFY_ENTRY_FREE vnet_classify_action_t action;u16 metadata;union{u64 hits;struct _vnet_classify_entry *next_free;};f64 last_heard;u32x4 key[0];}) vnet_classify_entry_t
unsigned long long u32x4
Definition: ixge.c:28
void vnet_classify_register_unformat_opaque_index_fn(unformat_function_t *fn)
unsigned int u32
Definition: types.h:88
static void vnet_classify_prefetch_bucket(vnet_classify_table_t *t, u64 hash)
u8 * format_classify_table(u8 *s, va_list *args)
static void vnet_classify_prefetch_entry(vnet_classify_table_t *t, u64 hash)
static int vnet_classify_entry_is_free(vnet_classify_entry_t *e)
static u64 vnet_classify_hash_packet_inline(vnet_classify_table_t *t, u8 *h)
unformat_function_t unformat_l3_match
#define v
Definition: acl.c:491
unsigned short u16
Definition: types.h:57
vnet_classify_entry_t * entries
#define PREDICT_FALSE(x)
Definition: clib.h:105
unformat_function_t unformat_l2_next_index
vnet_main_t vnet_main
Definition: misc.c:43
void vnet_classify_register_unformat_policer_next_index_fn(unformat_function_t *fn)
static vnet_classify_entry_t * vnet_classify_entry_at_index(vnet_classify_table_t *t, vnet_classify_entry_t *e, u32 index)
static uword vnet_classify_get_offset(vnet_classify_table_t *t, vnet_classify_entry_t *v)
vnet_classify_bucket_t saved_bucket
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:77
void vnet_classify_register_unformat_l2_next_index_fn(unformat_function_t *fn)
Definition: vnet_classify.c:80
void vnet_classify_register_unformat_acl_next_index_fn(unformat_function_t *fn)
Definition: vnet_classify.c:96
signed int i32
Definition: types.h:81
#define ASSERT(truth)
unformat_function_t unformat_ip6_match
static uword is_pow2(uword x)
Definition: clib.h:229
int vlib_main(vlib_main_t *volatile vm, unformat_input_t *input)
Definition: main.c:1695
static int vnet_classify_entry_is_busy(vnet_classify_entry_t *e)
struct _vlib_node_registration vlib_node_registration_t
template key/value backing page structure
Definition: bihash_doc.h:44
vlib_node_registration_t ip6_classify_node
(constructor) VLIB_REGISTER_NODE (ip6_classify_node)
Definition: ip_classify.c:41
vnet_classify_bucket_t * buckets
u64 uword
Definition: types.h:112
vnet_classify_entry_t * vnet_classify_find_entry(vnet_classify_table_t *t, u8 *h, u64 hash, f64 now)
vlib_node_registration_t ip4_classify_node
(constructor) VLIB_REGISTER_NODE (ip4_classify_node)
Definition: ip_classify.c:40
unformat_function_t unformat_l4_match
volatile u32 * writer_lock
void vnet_classify_register_unformat_ip_next_index_fn(unformat_function_t *fn)
Definition: vnet_classify.c:88
struct clib_bihash_value offset
template key/value backing page structure
unformat_function_t unformat_l2_match
u32 flags
Definition: vhost-user.h:77
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:62
vnet_classify_entry_t ** freelists
vnet_classify_main_t vnet_classify_main
Definition: vnet_classify.c:22
static u32 u32x4_zero_byte_mask(u32x4 x)
static vnet_classify_entry_t * vnet_classify_get_entry(vnet_classify_table_t *t, uword offset)
signed short i16
Definition: types.h:46
int vnet_classify_add_del_table(vnet_classify_main_t *cm, u8 *mask, u32 nbuckets, u32 memory_size, u32 skip, u32 match, u32 next_table_index, u32 miss_next_index, u32 *table_index, u8 current_data_flag, i16 current_data_offset, int is_add, int del_chain)