15 #ifndef __included_vnet_classify_h__ 16 #define __included_vnet_classify_h__ 41 #define CLASSIFY_TRACE 0 43 #if !defined( __aarch64__) && !defined(__arm__) 44 #define CLASSIFY_USE_SSE //Allow usage of SSE operations 47 #define U32X4_ALIGNED(p) PREDICT_TRUE((((intptr_t)p) & 0xf) == 0) 54 #define CLASSIFY_FLAG_USE_CURR_DATA 1 70 } __attribute__ ((packed)) vnet_classify_action_t;
72 struct _vnet_classify_main;
73 typedef struct _vnet_classify_main vnet_classify_main_t;
75 #define foreach_size_in_u32x4 \ 99 #define VNET_CLASSIFY_ENTRY_FREE (1<<0) 101 vnet_classify_action_t action;
107 struct _vnet_classify_entry * next_free;
114 }) vnet_classify_entry_t;
132 typedef CLIB_PACKED(struct { \ 136 }) vnet_classify_entry_##size##_t; 199 struct _vnet_classify_main
231 } xor_sum __attribute__ ((aligned (
sizeof (
u32x4))));
235 #ifdef CLASSIFY_USE_SSE 265 xor_sum.as_u64[0] = data64[0 + skip_u64] & ((
u64 *) mask)[0];
266 xor_sum.as_u64[1] = data64[1 + skip_u64] & ((
u64 *) mask)[1];
270 xor_sum.as_u64[0] ^= data64[8 + skip_u64] & ((
u64 *) mask)[8];
271 xor_sum.as_u64[1] ^= data64[9 + skip_u64] & ((
u64 *) mask)[9];
274 xor_sum.as_u64[0] ^= data64[6 + skip_u64] & ((
u64 *) mask)[6];
275 xor_sum.as_u64[1] ^= data64[7 + skip_u64] & ((
u64 *) mask)[7];
278 xor_sum.as_u64[0] ^= data64[4 + skip_u64] & ((
u64 *) mask)[4];
279 xor_sum.as_u64[1] ^= data64[5 + skip_u64] & ((
u64 *) mask)[5];
282 xor_sum.as_u64[0] ^= data64[2 + skip_u64] & ((
u64 *) mask)[2];
283 xor_sum.as_u64[1] ^= data64[3 + skip_u64] & ((
u64 *) mask)[3];
293 return clib_xxhash (xor_sum.as_u64[0] ^ xor_sum.as_u64[1]);
303 bucket_index = hash & (t->
nbuckets - 1);
308 static inline vnet_classify_entry_t *
319 vnet_classify_entry_t *
v)
326 ASSERT ((vp - hp) < 0x100000000ULL);
330 static inline vnet_classify_entry_t *
332 vnet_classify_entry_t * e,
u32 index)
338 eu8 += index * (
sizeof (vnet_classify_entry_t) +
341 return (vnet_classify_entry_t *) eu8;
350 vnet_classify_entry_t *e;
352 bucket_index = hash & (t->
nbuckets - 1);
362 value_index = hash & ((1 << b->log2_pages) - 1);
372 static inline vnet_classify_entry_t *
376 vnet_classify_entry_t *
v;
382 } result __attribute__ ((aligned (
sizeof (
u32x4))));
389 bucket_index = hash & (t->
nbuckets - 1);
399 value_index = hash & ((1 << b->log2_pages) - 1);
404 limit *= (1 << b->log2_pages);
409 #ifdef CLASSIFY_USE_SSE 413 for (i = 0; i < limit; i++)
416 result.as_u32x4 = (data[0 + t->
skip_n_vectors] & mask[0]) ^ key[0];
458 for (i = 0; i < limit; i++)
463 (data64[0 + skip_u64] & ((
u64 *) mask)[0]) ^ ((
u64 *) key)[0];
465 (data64[1 + skip_u64] & ((
u64 *) mask)[1]) ^ ((
u64 *) key)[1];
470 (data64[8 + skip_u64] & ((
u64 *) mask)[8]) ^ ((
u64 *) key)[8];
472 (data64[9 + skip_u64] & ((
u64 *) mask)[9]) ^ ((
u64 *) key)[9];
476 (data64[6 + skip_u64] & ((
u64 *) mask)[6]) ^ ((
u64 *) key)[6];
478 (data64[7 + skip_u64] & ((
u64 *) mask)[7]) ^ ((
u64 *) key)[7];
482 (data64[4 + skip_u64] & ((
u64 *) mask)[4]) ^ ((
u64 *) key)[4];
484 (data64[5 + skip_u64] & ((
u64 *) mask)[5]) ^ ((
u64 *) key)[5];
488 (data64[2 + skip_u64] & ((
u64 *) mask)[2]) ^ ((
u64 *) key)[2];
490 (data64[3 + skip_u64] & ((
u64 *) mask)[3]) ^ ((
u64 *) key)[3];
498 if (result.as_u64[0] == 0 && result.as_u64[1] == 0)
515 u8 * mask,
u32 nbuckets,
518 u32 match_n_vectors);
526 u8 action,
u32 metadata,
int is_add);
534 u32 next_table_index,
537 u8 current_data_flag,
538 i16 current_data_offset,
539 int is_add,
int del_chain);
u64 vnet_classify_hash_packet(vnet_classify_table_t *t, u8 *h)
unformat_function_t unformat_ip4_match
vnet_classify_entry_t ** working_copies
unformat_function_t unformat_vlan_tag
u8 pad[3]
log2 (size of the packing page block)
sll srl srl sll sra u16x4 i
unformat_function_t unformat_l2_mask
unformat_function_t unformat_ip_next_index
#define foreach_size_in_u32x4
static vnet_classify_entry_t * vnet_classify_find_entry_inline(vnet_classify_table_t *t, u8 *h, u64 hash, f64 now)
#define VNET_CLASSIFY_ENTRY_FREE
unformat_function_t unformat_ip6_mask
static u64 clib_xxhash(u64 key)
int vnet_classify_add_del_session(vnet_classify_main_t *cm, u32 table_index, u8 *match, u32 hit_next_index, u32 opaque_index, i32 advance, u8 action, u32 metadata, int is_add)
struct _vlib_node_registration vlib_node_registration_t
unformat_function_t unformat_classify_match
vnet_classify_table_t * vnet_classify_new_table(vnet_classify_main_t *cm, u8 *mask, u32 nbuckets, u32 memory_size, u32 skip_n_vectors, u32 match_n_vectors)
unformat_function_t unformat_l3_mask
unformat_function_t unformat_ip4_mask
unformat_function_t unformat_classify_mask
typedef CLIB_PACKED(struct _vnet_classify_entry{u32 next_index;union{struct{u32 opaque_index;i32 advance;};u64 opaque_count;};u8 flags;#define VNET_CLASSIFY_ENTRY_FREE vnet_classify_action_t action;u16 metadata;union{u64 hits;struct _vnet_classify_entry *next_free;};f64 last_heard;u32x4 key[0];}) vnet_classify_entry_t
void vnet_classify_register_unformat_opaque_index_fn(unformat_function_t *fn)
static void vnet_classify_prefetch_bucket(vnet_classify_table_t *t, u64 hash)
u8 * format_classify_table(u8 *s, va_list *args)
static void vnet_classify_prefetch_entry(vnet_classify_table_t *t, u64 hash)
static int vnet_classify_entry_is_free(vnet_classify_entry_t *e)
static u64 vnet_classify_hash_packet_inline(vnet_classify_table_t *t, u8 *h)
unformat_function_t unformat_l3_match
vnet_classify_entry_t * entries
unformat_function_t unformat_l2_next_index
void vnet_classify_register_unformat_policer_next_index_fn(unformat_function_t *fn)
static vnet_classify_entry_t * vnet_classify_entry_at_index(vnet_classify_table_t *t, vnet_classify_entry_t *e, u32 index)
static uword vnet_classify_get_offset(vnet_classify_table_t *t, vnet_classify_entry_t *v)
vnet_classify_bucket_t saved_bucket
#define CLIB_PREFETCH(addr, size, type)
void vnet_classify_register_unformat_l2_next_index_fn(unformat_function_t *fn)
void vnet_classify_register_unformat_acl_next_index_fn(unformat_function_t *fn)
unformat_function_t unformat_ip6_match
static uword is_pow2(uword x)
int vlib_main(vlib_main_t *volatile vm, unformat_input_t *input)
static int vnet_classify_entry_is_busy(vnet_classify_entry_t *e)
template key/value backing page structure
vlib_node_registration_t ip6_classify_node
(constructor) VLIB_REGISTER_NODE (ip6_classify_node)
vnet_classify_bucket_t * buckets
vnet_classify_entry_t * vnet_classify_find_entry(vnet_classify_table_t *t, u8 *h, u64 hash, f64 now)
vlib_node_registration_t ip4_classify_node
(constructor) VLIB_REGISTER_NODE (ip4_classify_node)
unformat_function_t unformat_l4_match
volatile u32 * writer_lock
void vnet_classify_register_unformat_ip_next_index_fn(unformat_function_t *fn)
struct clib_bihash_value offset
template key/value backing page structure
unformat_function_t unformat_l2_match
int * working_copy_lengths
#define CLIB_CACHE_LINE_BYTES
vnet_classify_entry_t ** freelists
vnet_classify_main_t vnet_classify_main
static u32 u32x4_zero_byte_mask(u32x4 x)
static vnet_classify_entry_t * vnet_classify_get_entry(vnet_classify_table_t *t, uword offset)
int vnet_classify_add_del_table(vnet_classify_main_t *cm, u8 *mask, u32 nbuckets, u32 memory_size, u32 skip, u32 match, u32 next_table_index, u32 miss_next_index, u32 *table_index, u8 current_data_flag, i16 current_data_offset, int is_add, int del_chain)