26 #define EMPTY_STRUCT {0} 38 dcm->cipher_algs[IPSEC_CRYPTO_ALG_##f].name = str; \ 39 dcm->cipher_algs[IPSEC_CRYPTO_ALG_##f].disabled = n_mains; 46 a->
type = RTE_CRYPTO_SYM_XFORM_CIPHER;
47 a->alg = RTE_CRYPTO_CIPHER_NULL;
52 a = &dcm->
cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_128];
53 a->
type = RTE_CRYPTO_SYM_XFORM_CIPHER;
54 a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
59 a = &dcm->
cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_192];
60 a->
type = RTE_CRYPTO_SYM_XFORM_CIPHER;
61 a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
66 a = &dcm->
cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_256];
67 a->
type = RTE_CRYPTO_SYM_XFORM_CIPHER;
68 a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
73 a = &dcm->
cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_128];
74 a->
type = RTE_CRYPTO_SYM_XFORM_CIPHER;
75 a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
80 a = &dcm->
cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_192];
81 a->
type = RTE_CRYPTO_SYM_XFORM_CIPHER;
82 a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
87 a = &dcm->
cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_256];
88 a->
type = RTE_CRYPTO_SYM_XFORM_CIPHER;
89 a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
94 #define AES_GCM_TYPE RTE_CRYPTO_SYM_XFORM_AEAD 95 #define AES_GCM_ALG RTE_CRYPTO_AEAD_AES_GCM 97 a = &dcm->
cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_128];
105 a = &dcm->
cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_192];
113 a = &dcm->
cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_256];
125 dcm->auth_algs[IPSEC_INTEG_ALG_##f].name = str; \ 126 dcm->auth_algs[IPSEC_INTEG_ALG_##f].disabled = n_mains; 131 a = &dcm->
auth_algs[IPSEC_INTEG_ALG_NONE];
132 a->
type = RTE_CRYPTO_SYM_XFORM_AUTH;
133 a->alg = RTE_CRYPTO_AUTH_NULL;
137 a = &dcm->
auth_algs[IPSEC_INTEG_ALG_MD5_96];
138 a->
type = RTE_CRYPTO_SYM_XFORM_AUTH;
139 a->alg = RTE_CRYPTO_AUTH_MD5_HMAC;
143 a = &dcm->
auth_algs[IPSEC_INTEG_ALG_SHA1_96];
144 a->
type = RTE_CRYPTO_SYM_XFORM_AUTH;
145 a->alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
149 a = &dcm->
auth_algs[IPSEC_INTEG_ALG_SHA_256_96];
150 a->
type = RTE_CRYPTO_SYM_XFORM_AUTH;
151 a->alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
155 a = &dcm->
auth_algs[IPSEC_INTEG_ALG_SHA_256_128];
156 a->
type = RTE_CRYPTO_SYM_XFORM_AUTH;
157 a->alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
161 a = &dcm->
auth_algs[IPSEC_INTEG_ALG_SHA_384_192];
162 a->
type = RTE_CRYPTO_SYM_XFORM_AUTH;
163 a->alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
167 a = &dcm->
auth_algs[IPSEC_INTEG_ALG_SHA_512_256];
168 a->
type = RTE_CRYPTO_SYM_XFORM_AUTH;
169 a->alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
196 if (cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
202 if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
203 (alg->
type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
204 (cap->sym.cipher.algo == alg->
alg) &&
207 if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) &&
208 (alg->
type == RTE_CRYPTO_SYM_XFORM_AEAD) &&
209 (cap->sym.aead.algo == alg->
alg) &&
224 if ((cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) ||
225 (cap->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH))
231 if ((cap->sym.auth.algo == alg->
alg) &&
249 ASSERT (c->
type == RTE_CRYPTO_SYM_XFORM_AEAD);
251 xform->type = RTE_CRYPTO_SYM_XFORM_AEAD;
252 xform->aead.algo = c->
alg;
254 xform->aead.key.length = c->
key_len;
255 xform->aead.iv.offset =
257 xform->aead.iv.length = 12;
259 xform->aead.aad_length = sa->
use_esn ? 12 : 8;
263 xform->aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT;
265 xform->aead.op = RTE_CRYPTO_AEAD_OP_DECRYPT;
277 ASSERT (c->
type == RTE_CRYPTO_SYM_XFORM_CIPHER);
279 xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
280 xform->cipher.algo = c->
alg;
282 xform->cipher.key.length = c->
key_len;
283 xform->cipher.iv.offset =
285 xform->cipher.iv.length = c->
iv_len;
289 xform->cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
291 xform->cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
303 ASSERT (a->
type == RTE_CRYPTO_SYM_XFORM_AUTH);
305 xform->type = RTE_CRYPTO_SYM_XFORM_AUTH;
306 xform->auth.algo = a->
alg;
308 xform->auth.key.length = a->
key_len;
313 xform->auth.op = RTE_CRYPTO_AUTH_OP_GENERATE;
315 xform->auth.op = RTE_CRYPTO_AUTH_OP_VERIFY;
328 struct rte_crypto_sym_xform cipher_xform = { 0 };
329 struct rte_crypto_sym_xform auth_xform = { 0 };
330 struct rte_crypto_sym_xform *xfs;
331 struct rte_cryptodev_sym_session **s;
341 if ((sa->
crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) |
342 (sa->
crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_192) |
343 (sa->
crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_256))
355 cipher_xform.next = &auth_xform;
360 auth_xform.next = &cipher_xform;
374 session[0] = rte_cryptodev_sym_session_create (data->
session_h);
385 struct rte_mempool **mp;
390 rte_cryptodev_sym_session_init (res->
dev_id, session[0], xfs, mp[0]);
405 struct rte_mempool *mp = rte_mempool_from_obj (obj);
407 memset (obj, 0, mp->elt_size);
409 rte_mempool_put (mp, obj);
417 return sess->sess_private_data[driver_id];
423 uint8_t driver_id,
void *private_data)
425 sess->sess_private_data[driver_id] = private_data;
460 ret = rte_cryptodev_sym_session_free (s->
session);
477 struct rte_cryptodev_sym_session *s;
489 case IPSEC_CRYPTO_ALG_AES_GCM_128:
490 case IPSEC_CRYPTO_ALG_AES_GCM_192:
491 case IPSEC_CRYPTO_ALG_AES_GCM_256:
508 s = (
struct rte_cryptodev_sym_session *) val;
517 s = (
struct rte_cryptodev_sym_session *) val;
546 if (sa->
integ_alg == IPSEC_INTEG_ALG_NONE)
549 case IPSEC_CRYPTO_ALG_AES_GCM_128:
550 case IPSEC_CRYPTO_ALG_AES_GCM_192:
551 case IPSEC_CRYPTO_ALG_AES_GCM_256:
560 if (sa->
crypto_alg != IPSEC_CRYPTO_ALG_NONE &&
566 if (sa->
integ_alg != IPSEC_INTEG_ALG_NONE &&
575 const struct rte_cryptodev_capabilities *cap,
582 for (; cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; cap++)
585 switch (cap->sym.xform_type)
587 case RTE_CRYPTO_SYM_XFORM_AEAD:
588 case RTE_CRYPTO_SYM_XFORM_CIPHER:
589 inc = cap->sym.cipher.key_size.increment;
591 for (len = cap->sym.cipher.key_size.min;
592 len <= cap->sym.cipher.key_size.max; len += inc)
603 case RTE_CRYPTO_SYM_XFORM_AUTH:
604 inc = cap->sym.auth.digest_size.increment;
606 for (len = cap->sym.auth.digest_size.min;
607 len <= cap->sym.auth.digest_size.max; len += inc)
624 #define DPDK_CRYPTO_N_QUEUE_DESC 2048 625 #define DPDK_CRYPTO_NB_SESS_OBJS 20000 630 struct rte_cryptodev_config dev_conf;
631 struct rte_cryptodev_qp_conf qp_conf;
636 dev_conf.socket_id = numa;
637 dev_conf.nb_queue_pairs = n_qp;
639 error_str =
"failed to configure crypto device %u";
640 ret = rte_cryptodev_configure (dev, &dev_conf);
644 error_str =
"failed to setup crypto device %u queue pair %u";
646 for (qp = 0; qp < n_qp; qp++)
648 ret = rte_cryptodev_queue_pair_setup (dev, qp, &qp_conf, numa,
NULL);
660 struct rte_cryptodev *cryptodev;
661 struct rte_cryptodev_info info;
666 u16 max_res_idx, res_idx, j;
672 for (i = 0; i < rte_cryptodev_count (); i++)
676 cryptodev = &rte_cryptodevs[
i];
677 rte_cryptodev_info_get (i, &info);
680 dev->name = cryptodev->data->name;
681 dev->numa = rte_cryptodev_socket_id (i);
682 dev->features = info.feature_flags;
683 dev->max_qp = info.max_nb_queue_pairs;
684 drv_id = info.driver_id;
689 dev->drv_id = drv_id;
692 if (!(info.feature_flags & RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING))
701 max_res_idx = (dev->max_qp / 2) - 1;
710 for (j = 0; j <= max_res_idx; j++, res_idx++)
712 vec_elt (dev->free_resources, max_res_idx - j) = res_idx;
717 res->
numa = dev->numa;
732 u32 thread_idx, skip_master;
747 if (thread_idx < skip_master)
801 void *_arg __attribute__ ((unused)),
802 void *_obj,
unsigned i __attribute__ ((unused)))
804 struct rte_crypto_op *op = _obj;
806 op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
807 op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
808 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
809 #if RTE_VERSION < RTE_VERSION_NUM(17, 11, 0, 0) 810 op->phys_addr = rte_mempool_virt2phy (
NULL, _obj);
812 op->phys_addr = rte_mempool_virt2iova (_obj);
814 op->mempool = mempool;
824 u32 pool_priv_size =
sizeof (
struct rte_crypto_op_pool_private);
825 struct rte_crypto_op_pool_private *priv;
826 struct rte_mempool *mp;
836 pool_name =
format (0,
"crypto_pool_numa%u%c", numa, 0);
840 pool_priv_size, 512, numa, &mp, &pri);
848 priv = rte_mempool_get_priv (mp);
849 priv->priv_size = pool_priv_size;
850 priv->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
866 struct rte_mempool *mp;
876 pool_name =
format (0,
"session_h_pool_numa%u%c", numa, 0);
878 elt_size = rte_cryptodev_get_header_session_size ();
882 0, 512, numa, &mp, &pri);
900 struct rte_mempool *mp;
914 pool_name =
format (0,
"session_drv%u_pool_numa%u%c", dev->
drv_id, numa, 0);
915 elt_size = rte_cryptodev_get_private_session_size (dev->
id);
919 0, 512, numa, &mp, &pri);
1000 u32 i, skip_master, n_mains;
1011 clib_warning (
"not enough DPDK crypto resources, default to OpenSSL");
1068 for (i = skip_master; i < n_mains; i++)
1077 .name =
"dpdk-ipsec-process",
1078 .process_log2_n_stack_bytes = 17,
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define hash_set(h, key, value)
sll srl srl sll sra u16x4 i
#define hash_unset(h, key)
static u8 cipher_alg_index(const crypto_alg_t *alg)
#define DPDK_CRYPTO_NB_SESS_OBJS
ipsec_integ_alg_t integ_alg
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
static clib_error_t * crypto_create_session_drv_pool(vlib_main_t *vm, crypto_dev_t *dev)
static u64 clib_cpu_time_now(void)
static_always_inline u32 crypto_op_get_priv_offset(void)
struct rte_cryptodev_sym_session * session
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
static void crypto_set_cipher_xform(struct rte_crypto_sym_xform *xform, ipsec_sa_t *sa, u8 is_outbound)
vlib_main_t ** vlib_mains
static clib_error_t * crypto_create_session_h_pool(vlib_main_t *vm, u8 numa)
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
#define vec_pop(V)
Returns last element of a vector and decrements its length.
static crypto_alg_t * cipher_cap_to_alg(const struct rte_cryptodev_capabilities *cap, u8 key_len)
static clib_error_t * add_del_sa_session(u32 sa_index, u8 is_add)
u16 cipher_resource_idx[IPSEC_CRYPTO_N_ALG]
u32 esp_encrypt_next_index
static void algos_init(u32 n_mains)
dpdk_crypto_main_t dpdk_crypto_main
dpdk_config_main_t dpdk_config_main
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
static void crypto_parse_capabilities(crypto_dev_t *dev, const struct rte_cryptodev_capabilities *cap, u32 n_mains)
ipsec_main_callbacks_t cb
#define vec_end(v)
End (last data address) of vector.
u8 * format_ipsec_crypto_alg(u8 *s, va_list *args)
static clib_error_t * crypto_dev_conf(u8 dev, u16 n_qp, u8 numa)
static void crypto_disable(void)
static void * get_session_private_data(const struct rte_cryptodev_sym_session *sess, uint8_t driver_id)
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
crypto_alg_t * cipher_algs
clib_error_t *(* check_support_cb)(ipsec_sa_t *sa)
static_always_inline u32 crypto_op_len(void)
#define foreach_ipsec_integ_alg
static clib_error_t * crypto_create_crypto_op_pool(vlib_main_t *vm, u8 numa)
static u8 auth_alg_index(const crypto_alg_t *alg)
struct rte_mempool ** session_drv
clib_error_t * dpdk_pool_create(vlib_main_t *vm, u8 *pool_name, u32 elt_size, u32 num_elts, u32 pool_priv_size, u16 cache_size, u8 numa, struct rte_mempool **_mp, vlib_physmem_region_index_t *pri)
#define foreach_ipsec_crypto_alg
u32 esp_encrypt_node_index
#define vec_free(V)
Free vector's memory (no header).
u32 esp_decrypt_next_index
#define clib_warning(format, args...)
#define clib_memcpy(a, b, c)
static u64 unix_time_now_nsec(void)
crypto_session_disposal_t * session_disposal
static void crypto_set_aead_xform(struct rte_crypto_sym_xform *xform, ipsec_sa_t *sa, u8 is_outbound)
vlib_node_t * vlib_get_node_by_name(vlib_main_t *vm, u8 *name)
static void set_session_private_data(struct rte_cryptodev_sym_session *sess, uint8_t driver_id, void *private_data)
#define vec_delete(V, N, M)
Delete N elements starting at element M.
struct rte_mempool * crypto_op
static clib_error_t * dpdk_crypto_session_disposal(crypto_session_disposal_t *v, u64 ts)
crypto_worker_main_t * workers_main
#define clib_error_report(e)
static void vlib_node_set_state(vlib_main_t *vm, u32 node_index, vlib_node_state_t new_state)
Set node dispatch state.
crypto_resource_t * resource
#define vec_elt(v, i)
Get vector value at index i.
u8 * format_ipsec_integ_alg(u8 *s, va_list *args)
static clib_error_t * crypto_create_pools(vlib_main_t *vm)
struct rte_mempool * session_h
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
void crypto_auto_placement(void)
uword * session_by_sa_index
static void crypto_scan_devs(u32 n_mains)
static void crypto_op_init(struct rte_mempool *mempool, void *_arg, void *_obj, unsigned i)
struct rte_crypto_op ** ops
clib_error_t * create_sym_session(struct rte_cryptodev_sym_session **session, u32 sa_idx, crypto_resource_t *res, crypto_worker_main_t *cwm, u8 is_outbound)
enum rte_crypto_sym_xform_type type
#define DPDK_CRYPTO_N_QUEUE_DESC
static u32 random_u32(u32 *seed)
32-bit random number generator
ipsec_crypto_alg_t crypto_alg
#define VLIB_REGISTER_NODE(x,...)
uword * session_by_drv_id_and_sa_index
static vlib_thread_main_t * vlib_get_thread_main()
static u32 vlib_num_workers()
static uword dpdk_ipsec_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
u16 auth_resource_idx[IPSEC_INTEG_N_ALG]
u8 auth_support[IPSEC_INTEG_N_ALG]
#define vec_validate_init_empty_aligned(V, I, INIT, A)
Make sure vector is long enough for given index and initialize empty space (no header, alignment alignment)
#define vec_foreach(var, vec)
Vector iterator.
clib_error_t *(* add_del_sa_sess_cb)(u32 sa_index, u8 is_add)
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
#define CLIB_CACHE_LINE_BYTES
u8 vlib_physmem_region_index_t
struct rte_cryptodev_sym_session ** sa_session
static void crypto_set_auth_xform(struct rte_crypto_sym_xform *xform, ipsec_sa_t *sa, u8 is_outbound)
static clib_error_t * dpdk_ipsec_check_support(ipsec_sa_t *sa)
u32 esp_decrypt_node_index
u8 cipher_support[IPSEC_CRYPTO_N_ALG]
static void clear_and_free_obj(void *obj)
static crypto_alg_t * auth_cap_to_alg(const struct rte_cryptodev_capabilities *cap, u8 trunc_size)