FD.io VPP  v18.01.2-1-g9b554f3
Vector Packet Processing
ipsec.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 Intel and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #ifndef __DPDK_IPSEC_H__
16 #define __DPDK_IPSEC_H__
17 
18 #include <vnet/vnet.h>
19 #include <vppinfra/cache.h>
20 #include <vnet/ipsec/ipsec.h>
21 
22 #undef always_inline
23 #include <rte_config.h>
24 #include <rte_crypto.h>
25 #include <rte_cryptodev.h>
26 
27 #if CLIB_DEBUG > 0
28 #define always_inline static inline
29 #else
30 #define always_inline static inline __attribute__ ((__always_inline__))
31 #endif
32 
33 #define foreach_dpdk_crypto_input_next \
34  _(DROP, "error-drop") \
35  _(IP4_LOOKUP, "ip4-lookup") \
36  _(IP6_LOOKUP, "ip6-lookup") \
37  _(INTERFACE_OUTPUT, "interface-output") \
38  _(DECRYPT_POST, "dpdk-esp-decrypt-post")
39 
40 typedef enum
41 {
42 #define _(f,s) DPDK_CRYPTO_INPUT_NEXT_##f,
44 #undef _
47 
48 #define MAX_QP_PER_LCORE 16
49 
50 typedef struct
51 {
53  u32 iv[2];
56 
57 typedef struct
58 {
60  dpdk_gcm_cnt_blk cb __attribute__ ((aligned (16)));
61  u8 aad[16];
62  u8 icv[32];
64 
65 typedef struct
66 {
68  struct rte_crypto_op **ops;
69  u16 cipher_resource_idx[IPSEC_CRYPTO_N_ALG];
70  u16 auth_resource_idx[IPSEC_INTEG_N_ALG];
71 } crypto_worker_main_t __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES)));
72 
73 typedef struct
74 {
75  char *name;
76  enum rte_crypto_sym_xform_type type;
84 } crypto_alg_t __attribute__ ((aligned (8)));
85 
86 typedef struct
87 {
90  u8 cipher_support[IPSEC_CRYPTO_N_ALG];
91  u8 auth_support[IPSEC_INTEG_N_ALG];
95  const i8 *name;
98 } crypto_dev_t;
99 
100 typedef struct
101 {
102  const i8 *name;
104 } crypto_drv_t;
105 
106 typedef struct
107 {
109  u8 remove;
114  u16 inflights[2];
116  u16 __unused;
117  struct rte_crypto_op *ops[VLIB_FRAME_SIZE];
119 } crypto_resource_t __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES)));
120 
121 typedef struct
122 {
124  struct rte_cryptodev_sym_session *session;
126 
127 typedef struct
128 {
129  struct rte_mempool *crypto_op;
130  struct rte_mempool *session_h;
131  struct rte_mempool **session_drv;
138 } crypto_data_t;
139 
140 typedef struct
141 {
143  struct rte_cryptodev_sym_session **sa_session;
150  u64 session_timeout; /* nsec */
153 
155 
156 static const u8 pad_data[] =
157  { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0 };
158 
159 void crypto_auto_placement (void);
160 
161 clib_error_t *create_sym_session (struct rte_cryptodev_sym_session **session,
162  u32 sa_idx, crypto_resource_t * res,
163  crypto_worker_main_t * cwm, u8 is_outbound);
164 
167 {
168  const u32 align = 4;
169  u32 op_size =
170  sizeof (struct rte_crypto_op) + sizeof (struct rte_crypto_sym_op);
171 
172  return ((op_size + align - 1) & ~(align - 1)) + sizeof (dpdk_op_priv_t);
173 }
174 
177 {
178  const u32 align = 16;
179  u32 offset;
180 
181  offset = sizeof (struct rte_crypto_op) + sizeof (struct rte_crypto_sym_op);
182  offset = (offset + align - 1) & ~(align - 1);
183 
184  return offset;
185 }
186 
188 crypto_op_get_priv (struct rte_crypto_op * op)
189 {
190  return (dpdk_op_priv_t *) (((u8 *) op) + crypto_op_get_priv_offset ());
191 }
192 
193 /* XXX this requires 64 bit builds so hash_xxx macros use u64 key */
194 typedef union
195 {
197  struct
198  {
201  };
203 
205 crypto_get_session (struct rte_cryptodev_sym_session **session,
206  u32 sa_idx,
207  crypto_resource_t * res,
208  crypto_worker_main_t * cwm, u8 is_outbound)
209 {
211  crypto_data_t *data;
212  uword *val;
213  crypto_session_key_t key = { 0 };
214 
215  key.drv_id = res->drv_id;
216  key.sa_idx = sa_idx;
217 
218  data = vec_elt_at_index (dcm->data, res->numa);
219  val = hash_get (data->session_by_drv_id_and_sa_index, key.val);
220 
221  if (PREDICT_FALSE (!val))
222  return create_sym_session (session, sa_idx, res, cwm, is_outbound);
223 
224  session[0] = (struct rte_cryptodev_sym_session *) val[0];
225 
226  return NULL;
227 }
228 
231 {
232  u16 cipher_res = cwm->cipher_resource_idx[sa->crypto_alg];
233  u16 auth_res = cwm->auth_resource_idx[sa->integ_alg];
234  u8 is_aead;
235 
236  /* Not allowed to setup SA with no-aead-cipher/NULL or NULL/NULL */
237 
238  is_aead = ((sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) |
239  (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_192) |
240  (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_256));
241 
242  if (sa->crypto_alg == IPSEC_CRYPTO_ALG_NONE)
243  return auth_res;
244 
245  if (cipher_res == auth_res)
246  return cipher_res;
247 
248  if (is_aead)
249  return cipher_res;
250 
251  return (u16) ~ 0;
252 }
253 
255 crypto_alloc_ops (u8 numa, struct rte_crypto_op ** ops, u32 n)
256 {
258  crypto_data_t *data = vec_elt_at_index (dcm->data, numa);
259  i32 ret;
260 
261  ret = rte_mempool_get_bulk (data->crypto_op, (void **) ops, n);
262 
263  /* *INDENT-OFF* */
264  data->crypto_op_get_failed += ! !ret;
265  /* *INDENT-ON* */
266 
267  return ret;
268 }
269 
271 crypto_free_ops (u8 numa, struct rte_crypto_op **ops, u32 n)
272 {
274  crypto_data_t *data = vec_elt_at_index (dcm->data, numa);
275 
276  if (!n)
277  return;
278 
279  rte_mempool_put_bulk (data->crypto_op, (void **) ops, n);
280 }
281 
284  u32 node_index, u32 error, u8 numa)
285 {
287  crypto_resource_t *res;
288  u16 *res_idx;
289 
290  /* *INDENT-OFF* */
291  vec_foreach (res_idx, cwm->resource_idx)
292  {
293  u16 enq;
294  res = vec_elt_at_index (dcm->resource, res_idx[0]);
295 
296  if (!res->n_ops)
297  continue;
298 
299  enq = rte_cryptodev_enqueue_burst (res->dev_id, res->qp_id + outbound,
300  res->ops, res->n_ops);
301  res->inflights[outbound] += enq;
302 
303  if (PREDICT_FALSE (enq < res->n_ops))
304  {
305  crypto_free_ops (numa, &res->ops[enq], res->n_ops - enq);
306  vlib_buffer_free (vm, &res->bi[enq], res->n_ops - enq);
307 
308  vlib_node_increment_counter (vm, node_index, error,
309  res->n_ops - enq);
310  }
311  res->n_ops = 0;
312  }
313  /* *INDENT-ON* */
314 }
315 
317 crypto_set_icb (dpdk_gcm_cnt_blk * icb, u32 salt, u32 seq, u32 seq_hi)
318 {
319  icb->salt = salt;
320  icb->iv[0] = seq;
321  icb->iv[1] = seq_hi;
322 }
323 
325 crypto_op_setup (u8 is_aead, struct rte_mbuf *mb0,
326  struct rte_crypto_op *op, void *session,
327  u32 cipher_off, u32 cipher_len,
328  u32 auth_off, u32 auth_len,
329  u8 * aad, u8 * digest, u64 digest_paddr)
330 {
331  struct rte_crypto_sym_op *sym_op;
332 
333  sym_op = (struct rte_crypto_sym_op *) (op + 1);
334 
335  sym_op->m_src = mb0;
336  sym_op->session = session;
337 
338  if (is_aead)
339  {
340  sym_op->aead.data.offset = cipher_off;
341  sym_op->aead.data.length = cipher_len;
342 
343  sym_op->aead.aad.data = aad;
344  sym_op->aead.aad.phys_addr =
345  op->phys_addr + (uintptr_t) aad - (uintptr_t) op;
346 
347  sym_op->aead.digest.data = digest;
348  sym_op->aead.digest.phys_addr = digest_paddr;
349  }
350  else
351  {
352  sym_op->cipher.data.offset = cipher_off;
353  sym_op->cipher.data.length = cipher_len;
354 
355  sym_op->auth.data.offset = auth_off;
356  sym_op->auth.data.length = auth_len;
357 
358  sym_op->auth.digest.data = digest;
359  sym_op->auth.digest.phys_addr = digest_paddr;
360  }
361 }
362 
363 #endif /* __DPDK_IPSEC_H__ */
364 
365 /*
366  * fd.io coding-style-patch-verification: ON
367  *
368  * Local Variables:
369  * eval: (c-set-style "gnu")
370  * End:
371  */
u32 alg
Definition: ipsec.h:77
static_always_inline void crypto_op_setup(u8 is_aead, struct rte_mbuf *mb0, struct rte_crypto_op *op, void *session, u32 cipher_off, u32 cipher_len, u32 auth_off, u32 auth_len, u8 *aad, u8 *digest, u64 digest_paddr)
Definition: ipsec.h:325
clib_error_t * create_sym_session(struct rte_cryptodev_sym_session **session, u32 sa_idx, crypto_resource_t *res, crypto_worker_main_t *cwm, u8 is_outbound)
Definition: ipsec.c:319
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:356
#define foreach_dpdk_crypto_input_next
Definition: ipsec.h:33
u8 numa
Definition: ipsec.h:93
u64 crypto_op_get_failed
Definition: ipsec.h:135
#define NULL
Definition: clib.h:55
u64 session_h_failed
Definition: ipsec.h:136
static_always_inline i32 crypto_alloc_ops(u8 numa, struct rte_crypto_op **ops, u32 n)
Definition: ipsec.h:255
ipsec_integ_alg_t integ_alg
Definition: ipsec.h:121
static_always_inline void crypto_set_icb(dpdk_gcm_cnt_blk *icb, u32 salt, u32 seq, u32 seq_hi)
Definition: ipsec.h:317
static_always_inline u32 crypto_op_get_priv_offset(void)
Definition: ipsec.h:176
struct rte_cryptodev_sym_session * session
Definition: ipsec.h:124
u8 disabled
Definition: ipsec.h:82
static_always_inline void crypto_enqueue_ops(vlib_main_t *vm, crypto_worker_main_t *cwm, u8 outbound, u32 node_index, u32 error, u8 numa)
Definition: ipsec.h:283
const i8 * name
Definition: ipsec.h:95
dpdk_crypto_main_t dpdk_crypto_main
Definition: ipsec.c:24
u32 bi[VLIB_FRAME_SIZE]
Definition: ipsec.h:118
u16 * devs
Definition: ipsec.h:103
u16 * resource_idx
Definition: ipsec.h:67
static_always_inline clib_error_t * crypto_get_session(struct rte_cryptodev_sym_session **session, u32 sa_idx, crypto_resource_t *res, crypto_worker_main_t *cwm, u8 is_outbound)
Definition: ipsec.h:205
static_always_inline void crypto_free_ops(u8 numa, struct rte_crypto_op **ops, u32 n)
Definition: ipsec.h:271
#define static_always_inline
Definition: clib.h:93
u16 cipher_resource_idx[IPSEC_CRYPTO_N_ALG]
Definition: ipsec.h:69
u8 resources
Definition: ipsec.h:83
int i32
Definition: types.h:81
crypto_drv_t * drv
Definition: ipsec.h:149
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
char i8
Definition: types.h:45
u8 drv_id
Definition: ipsec.h:92
unsigned long u64
Definition: types.h:89
u16 * free_resources
Definition: ipsec.h:88
crypto_alg_t * auth_algs
Definition: ipsec.h:147
u32 iv[2]
Definition: ipsec.h:53
u8 trunc_size
Definition: ipsec.h:80
u32 next
Definition: ipsec.h:59
void crypto_auto_placement(void)
Definition: ipsec.c:726
u16 * used_resources
Definition: ipsec.h:89
#define hash_get(h, key)
Definition: hash.h:248
crypto_alg_t * cipher_algs
Definition: ipsec.h:146
u64 features
Definition: ipsec.h:97
#define PREDICT_FALSE(x)
Definition: clib.h:105
#define VLIB_FRAME_SIZE
Definition: node.h:328
u64 session_timeout
Definition: ipsec.h:150
static_always_inline u32 crypto_op_len(void)
Definition: ipsec.h:166
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
Definition: node_funcs.h:1158
struct rte_mempool ** session_drv
Definition: ipsec.h:131
vlib_main_t * vm
Definition: buffer.c:283
crypto_session_disposal_t * session_disposal
Definition: ipsec.h:132
dpdk_crypto_input_next_t
Definition: ipsec.h:40
u16 id
Definition: ipsec.h:94
u8 iv_len
Definition: ipsec.h:79
unsigned int u32
Definition: types.h:88
struct rte_mempool * crypto_op
Definition: ipsec.h:129
u32 max_qp
Definition: ipsec.h:96
u16 inflights[2]
Definition: ipsec.h:114
crypto_worker_main_t * workers_main
Definition: ipsec.h:142
u8 boundary
Definition: ipsec.h:81
crypto_resource_t * resource
Definition: ipsec.h:145
u64 uword
Definition: types.h:112
struct rte_mempool * session_h
Definition: ipsec.h:130
const i8 * name
Definition: ipsec.h:102
unsigned short u16
Definition: types.h:57
char * name
Definition: ipsec.h:75
unsigned char u8
Definition: types.h:56
u64 * session_drv_failed
Definition: ipsec.h:137
uword * session_by_sa_index
Definition: ipsec.h:133
struct rte_crypto_op ** ops
Definition: ipsec.h:68
struct clib_bihash_value offset
template key/value backing page structure
crypto_dev_t * dev
Definition: ipsec.h:144
static_always_inline u16 get_resource(crypto_worker_main_t *cwm, ipsec_sa_t *sa)
Definition: ipsec.h:230
crypto_data_t * data
Definition: ipsec.h:148
ipsec_crypto_alg_t crypto_alg
Definition: ipsec.h:117
uword * session_by_drv_id_and_sa_index
Definition: ipsec.h:134
u16 auth_resource_idx[IPSEC_INTEG_N_ALG]
Definition: ipsec.h:70
#define vec_foreach(var, vec)
Vector iterator.
u8 key_len
Definition: ipsec.h:78
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
struct rte_cryptodev_sym_session ** sa_session
Definition: ipsec.h:143
struct rte_crypto_op * ops[VLIB_FRAME_SIZE]
Definition: ipsec.h:117
static_always_inline dpdk_op_priv_t * crypto_op_get_priv(struct rte_crypto_op *op)
Definition: ipsec.h:188