FD.io VPP  v17.07.01-10-g3be13f0
Vector Packet Processing
memory_shared.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * memclnt_shared.c - API message handling, common code for both clients
4  * and the vlib process itself.
5  *
6  *
7  * Copyright (c) 2009 Cisco and/or its affiliates.
8  * Licensed under the Apache License, Version 2.0 (the "License");
9  * you may not use this file except in compliance with the License.
10  * You may obtain a copy of the License at:
11  *
12  * http://www.apache.org/licenses/LICENSE-2.0
13  *
14  * Unless required by applicable law or agreed to in writing, software
15  * distributed under the License is distributed on an "AS IS" BASIS,
16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17  * See the License for the specific language governing permissions and
18  * limitations under the License.
19  *------------------------------------------------------------------
20  */
21 
22 #include <stdio.h>
23 #include <stdlib.h>
24 #include <stddef.h>
25 #include <string.h>
26 #include <unistd.h>
27 #include <signal.h>
28 #include <vppinfra/format.h>
29 #include <vppinfra/byte_order.h>
30 #include <vppinfra/error.h>
31 #include <vlib/vlib.h>
32 #include <vlib/unix/unix.h>
33 #include <vlibmemory/api.h>
35 
37 
38 #define vl_typedefs
40 #undef vl_typedefs
41 
42 static inline void *
43 vl_msg_api_alloc_internal (int nbytes, int pool, int may_return_null)
44 {
45  int i;
46  msgbuf_t *rv;
47  ring_alloc_t *ap;
49  void *oldheap;
51  api_main_t *am = &api_main;
52 
53  shmem_hdr = am->shmem_hdr;
54 
55  if (shmem_hdr == 0)
56  {
57  clib_warning ("shared memory header NULL");
58  return 0;
59  }
60 
61  /* account for the msgbuf_t header */
62  nbytes += sizeof (msgbuf_t);
63 
64  if (shmem_hdr->vl_rings == 0)
65  {
66  clib_warning ("vl_rings NULL");
67  ASSERT (0);
68  abort ();
69  }
70 
71  if (shmem_hdr->client_rings == 0)
72  {
73  clib_warning ("client_rings NULL");
74  ASSERT (0);
75  abort ();
76  }
77 
78  ap = pool ? shmem_hdr->vl_rings : shmem_hdr->client_rings;
79  for (i = 0; i < vec_len (ap); i++)
80  {
81  /* Too big? */
82  if (nbytes > ap[i].size)
83  {
84  continue;
85  }
86 
87  q = ap[i].rp;
88  if (pool == 0)
89  {
90  pthread_mutex_lock (&q->mutex);
91  }
92  rv = (msgbuf_t *) (&q->data[0] + q->head * q->elsize);
93  /*
94  * Is this item still in use?
95  */
96  if (rv->q)
97  {
98  u32 now = (u32) time (0);
99 
100  if (PREDICT_TRUE (rv->gc_mark_timestamp == 0))
101  rv->gc_mark_timestamp = now;
102  else
103  {
104  if (now - rv->gc_mark_timestamp > 10)
105  {
106  if (CLIB_DEBUG > 0)
107  {
108  u16 *msg_idp, msg_id;
110  ("garbage collect pool %d ring %d index %d", pool, i,
111  q->head);
112  msg_idp = (u16 *) (rv->data);
113  msg_id = clib_net_to_host_u16 (*msg_idp);
114  if (msg_id < vec_len (api_main.msg_names))
115  clib_warning ("msg id %d name %s", (u32) msg_id,
116  api_main.msg_names[msg_id]);
117  }
118  shmem_hdr->garbage_collects++;
119  goto collected;
120  }
121  }
122 
123 
124  /* yes, loser; try next larger pool */
125  ap[i].misses++;
126  if (pool == 0)
127  pthread_mutex_unlock (&q->mutex);
128  continue;
129  }
130  collected:
131 
132  /* OK, we have a winner */
133  ap[i].hits++;
134  /*
135  * Remember the source queue, although we
136  * don't need to know the queue to free the item.
137  */
138  rv->q = q;
139  rv->gc_mark_timestamp = 0;
140  q->head++;
141  if (q->head == q->maxsize)
142  q->head = 0;
143 
144  if (pool == 0)
145  pthread_mutex_unlock (&q->mutex);
146  goto out;
147  }
148 
149  /*
150  * Request too big, or head element of all size-compatible rings
151  * still in use. Fall back to shared-memory malloc.
152  */
153  am->ring_misses++;
154 
155  pthread_mutex_lock (&am->vlib_rp->mutex);
156  oldheap = svm_push_data_heap (am->vlib_rp);
157  if (may_return_null)
158  {
159  rv = clib_mem_alloc_or_null (nbytes);
160  if (PREDICT_FALSE (rv == 0))
161  {
162  svm_pop_heap (oldheap);
163  pthread_mutex_unlock (&am->vlib_rp->mutex);
164  return 0;
165  }
166  }
167  else
168  rv = clib_mem_alloc (nbytes);
169 
170  rv->q = 0;
171  svm_pop_heap (oldheap);
172  pthread_mutex_unlock (&am->vlib_rp->mutex);
173 
174 out:
175  rv->data_len = htonl (nbytes - sizeof (msgbuf_t));
176  return (rv->data);
177 }
178 
179 void *
180 vl_msg_api_alloc (int nbytes)
181 {
182  int pool;
183  api_main_t *am = &api_main;
185 
186  /*
187  * Clients use pool-0, vlib proc uses pool 1
188  */
189  pool = (am->our_pid == shmem_hdr->vl_pid);
190  return vl_msg_api_alloc_internal (nbytes, pool, 0 /* may_return_null */ );
191 }
192 
193 void *
195 {
196  int pool;
197  api_main_t *am = &api_main;
199 
200  pool = (am->our_pid == shmem_hdr->vl_pid);
201  return vl_msg_api_alloc_internal (nbytes, pool, 1 /* may_return_null */ );
202 }
203 
204 void *
206 {
207  return vl_msg_api_alloc_internal (nbytes, 0, 0 /* may_return_null */ );
208 }
209 
210 void *
212 {
213  return vl_msg_api_alloc_internal (nbytes, 0, 1 /* may_return_null */ );
214 }
215 
216 void
218 {
219  msgbuf_t *rv;
220  void *oldheap;
221  api_main_t *am = &api_main;
222 
223  rv = (msgbuf_t *) (((u8 *) a) - offsetof (msgbuf_t, data));
224 
225  /*
226  * Here's the beauty of the scheme. Only one proc/thread has
227  * control of a given message buffer. To free a buffer, we just clear the
228  * queue field, and leave. No locks, no hits, no errors...
229  */
230  if (rv->q)
231  {
232  rv->q = 0;
233  rv->gc_mark_timestamp = 0;
234  return;
235  }
236 
237  pthread_mutex_lock (&am->vlib_rp->mutex);
238  oldheap = svm_push_data_heap (am->vlib_rp);
239  clib_mem_free (rv);
240  svm_pop_heap (oldheap);
241  pthread_mutex_unlock (&am->vlib_rp->mutex);
242 }
243 
244 static void
246 {
247  msgbuf_t *rv;
248  void *oldheap;
249  api_main_t *am = &api_main;
250 
251  rv = (msgbuf_t *) (((u8 *) a) - offsetof (msgbuf_t, data));
252  /*
253  * Here's the beauty of the scheme. Only one proc/thread has
254  * control of a given message buffer. To free a buffer, we just clear the
255  * queue field, and leave. No locks, no hits, no errors...
256  */
257  if (rv->q)
258  {
259  rv->q = 0;
260  return;
261  }
262 
263  oldheap = svm_push_data_heap (am->vlib_rp);
264  clib_mem_free (rv);
265  svm_pop_heap (oldheap);
266 }
267 
268 void
269 vl_set_memory_root_path (const char *name)
270 {
271  api_main_t *am = &api_main;
272 
273  am->root_path = name;
274 }
275 
276 void
278 {
279  api_main_t *am = &api_main;
280 
281  am->api_uid = uid;
282 }
283 
284 void
286 {
287  api_main_t *am = &api_main;
288 
289  am->api_gid = gid;
290 }
291 
292 void
294 {
295  api_main_t *am = &api_main;
296 
297  am->global_baseva = baseva;
298 }
299 
300 void
302 {
303  api_main_t *am = &api_main;
304 
305  am->global_size = size;
306 }
307 
308 void
310 {
311  api_main_t *am = &api_main;
312 
313  am->api_size = size;
314 }
315 
316 void
318 {
319  api_main_t *am = &api_main;
320 
322 }
323 
324 void
326 {
327  api_main_t *am = &api_main;
328 
329  am->api_pvt_heap_size = size;
330 }
331 
332 int
333 vl_map_shmem (const char *region_name, int is_vlib)
334 {
335  svm_map_region_args_t _a, *a = &_a;
336  svm_region_t *vlib_rp, *root_rp;
337  void *oldheap;
339  api_main_t *am = &api_main;
340  int i;
341  struct timespec ts, tsrem;
342  u32 vlib_input_queue_length;
343 
344  if (is_vlib == 0)
346 
347  memset (a, 0, sizeof (*a));
348 
349  a->name = region_name;
350  a->size = am->api_size ? am->api_size : (16 << 20);
351  a->flags = SVM_FLAGS_MHEAP;
352  a->uid = am->api_uid;
353  a->gid = am->api_gid;
355 
356  vlib_rp = svm_region_find_or_create (a);
357 
358  if (vlib_rp == 0)
359  return (-2);
360 
361  pthread_mutex_lock (&vlib_rp->mutex);
362  /* Has someone else set up the shared-memory variable table? */
363  if (vlib_rp->user_ctx)
364  {
365  am->shmem_hdr = (void *) vlib_rp->user_ctx;
366  am->our_pid = getpid ();
367  if (is_vlib)
368  {
370  uword old_msg;
371  /*
372  * application restart. Reset cached pids, API message
373  * rings, list of clients; otherwise, various things
374  * fail. (e.g. queue non-empty notification)
375  */
376 
377  /* ghosts keep the region from disappearing properly */
380  q = am->shmem_hdr->vl_input_queue;
381  am->shmem_hdr->vl_pid = getpid ();
382  q->consumer_pid = am->shmem_hdr->vl_pid;
383  /* Drain the input queue, freeing msgs */
384  for (i = 0; i < 10; i++)
385  {
386  if (pthread_mutex_trylock (&q->mutex) == 0)
387  {
388  pthread_mutex_unlock (&q->mutex);
389  goto mutex_ok;
390  }
391  ts.tv_sec = 0;
392  ts.tv_nsec = 10000 * 1000; /* 10 ms */
393  while (nanosleep (&ts, &tsrem) < 0)
394  ts = tsrem;
395  }
396  /* Mutex buggered, "fix" it */
397  memset (&q->mutex, 0, sizeof (q->mutex));
398  clib_warning ("forcibly release main input queue mutex");
399 
400  mutex_ok:
401  am->vlib_rp = vlib_rp;
403  (u8 *) & old_msg,
404  1 /* nowait */ )
405  != -2 /* queue underflow */ )
406  {
407  vl_msg_api_free_nolock ((void *) old_msg);
409  }
410  pthread_mutex_unlock (&vlib_rp->mutex);
411  root_rp = svm_get_root_rp ();
412  ASSERT (root_rp);
413  /* Clean up the root region client list */
414  pthread_mutex_lock (&root_rp->mutex);
416  pthread_mutex_unlock (&root_rp->mutex);
417  }
418  else
419  {
420  pthread_mutex_unlock (&vlib_rp->mutex);
421  }
422  am->vlib_rp = vlib_rp;
423  vec_add1 (am->mapped_shmem_regions, vlib_rp);
424  return 0;
425  }
426  /* Clients simply have to wait... */
427  if (!is_vlib)
428  {
429  pthread_mutex_unlock (&vlib_rp->mutex);
430 
431  /* Wait up to 100 seconds... */
432  for (i = 0; i < 10000; i++)
433  {
434  ts.tv_sec = 0;
435  ts.tv_nsec = 10000 * 1000; /* 10 ms */
436  while (nanosleep (&ts, &tsrem) < 0)
437  ts = tsrem;
438  if (vlib_rp->user_ctx)
439  goto ready;
440  }
441  /* Clean up and leave... */
442  svm_region_unmap (vlib_rp);
443  clib_warning ("region init fail");
444  return (-2);
445 
446  ready:
447  am->shmem_hdr = (void *) vlib_rp->user_ctx;
448  am->our_pid = getpid ();
449  am->vlib_rp = vlib_rp;
450  vec_add1 (am->mapped_shmem_regions, vlib_rp);
451  return 0;
452  }
453 
454  /* Nope, it's our problem... */
455 
456  oldheap = svm_push_data_heap (vlib_rp);
457 
458  vec_validate (shmem_hdr, 0);
459  shmem_hdr->version = VL_SHM_VERSION;
460 
461  /* vlib main input queue */
462  vlib_input_queue_length = 1024;
463  if (am->vlib_input_queue_length)
464  vlib_input_queue_length = am->vlib_input_queue_length;
465 
466  shmem_hdr->vl_input_queue =
467  unix_shared_memory_queue_init (vlib_input_queue_length, sizeof (uword),
468  getpid (), am->vlib_signal);
469 
470  /* Set up the msg ring allocator */
471 #define _(sz,n) \
472  do { \
473  ring_alloc_t _rp; \
474  _rp.rp = unix_shared_memory_queue_init ((n), (sz), 0, 0); \
475  _rp.size = (sz); \
476  _rp.nitems = n; \
477  _rp.hits = 0; \
478  _rp.misses = 0; \
479  vec_add1(shmem_hdr->vl_rings, _rp); \
480  } while (0);
481 
483 #undef _
484 
485 #define _(sz,n) \
486  do { \
487  ring_alloc_t _rp; \
488  _rp.rp = unix_shared_memory_queue_init ((n), (sz), 0, 0); \
489  _rp.size = (sz); \
490  _rp.nitems = n; \
491  _rp.hits = 0; \
492  _rp.misses = 0; \
493  vec_add1(shmem_hdr->client_rings, _rp); \
494  } while (0);
495 
497 #undef _
498 
499  am->shmem_hdr = shmem_hdr;
500  am->vlib_rp = vlib_rp;
501  am->our_pid = getpid ();
502  if (is_vlib)
503  am->shmem_hdr->vl_pid = am->our_pid;
504 
505  svm_pop_heap (oldheap);
506 
507  /*
508  * After absolutely everything that a client might see is set up,
509  * declare the shmem region valid
510  */
511  vlib_rp->user_ctx = shmem_hdr;
512 
513  pthread_mutex_unlock (&vlib_rp->mutex);
514  vec_add1 (am->mapped_shmem_regions, vlib_rp);
515  return 0;
516 }
517 
518 void
520 {
521  api_main_t *am = &api_main;
522 
523  vec_add1 (am->mapped_shmem_regions, rp);
524 }
525 
526 void
528 {
529  svm_region_t *rp;
530  int i;
531  api_main_t *am = &api_main;
532 
533  if (!svm_get_root_rp ())
534  return;
535 
536  for (i = 0; i < vec_len (am->mapped_shmem_regions); i++)
537  {
538  rp = am->mapped_shmem_regions[i];
539  svm_region_unmap (rp);
540  }
541 
543  am->shmem_hdr = 0;
544 
545  svm_region_exit ();
546  /* $$$ more careful cleanup, valgrind run... */
547  vec_free (am->msg_handlers);
550 }
551 
552 void
554 {
555  api_main_t *am = &api_main;
556  uword *trace = (uword *) elem;
557 
558  if (am->tx_trace && am->tx_trace->enabled)
559  vl_msg_api_trace (am, am->tx_trace, (void *) trace[0]);
560 
561  (void) unix_shared_memory_queue_add (q, elem, 0 /* nowait */ );
562 }
563 
564 void
566 {
567  api_main_t *am = &api_main;
568  uword *trace = (uword *) elem;
569 
570  if (am->tx_trace && am->tx_trace->enabled)
571  vl_msg_api_trace (am, am->tx_trace, (void *) trace[0]);
572 
573  (void) unix_shared_memory_queue_add_nolock (q, elem);
574 }
575 
576 u32
577 vl_api_get_msg_index (u8 * name_and_crc)
578 {
579  api_main_t *am = &api_main;
580  uword *p;
581 
583  {
584  p = hash_get_mem (am->msg_index_by_name_and_crc, name_and_crc);
585  if (p)
586  return p[0];
587  }
588  return ~0;
589 }
590 
591 static inline vl_api_registration_t *
593 {
594  vl_api_registration_t **regpp;
595  vl_api_registration_t *regp;
596  api_main_t *am = &api_main;
597  u32 index;
598 
599  index = vl_msg_api_handle_get_index (handle);
601  != vl_msg_api_handle_get_epoch (handle))
602  {
604  return 0;
605  }
606 
607  regpp = am->vl_clients + index;
608 
609  if (pool_is_free (am->vl_clients, regpp))
610  {
612  return 0;
613  }
614  regp = *regpp;
615  return (regp);
616 }
617 
620 {
622 }
623 
626 {
627  vl_api_registration_t *regp;
628  api_main_t *am = &api_main;
629 
630  /* Special case: vlib trying to send itself a message */
631  if (index == (u32) ~ 0)
632  return (am->shmem_hdr->vl_input_queue);
633 
635  if (!regp)
636  return 0;
637  return (regp->vl_input_queue);
638 }
639 
640 /*
641  * fd.io coding-style-patch-verification: ON
642  *
643  * Local Variables:
644  * eval: (c-set-style "gnu")
645  * End:
646  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:436
u64 api_pvt_heap_size
Definition: api_common.h:221
svm_region_t * svm_get_root_rp(void)
Definition: svm.c:54
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:337
static void svm_pop_heap(void *oldheap)
Definition: svm.h:94
struct msgbuf_ msgbuf_t
a
Definition: bitmap.h:516
void vl_set_global_memory_baseva(u64 baseva)
static vlib_cli_command_t trace
(constructor) VLIB_CLI_COMMAND (trace)
Definition: memory_vlib.c:1173
int vl_map_shmem(const char *region_name, int is_vlib)
u64 api_size
Definition: api_common.h:215
u32 application_restarts
Definition: api_common.h:79
#define PREDICT_TRUE(x)
Definition: clib.h:98
void vl_set_memory_uid(int uid)
unix_shared_memory_queue_t * vl_input_queue
Definition: api_common.h:68
void vl_set_global_pvt_heap_size(u64 size)
void vl_unmap_shmem(void)
u32 gc_mark_timestamp
Definition: api_common.h:124
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:522
ring_alloc_t * client_rings
Definition: api_common.h:76
#define foreach_clnt_aring_size
Definition: api_common.h:55
u8 data[0]
Definition: api_common.h:125
#define pool_is_free(P, E)
Use free bitmap to query whether given element is free.
Definition: pool.h:230
static vl_api_registration_t * vl_api_client_index_to_registration_internal(u32 handle)
void * vl_msg_api_alloc(int nbytes)
#define VL_API_EPOCH_MASK
Definition: api_common.h:91
void vl_set_global_memory_size(u64 size)
u32 vl_api_get_msg_index(u8 *name_and_crc)
static void * svm_push_data_heap(svm_region_t *rp)
Definition: svm.h:86
vl_api_registration_t ** vl_clients
Definition: api_common.h:190
void * svm_region_find_or_create(svm_map_region_args_t *a)
Definition: svm.c:840
volatile void * user_ctx
Definition: svm_common.h:47
const char * root_path
Definition: api_common.h:247
svm_region_t * vlib_rp
Definition: api_common.h:187
int unix_shared_memory_queue_add_nolock(unix_shared_memory_queue_t *q, u8 *elem)
unsigned long u64
Definition: types.h:89
struct vl_shmem_hdr_ * shmem_hdr
Definition: api_common.h:189
#define VL_SHM_VERSION
Definition: api_common.h:89
vl_shmem_hdr_t * shmem_hdr
int unix_shared_memory_queue_add(unix_shared_memory_queue_t *q, u8 *elem, int nowait)
void vl_set_memory_root_path(const char *name)
#define SVM_FLAGS_MHEAP
Definition: svm_common.h:27
volatile int vl_pid
Definition: api_common.h:65
void vl_register_mapped_shmem_region(svm_region_t *rp)
#define foreach_vl_aring_size
Definition: api_common.h:50
vl_api_registration_t * vl_api_client_index_to_registration(u32 index)
void(** msg_print_handlers)(void *, void *)
Definition: api_common.h:174
void vl_msg_api_send_shmem_nolock(unix_shared_memory_queue_t *q, u8 *elem)
void vl_msg_api_send_shmem(unix_shared_memory_queue_t *q, u8 *elem)
#define PREDICT_FALSE(x)
Definition: clib.h:97
void svm_region_exit()
Definition: svm.c:1079
static u32 vl_msg_api_handle_get_index(u32 index)
Definition: api.h:37
void vl_set_api_pvt_heap_size(u64 size)
u64 global_size
Definition: api_common.h:212
int unix_shared_memory_queue_sub(unix_shared_memory_queue_t *q, u8 *elem, int nowait)
api_main_t api_main
Definition: api_shared.c:35
ring_alloc_t * vl_rings
Definition: api_common.h:73
static void * clib_mem_alloc_or_null(uword size)
Definition: mem.h:125
u64 global_baseva
Definition: api_common.h:209
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:340
unix_shared_memory_queue_t * vl_input_queue
Definition: api_common.h:51
unix_shared_memory_queue_t * q
Definition: api_common.h:122
#define clib_warning(format, args...)
Definition: error.h:59
void vl_msg_api_trace(api_main_t *am, vl_api_trace_t *tp, void *msg)
Definition: api_shared.c:66
void vl_set_memory_gid(int gid)
u32 garbage_collects
Definition: api_common.h:85
static void vl_msg_api_free_nolock(void *a)
void vl_msg_api_free(void *a)
const char ** msg_names
Definition: api_common.h:175
void svm_region_init_chroot(const char *root_path)
Definition: svm.c:800
void vl_msg_api_increment_missing_client_counter(void)
Definition: api_shared.c:44
vl_api_trace_t * tx_trace
Definition: api_common.h:183
#define ASSERT(truth)
unsigned int u32
Definition: types.h:88
unix_shared_memory_queue_t * unix_shared_memory_queue_init(int nels, int elsize, int consumer_pid, int signal_when_queue_non_empty)
u64 size
Definition: vhost-user.h:75
static void clib_mem_free(void *p)
Definition: mem.h:176
u64 global_pvt_heap_size
Definition: api_common.h:218
static void * clib_mem_alloc(uword size)
Definition: mem.h:109
u32 restart_reclaims
Definition: api_common.h:82
i32 vlib_signal
Definition: api_common.h:238
u64 uword
Definition: types.h:112
unsigned short u16
Definition: types.h:57
void(** msg_endian_handlers)(void *)
Definition: api_common.h:173
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
unsigned char u8
Definition: types.h:56
const char * name
Definition: svm_common.h:68
void svm_region_unmap(void *rp_arg)
Definition: svm.c:959
void svm_client_scan_this_region_nolock(svm_region_t *rp)
Definition: svm.c:1127
unix_shared_memory_queue_t * vl_api_client_index_to_input_queue(u32 index)
static void * vl_msg_api_alloc_internal(int nbytes, int pool, int may_return_null)
Definition: memory_shared.c:43
unix_shared_memory_queue_t * rp
Definition: api_common.h:38
void * vl_msg_api_alloc_as_if_client_or_null(int nbytes)
#define hash_get_mem(h, key)
Definition: hash.h:268
void(** msg_handlers)(void *)
Definition: api_common.h:170
void vl_set_api_memory_size(u64 size)
void * vl_msg_api_alloc_as_if_client(int nbytes)
static u32 vl_msg_api_handle_get_epoch(u32 index)
Definition: api.h:31
void * vl_msg_api_alloc_or_null(int nbytes)
svm_region_t ** mapped_shmem_regions
Definition: api_common.h:188
uword * msg_index_by_name_and_crc
Definition: api_common.h:244
u32 vlib_input_queue_length
Definition: api_common.h:241
pthread_mutex_t mutex
Definition: svm_common.h:37
struct _unix_shared_memory_queue unix_shared_memory_queue_t
static svm_region_t * root_rp
Definition: svm.c:46