FD.io VPP  v21.01.1
Vector Packet Processing
memory_api.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 #include <signal.h>
18 
19 #include <vlib/vlib.h>
20 #include <vlibapi/api.h>
21 #include <vlibmemory/api.h>
22 #include <vlibmemory/memory_api.h>
23 
24 #include <vlibmemory/vl_memory_msg_enum.h> /* enumerate all vlib messages */
25 
26 #define vl_typedefs /* define message structures */
28 #undef vl_typedefs
29 
30 /* instantiate all the print functions we know about */
31 #define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
32 #define vl_printfun
34 #undef vl_printfun
35 
36 /* instantiate all the endian swap functions we know about */
37 #define vl_endianfun
39 #undef vl_endianfun
40 
41 static inline void *
43 {
44  vl_print (handle, "vl_api_memclnt_create_t:\n");
45  vl_print (handle, "name: %s\n", a->name);
46  vl_print (handle, "input_queue: 0x%wx\n", a->input_queue);
47  vl_print (handle, "context: %u\n", (unsigned) a->context);
48  vl_print (handle, "ctx_quota: %ld\n", (long) a->ctx_quota);
49  return handle;
50 }
51 
52 static inline void *
54 {
55  vl_print (handle, "vl_api_memclnt_delete_t:\n");
56  vl_print (handle, "index: %u\n", (unsigned) a->index);
57  vl_print (handle, "handle: 0x%wx\n", a->handle);
58  return handle;
59 }
60 
61 volatile int **vl_api_queue_cursizes;
62 
63 static void
65 {
66  int i;
68 
69  if (PREDICT_FALSE (vec_len (vl_api_queue_cursizes) !=
70  1 + vec_len (am->vlib_private_rps)))
71  {
72  vl_shmem_hdr_t *shmem_hdr = am->shmem_hdr;
73  svm_queue_t *q;
74 
75  if (shmem_hdr == 0)
76  return;
77 
78  q = shmem_hdr->vl_input_queue;
79  if (q == 0)
80  return;
81 
82  vec_add1 (vl_api_queue_cursizes, &q->cursize);
83 
84  for (i = 0; i < vec_len (am->vlib_private_rps); i++)
85  {
86  svm_region_t *vlib_rp = am->vlib_private_rps[i];
87 
88  shmem_hdr = (void *) vlib_rp->user_ctx;
89  q = shmem_hdr->vl_input_queue;
90  vec_add1 (vl_api_queue_cursizes, &q->cursize);
91  }
92  }
93 
94  for (i = 0; i < vec_len (vl_api_queue_cursizes); i++)
95  {
96  if (*vl_api_queue_cursizes[i])
97  {
98  vm->queue_signal_pending = 1;
99  vm->api_queue_nonempty = 1;
101  /* event_type */ QUEUE_SIGNAL_EVENT,
102  /* event_data */ 0);
103  break;
104  }
105  }
106  if (vec_len (vm->pending_rpc_requests))
107  {
108  vm->queue_signal_pending = 1;
109  vm->api_queue_nonempty = 1;
111  /* event_type */ QUEUE_SIGNAL_EVENT,
112  /* event_data */ 0);
113  }
114 }
115 
116 /*
117  * vl_api_memclnt_create_internal
118  */
119 u32
121 {
122  vl_api_registration_t **regpp;
123  vl_api_registration_t *regp;
124  void *oldheap;
125  api_main_t *am = vlibapi_get_main ();
126 
127  ASSERT (vlib_get_thread_index () == 0);
128  pool_get (am->vl_clients, regpp);
129 
130 
131  oldheap = vl_msg_push_heap ();
132  *regpp = clib_mem_alloc (sizeof (vl_api_registration_t));
133 
134  regp = *regpp;
135  clib_memset (regp, 0, sizeof (*regp));
137  regp->vl_api_registration_pool_index = regpp - am->vl_clients;
138  regp->vlib_rp = am->vlib_rp;
139  regp->shmem_hdr = am->shmem_hdr;
140 
141  regp->vl_input_queue = q;
142  regp->name = format (0, "%s%c", name, 0);
143 
144  vl_msg_pop_heap (oldheap);
148 }
149 
150 /*
151  * vl_api_memclnt_create_t_handler
152  */
153 void
155 {
156  vl_api_registration_t **regpp;
157  vl_api_registration_t *regp;
159  svm_queue_t *q;
160  int rv = 0;
161  void *oldheap;
162  api_main_t *am = vlibapi_get_main ();
163  u8 *msg_table;
164 
165  /*
166  * This is tortured. Maintain a vlib-address-space private
167  * pool of client registrations. We use the shared-memory virtual
168  * address of client structure as a handle, to allow direct
169  * manipulation of context quota vbls from the client library.
170  *
171  * This scheme causes trouble w/ API message trace replay, since
172  * some random VA from clib_mem_alloc() certainly won't
173  * occur in the Linux sim. The (very) few places
174  * that care need to use the pool index.
175  *
176  * Putting the registration object(s) into a pool in shared memory and
177  * using the pool index as a handle seems like a great idea.
178  * Unfortunately, each and every reference to that pool would need
179  * to be protected by a mutex:
180  *
181  * Client VLIB
182  * ------ ----
183  * convert pool index to
184  * pointer.
185  * <deschedule>
186  * expand pool
187  * <deschedule>
188  * kaboom!
189  */
190 
191  pool_get (am->vl_clients, regpp);
192 
193  oldheap = vl_msg_push_heap ();
194  *regpp = clib_mem_alloc (sizeof (vl_api_registration_t));
195 
196  regp = *regpp;
197  clib_memset (regp, 0, sizeof (*regp));
199  regp->vl_api_registration_pool_index = regpp - am->vl_clients;
200  regp->vlib_rp = am->vlib_rp;
201  regp->shmem_hdr = am->shmem_hdr;
203 
204  q = regp->vl_input_queue = (svm_queue_t *) (uword) mp->input_queue;
206 
207  regp->name = format (0, "%s", mp->name);
208  vec_add1 (regp->name, 0);
209 
213 
214  if (am->vlib_rp != am->vlib_primary_rp)
215  msg_table = vl_api_serialize_message_table (am, 0);
216  else
217  msg_table = am->serialized_message_table_in_shmem;
218 
219  vl_msg_pop_heap (oldheap);
220 
221  rp = vl_msg_api_alloc (sizeof (*rp));
222  rp->_vl_msg_id = ntohs (VL_API_MEMCLNT_CREATE_REPLY);
223  rp->handle = (uword) regp;
227  rp->context = mp->context;
228  rp->response = ntohl (rv);
229  rp->message_table = pointer_to_uword (msg_table);
230 
231  vl_msg_api_send_shmem (q, (u8 *) & rp);
232 }
233 
234 void
236 {
237  clib_error_t *error = 0;
238  _vl_msg_api_function_list_elt_t *i;
239 
241  while (i)
242  {
243  error = i->f (client_index);
244  if (error)
245  clib_error_report (error);
246  i = i->next_init_function;
247  }
248 }
249 
250 /*
251  * vl_api_memclnt_delete_t_handler
252  */
253 void
255 {
256  vl_api_registration_t **regpp;
257  vl_api_registration_t *regp;
259  void *oldheap;
260  api_main_t *am = vlibapi_get_main ();
261  u32 handle, client_index, epoch;
262 
263  handle = mp->index;
264 
266 
267  epoch = vl_msg_api_handle_get_epoch (handle);
268  client_index = vl_msg_api_handle_get_index (handle);
269 
270  if (epoch != (am->shmem_hdr->application_restarts & VL_API_EPOCH_MASK))
271  {
273  ("Stale clnt delete index %d old epoch %d cur epoch %d",
274  client_index, epoch,
276  return;
277  }
278 
279  regpp = pool_elt_at_index (am->vl_clients, client_index);
280 
281  if (!pool_is_free (am->vl_clients, regpp))
282  {
283  int i;
284  regp = *regpp;
285  int private_registration = 0;
286 
287  /* Send reply unless client asked us to do the cleanup */
288  if (!mp->do_cleanup)
289  {
290  /*
291  * Note: the API message handling path will set am->vlib_rp
292  * as appropriate for pairwise / private memory segments
293  */
294  rp = vl_msg_api_alloc (sizeof (*rp));
295  rp->_vl_msg_id = ntohs (VL_API_MEMCLNT_DELETE_REPLY);
296  rp->handle = mp->handle;
297  rp->response = 1;
298 
299  vl_msg_api_send_shmem (regp->vl_input_queue, (u8 *) & rp);
300  if (client_index != regp->vl_api_registration_pool_index)
301  {
302  clib_warning ("mismatch client_index %d pool_index %d",
303  client_index,
305  vl_msg_api_free (rp);
306  return;
307  }
308  }
309 
310  /* No dangling references, please */
311  *regpp = 0;
312 
313  /* For horizontal scaling, add a hash table... */
314  for (i = 0; i < vec_len (am->vlib_private_rps); i++)
315  {
316  /* Is this a pairwise / private API segment? */
317  if (am->vlib_private_rps[i] == am->vlib_rp)
318  {
319  /* Note: account for the memfd header page */
320  uword virtual_base = am->vlib_rp->virtual_base - MMAP_PAGESIZE;
321  uword virtual_size = am->vlib_rp->virtual_size + MMAP_PAGESIZE;
322 
323  /*
324  * Kill the registration pool element before we make
325  * the index vanish forever
326  */
329 
330  vec_delete (am->vlib_private_rps, 1, i);
331  /* Kill it, accounting for the memfd header page */
332  if (munmap ((void *) virtual_base, virtual_size) < 0)
333  clib_unix_warning ("munmap");
334  /* Reset the queue-length-address cache */
335  vec_reset_length (vl_api_queue_cursizes);
336  private_registration = 1;
337  break;
338  }
339  }
340 
341  if (private_registration == 0)
342  {
345  oldheap = vl_msg_push_heap ();
346  if (mp->do_cleanup)
348  vec_free (regp->name);
349  /* Poison the old registration */
350  clib_memset (regp, 0xF1, sizeof (*regp));
351  clib_mem_free (regp);
352  vl_msg_pop_heap (oldheap);
353  /*
354  * These messages must be freed manually, since they're set up
355  * as "bounce" messages. In the private_registration == 1 case,
356  * we kill the shared-memory segment which contains the message
357  * with munmap.
358  */
359  vl_msg_api_free (mp);
360  }
361  }
362  else
363  {
364  clib_warning ("unknown client ID %d", mp->index);
365  }
366 }
367 
368 /**
369  * client answered a ping, stave off the grim reaper...
370  */
371 void
373  (vl_api_memclnt_keepalive_reply_t * mp)
374 {
375  vl_api_registration_t *regp;
377 
378  regp = vl_api_client_index_to_registration (mp->context);
379  if (regp)
380  {
381  regp->last_heard = vlib_time_now (vm);
382  regp->unanswered_pings = 0;
383  }
384  else
385  clib_warning ("BUG: anonymous memclnt_keepalive_reply");
386 }
387 
388 /**
389  * We can send ourselves these messages if someone uses the
390  * builtin binary api test tool...
391  */
392 static void
394 {
395  vl_api_memclnt_keepalive_reply_t *rmp;
396  api_main_t *am;
397  vl_shmem_hdr_t *shmem_hdr;
398 
399  am = vlibapi_get_main ();
400  shmem_hdr = am->shmem_hdr;
401 
402  rmp = vl_msg_api_alloc_as_if_client (sizeof (*rmp));
403  clib_memset (rmp, 0, sizeof (*rmp));
404  rmp->_vl_msg_id = ntohs (VL_API_MEMCLNT_KEEPALIVE_REPLY);
405  rmp->context = mp->context;
406  vl_msg_api_send_shmem (shmem_hdr->vl_input_queue, (u8 *) & rmp);
407 }
408 
409 /*
410  * To avoid filling the API trace buffer with boring messages,
411  * don't trace memclnt_keepalive[_reply] msgs
412  */
413 
414 #define foreach_vlib_api_msg \
415 _(MEMCLNT_CREATE, memclnt_create, 1) \
416 _(MEMCLNT_DELETE, memclnt_delete, 1) \
417 _(MEMCLNT_KEEPALIVE, memclnt_keepalive, 0) \
418 _(MEMCLNT_KEEPALIVE_REPLY, memclnt_keepalive_reply, 0)
419 
420 /*
421  * memory_api_init
422  */
423 int
424 vl_mem_api_init (const char *region_name)
425 {
426  int rv;
427  api_main_t *am = vlibapi_get_main ();
429  vl_msg_api_msg_config_t *c = &cfg;
430  vl_shmem_hdr_t *shm;
432 
433  clib_memset (c, 0, sizeof (*c));
434 
435  if ((rv = vl_map_shmem (region_name, 1 /* is_vlib */ )) < 0)
436  return rv;
437 
438 #define _(N,n,t) do { \
439  c->id = VL_API_##N; \
440  c->name = #n; \
441  c->handler = vl_api_##n##_t_handler; \
442  c->cleanup = vl_noop_handler; \
443  c->endian = vl_api_##n##_t_endian; \
444  c->print = vl_api_##n##_t_print; \
445  c->size = sizeof(vl_api_##n##_t); \
446  c->traced = t; /* trace, so these msgs print */ \
447  c->replay = 0; /* don't replay client create/delete msgs */ \
448  c->message_bounce = 0; /* don't bounce this message */ \
449  vl_msg_api_config(c);} while (0);
450 
452 #undef _
453 
454  /*
455  * special-case freeing of memclnt_delete messages, so we can
456  * simply munmap pairwise / private API segments...
457  */
458  am->message_bounce[VL_API_MEMCLNT_DELETE] = 1;
459  am->is_mp_safe[VL_API_MEMCLNT_KEEPALIVE_REPLY] = 1;
460  am->is_mp_safe[VL_API_MEMCLNT_KEEPALIVE] = 1;
461 
463 
464  shm = am->shmem_hdr;
465  ASSERT (shm && shm->vl_input_queue);
466 
467  /* Make a note so we can always find the primary region easily */
468  am->vlib_primary_rp = am->vlib_rp;
469 
470  return 0;
471 }
472 
473 clib_error_t *
475 {
476  api_main_t *am = vlibapi_get_main ();
477  int rv;
478 
479  if ((rv = vl_mem_api_init (am->region_name)) < 0)
480  {
481  return clib_error_return (0, "vl_mem_api_init (%s) failed",
482  am->region_name);
483  }
484  return 0;
485 }
486 
487 static void
489 {
491  svm_queue_t *q;
492  api_main_t *am = vlibapi_get_main ();
493 
494  q = regp->vl_input_queue;
495 
496  /*
497  * If the queue head is moving, assume that the client is processing
498  * messages and skip the ping. This heuristic may fail if the queue
499  * is in the same position as last time, net of wrapping; in which
500  * case, the client will receive a keepalive.
501  */
502  if (regp->last_queue_head != q->head)
503  {
504  regp->last_heard = now;
505  regp->unanswered_pings = 0;
506  regp->last_queue_head = q->head;
507  return;
508  }
509 
510  /*
511  * push/pop shared memory segment, so this routine
512  * will work with "normal" as well as "private segment"
513  * memory clients..
514  */
515 
516  mp = vl_mem_api_alloc_as_if_client_w_reg (regp, sizeof (*mp));
517  clib_memset (mp, 0, sizeof (*mp));
518  mp->_vl_msg_id = clib_host_to_net_u16 (VL_API_MEMCLNT_KEEPALIVE);
519  mp->context = mp->client_index =
523 
524  regp->unanswered_pings++;
525 
526  /* Failure-to-send due to a stuffed queue is absolutely expected */
527  if (svm_queue_add (q, (u8 *) & mp, 1 /* nowait */ ))
528  vl_msg_api_free_w_region (regp->vlib_rp, mp);
529 }
530 
531 static void
533  vl_api_registration_t ** regpp,
534  u32 ** dead_indices,
535  u32 ** confused_indices)
536 {
537  vl_api_registration_t *regp = *regpp;
538  if (regp)
539  {
540  /* If we haven't heard from this client recently... */
541  if (regp->last_heard < (now - 10.0))
542  {
543  if (regp->unanswered_pings == 2)
544  {
545  svm_queue_t *q;
546  q = regp->vl_input_queue;
547  if (kill (q->consumer_pid, 0) >= 0)
548  {
549  clib_warning ("REAPER: lazy binary API client '%s'",
550  regp->name);
551  regp->unanswered_pings = 0;
552  regp->last_heard = now;
553  }
554  else
555  {
556  clib_warning ("REAPER: binary API client '%s' died",
557  regp->name);
558  vec_add1 (*dead_indices, regpp - am->vl_clients);
559  }
560  }
561  else
562  send_memclnt_keepalive (regp, now);
563  }
564  else
565  regp->unanswered_pings = 0;
566  }
567  else
568  {
569  clib_warning ("NULL client registration index %d",
570  regpp - am->vl_clients);
571  vec_add1 (*confused_indices, regpp - am->vl_clients);
572  }
573 }
574 
575 void
577 {
578  vl_api_registration_t **regpp;
579  static u32 *dead_indices;
580  static u32 *confused_indices;
581 
582  vec_reset_length (dead_indices);
583  vec_reset_length (confused_indices);
584 
585  /* *INDENT-OFF* */
586  pool_foreach (regpp, am->vl_clients) {
587  vl_mem_send_client_keepalive_w_reg (am, now, regpp, &dead_indices,
588  &confused_indices);
589  }
590  /* *INDENT-ON* */
591 
592  /* This should "never happen," but if it does, fix it... */
593  if (PREDICT_FALSE (vec_len (confused_indices) > 0))
594  {
595  int i;
596  for (i = 0; i < vec_len (confused_indices); i++)
597  {
598  pool_put_index (am->vl_clients, confused_indices[i]);
599  }
600  }
601 
602  if (PREDICT_FALSE (vec_len (dead_indices) > 0))
603  {
604  int i;
605  void *oldheap;
606 
607  /* Allow the application to clean up its registrations */
608  for (i = 0; i < vec_len (dead_indices); i++)
609  {
610  regpp = pool_elt_at_index (am->vl_clients, dead_indices[i]);
611  if (regpp)
612  {
613  u32 handle;
614 
616  (dead_indices[i], shm->application_restarts);
618  }
619  }
620 
621  oldheap = vl_msg_push_heap ();
622 
623  for (i = 0; i < vec_len (dead_indices); i++)
624  {
625  regpp = pool_elt_at_index (am->vl_clients, dead_indices[i]);
626  if (regpp)
627  {
628  /* Is this a pairwise SVM segment? */
629  if ((*regpp)->vlib_rp != am->vlib_rp)
630  {
631  int i;
632  svm_region_t *dead_rp = (*regpp)->vlib_rp;
633  /* Note: account for the memfd header page */
634  uword virtual_base = dead_rp->virtual_base - MMAP_PAGESIZE;
635  uword virtual_size = dead_rp->virtual_size + MMAP_PAGESIZE;
636 
637  /* For horizontal scaling, add a hash table... */
638  for (i = 0; i < vec_len (am->vlib_private_rps); i++)
639  if (am->vlib_private_rps[i] == dead_rp)
640  {
641  vec_delete (am->vlib_private_rps, 1, i);
642  goto found;
643  }
644  svm_pop_heap (oldheap);
645  clib_warning ("private rp %llx AWOL", dead_rp);
646  oldheap = svm_push_data_heap (am->vlib_rp);
647 
648  found:
649  /* Kill it, accounting for the memfd header page */
650  svm_pop_heap (oldheap);
651  if (munmap ((void *) virtual_base, virtual_size) < 0)
652  clib_unix_warning ("munmap");
653  /* Reset the queue-length-address cache */
654  vec_reset_length (vl_api_queue_cursizes);
655  oldheap = svm_push_data_heap (am->vlib_rp);
656  }
657  else
658  {
659  /* Poison the old registration */
660  clib_memset (*regpp, 0xF3, sizeof (**regpp));
661  clib_mem_free (*regpp);
662  }
663  /* no dangling references, please */
664  *regpp = 0;
665  }
666  else
667  {
668  svm_pop_heap (oldheap);
669  clib_warning ("Duplicate free, client index %d",
670  regpp - am->vl_clients);
671  oldheap = svm_push_data_heap (am->vlib_rp);
672  }
673  }
674 
676 
677  vl_msg_pop_heap (oldheap);
678  for (i = 0; i < vec_len (dead_indices); i++)
679  pool_put_index (am->vl_clients, dead_indices[i]);
680  }
681 }
682 
683 static inline int
686  u8 is_private)
687 {
688  svm_queue_t *q;
689  uword mp;
690 
691  q = ((vl_shmem_hdr_t *) (void *) vlib_rp->user_ctx)->vl_input_queue;
692 
693  if (!svm_queue_sub2 (q, (u8 *) & mp))
694  {
695  VL_MSG_API_UNPOISON ((void *) mp);
696  vl_msg_api_handler_with_vm_node (am, vlib_rp, (void *) mp, vm, node,
697  is_private);
698  return 0;
699  }
700  return -1;
701 }
702 
703 int
705 {
706  api_main_t *am = vlibapi_get_main ();
707  return void_mem_api_handle_msg_i (am, am->vlib_rp, vm, node,
708  0 /* is_private */ );
709 }
710 
711 int
713 {
714  api_main_t *am = vlibapi_get_main ();
715  int i;
716  uword *tmp, mp;
717 
718  /*
719  * Swap pending and processing vectors, then process the RPCs
720  * Avoid deadlock conditions by construction.
721  */
723  tmp = vm->processing_rpc_requests;
724  vec_reset_length (tmp);
726  vm->pending_rpc_requests = tmp;
728 
729  /*
730  * RPCs are used to reflect function calls to thread 0
731  * when the underlying code is not thread-safe.
732  *
733  * Grabbing the thread barrier across a set of RPCs
734  * greatly increases efficiency, and avoids
735  * running afoul of the barrier sync holddown timer.
736  * The barrier sync code supports recursive locking.
737  *
738  * We really need to rewrite RPC-based code...
739  */
741  {
743  for (i = 0; i < vec_len (vm->processing_rpc_requests); i++)
744  {
745  mp = vm->processing_rpc_requests[i];
746  vl_msg_api_handler_with_vm_node (am, am->vlib_rp, (void *) mp, vm,
747  node, 0 /* is_private */ );
748  }
750  }
751 
752  return 0;
753 }
754 
755 int
757  u32 reg_index)
758 {
759  api_main_t *am = vlibapi_get_main ();
760  return void_mem_api_handle_msg_i (am, am->vlib_private_rps[reg_index], vm,
761  node, 1 /* is_private */ );
762 }
763 
766 {
767  vl_api_registration_t **regpp;
768  vl_api_registration_t *regp;
769  api_main_t *am = vlibapi_get_main ();
770  vl_shmem_hdr_t *shmem_hdr;
771  u32 index;
772 
773  index = vl_msg_api_handle_get_index (handle);
774  regpp = am->vl_clients + index;
775 
776  if (pool_is_free (am->vl_clients, regpp))
777  {
779  return 0;
780  }
781  regp = *regpp;
782 
783  shmem_hdr = (vl_shmem_hdr_t *) regp->shmem_hdr;
784  if (!vl_msg_api_handle_is_valid (handle, shmem_hdr->application_restarts))
785  {
787  return 0;
788  }
789 
790  return (regp);
791 }
792 
793 svm_queue_t *
795 {
796  vl_api_registration_t *regp;
797  api_main_t *am = vlibapi_get_main ();
798 
799  /* Special case: vlib trying to send itself a message */
800  if (index == (u32) ~ 0)
801  return (am->shmem_hdr->vl_input_queue);
802 
804  if (!regp)
805  return 0;
806  return (regp->vl_input_queue);
807 }
808 
809 static clib_error_t *
811 {
812  atexit (vl_unmap_shmem);
813  return 0;
814 }
815 
817 
818 u8 *
819 format_api_message_rings (u8 * s, va_list * args)
820 {
821  api_main_t *am = va_arg (*args, api_main_t *);
822  vl_shmem_hdr_t *shmem_hdr = va_arg (*args, vl_shmem_hdr_t *);
823  int main_segment = va_arg (*args, int);
824  ring_alloc_t *ap;
825  int i;
826 
827  if (shmem_hdr == 0)
828  return format (s, "%8s %8s %8s %8s %8s\n",
829  "Owner", "Size", "Nitems", "Hits", "Misses");
830 
831  ap = shmem_hdr->vl_rings;
832 
833  for (i = 0; i < vec_len (shmem_hdr->vl_rings); i++)
834  {
835  s = format (s, "%8s %8d %8d %8d %8d\n",
836  "vlib", ap->size, ap->nitems, ap->hits, ap->misses);
837  ap++;
838  }
839 
840  ap = shmem_hdr->client_rings;
841 
842  for (i = 0; i < vec_len (shmem_hdr->client_rings); i++)
843  {
844  s = format (s, "%8s %8d %8d %8d %8d\n",
845  "clnt", ap->size, ap->nitems, ap->hits, ap->misses);
846  ap++;
847  }
848 
849  if (main_segment)
850  {
851  s = format (s, "%d ring miss fallback allocations\n", am->ring_misses);
852  s = format
853  (s,
854  "%d application restarts, %d reclaimed msgs, %d garbage collects\n",
855  shmem_hdr->application_restarts, shmem_hdr->restart_reclaims,
856  shmem_hdr->garbage_collects);
857  }
858  return s;
859 }
860 
861 static clib_error_t *
863  unformat_input_t * input, vlib_cli_command_t * cli_cmd)
864 {
865  int i;
866  vl_shmem_hdr_t *shmem_hdr;
867  api_main_t *am = vlibapi_get_main ();
868 
869  /* First, dump the primary region rings.. */
870 
871  if (am->vlib_primary_rp == 0 || am->vlib_primary_rp->user_ctx == 0)
872  {
873  vlib_cli_output (vm, "Shared memory segment not initialized...\n");
874  return 0;
875  }
876 
877  shmem_hdr = (void *) am->vlib_primary_rp->user_ctx;
878 
879  vlib_cli_output (vm, "Main API segment rings:");
880 
882  0 /* print header */ , 0 /* notused */ );
883 
885  shmem_hdr, 1 /* main segment */ );
886 
887  for (i = 0; i < vec_len (am->vlib_private_rps); i++)
888  {
889  svm_region_t *vlib_rp = am->vlib_private_rps[i];
890  shmem_hdr = (void *) vlib_rp->user_ctx;
891  vl_api_registration_t **regpp;
892  vl_api_registration_t *regp = 0;
893 
894  /* For horizontal scaling, add a hash table... */
895  /* *INDENT-OFF* */
896  pool_foreach (regpp, am->vl_clients)
897  {
898  regp = *regpp;
899  if (regp && regp->vlib_rp == vlib_rp)
900  {
901  vlib_cli_output (vm, "%s segment rings:", regp->name);
902  goto found;
903  }
904  }
905  vlib_cli_output (vm, "regp %llx not found?", regp);
906  continue;
907  /* *INDENT-ON* */
908  found:
910  0 /* print header */ , 0 /* notused */ );
912  shmem_hdr, 0 /* main segment */ );
913  }
914 
915  return 0;
916 }
917 
918 /*?
919  * Display binary api message allocation ring statistics
920 ?*/
921 /* *INDENT-OFF* */
922 VLIB_CLI_COMMAND (cli_show_api_ring_command, static) =
923 {
924  .path = "show api ring-stats",
925  .short_help = "Message ring statistics",
926  .function = vl_api_ring_command,
927 };
928 /* *INDENT-ON* */
929 
930 clib_error_t *
932 {
933  api_main_t *am = vlibapi_get_main ();
934  svm_map_region_args_t _a, *a = &_a;
935  u8 *remove_path1, *remove_path2;
936  void vlibsocket_reference (void);
937 
939 
940  /*
941  * By popular request / to avoid support fires, remove any old api segment
942  * files Right Here.
943  */
944  if (am->root_path == 0)
945  {
946  remove_path1 = format (0, "/dev/shm/global_vm%c", 0);
947  remove_path2 = format (0, "/dev/shm/vpe-api%c", 0);
948  }
949  else
950  {
951  remove_path1 = format (0, "/dev/shm/%s-global_vm%c", am->root_path, 0);
952  remove_path2 = format (0, "/dev/shm/%s-vpe-api%c", am->root_path, 0);
953  }
954 
955  (void) unlink ((char *) remove_path1);
956  (void) unlink ((char *) remove_path2);
957 
958  vec_free (remove_path1);
959  vec_free (remove_path2);
960 
961  clib_memset (a, 0, sizeof (*a));
962  a->root_path = am->root_path;
964  a->baseva = (am->global_baseva != 0) ?
966  a->size = (am->global_size != 0) ? am->global_size : SVM_GLOBAL_REGION_SIZE;
967  a->flags = SVM_FLAGS_NODATA;
968  a->uid = am->api_uid;
969  a->gid = am->api_gid;
970  a->pvt_heap_size =
971  (am->global_pvt_heap_size !=
973 
975 
976  return 0;
977 }
978 
979 void
981 {
982  api_main_t *am = vlibapi_get_main ();
983  am->region_name = name;
984 }
985 
986 /*
987  * fd.io coding-style-patch-verification: ON
988  *
989  * Local Variables:
990  * eval: (c-set-style "gnu")
991  * End:
992  */
int vl_mem_api_handle_msg_private(vlib_main_t *vm, vlib_node_runtime_t *node, u32 reg_index)
Definition: memory_api.c:756
int vl_mem_api_handle_msg_main(vlib_main_t *vm, vlib_node_runtime_t *node)
Definition: memory_api.c:704
void vlibsocket_reference()
Definition: socket_api.c:830
uword * pending_rpc_requests
Definition: main.h:314
static u32 vl_msg_api_handle_get_index(u32 index)
Definition: memory_api.h:48
void vl_api_memclnt_create_t_handler(vl_api_memclnt_create_t *mp)
Definition: memory_api.c:154
u8 * name
Client name.
Definition: api_common.h:54
#define SVM_GLOBAL_REGION_NAME
Definition: svm_common.h:101
const char * root_path
Definition: svm_common.h:67
static void * vl_api_memclnt_create_t_print(vl_api_memclnt_create_t *a, void *handle)
Definition: memory_api.c:42
static void vl_api_memclnt_keepalive_t_handler(vl_api_memclnt_keepalive_t *mp)
We can send ourselves these messages if someone uses the builtin binary api test tool...
Definition: memory_api.c:393
static void svm_pop_heap(void *oldheap)
Definition: svm.h:94
clib_spinlock_t pending_rpc_lock
Definition: main.h:316
#define vl_print(handle,...)
Definition: memory_api.c:31
int svm_queue_add(svm_queue_t *q, u8 *elem, int nowait)
Definition: queue.c:260
u32 vl_api_memclnt_create_internal(char *name, svm_queue_t *q)
Definition: memory_api.c:120
a
Definition: bitmap.h:544
#define SVM_FLAGS_NODATA
Definition: svm_common.h:29
static void send_memclnt_keepalive(vl_api_registration_t *regp, f64 now)
Definition: memory_api.c:488
int vl_map_shmem(const char *region_name, int is_vlib)
#define ntohs(x)
Definition: af_xdp.bpf.c:29
uword * processing_rpc_requests
Definition: main.h:315
volatile int ** vl_api_queue_cursizes
Definition: memory_api.c:61
#define pool_foreach(VAR, POOL)
Iterate through pool.
Definition: pool.h:527
u32 application_restarts
Definition: memory_shared.h:95
#define PREDICT_TRUE(x)
Definition: clib.h:122
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:334
void vl_unmap_shmem(void)
int svm_queue_sub2(svm_queue_t *q, u8 *elem)
Definition: queue.c:434
u8 * message_bounce
Don&#39;t automatically free message buffer vetor.
Definition: api_common.h:247
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:129
Message configuration definition.
Definition: api_common.h:121
svm_region_t * vlib_primary_rp
Primary api segment descriptor.
Definition: api_common.h:286
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:592
uword virtual_base
Definition: svm_common.h:42
#define SVM_PVT_MHEAP_SIZE
Definition: svm_common.h:32
int api_uid
uid for the api shared memory region
Definition: api_common.h:311
ring_alloc_t * client_rings
Definition: memory_shared.h:92
vlib_main_t * vm
Definition: in2out_ed.c:1580
#define pool_is_free(P, E)
Use free bitmap to query whether given element is free.
Definition: pool.h:290
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:251
void * vl_msg_api_alloc(int nbytes)
int api_gid
gid for the api shared memory region
Definition: api_common.h:314
unsigned char u8
Definition: types.h:56
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
double f64
Definition: types.h:142
_vl_msg_api_function_list_elt_t * reaper_function_registrations
List of API client reaper functions.
Definition: api_common.h:368
u8 * format_api_message_rings(u8 *s, va_list *args)
Definition: memory_api.c:819
u32 clib_file_index
Socket only: file index.
Definition: api_common.h:67
void vl_msg_api_barrier_sync(void)
Definition: api_shared.c:427
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:173
u32 ring_misses
Number of times that the ring allocator failed.
Definition: api_common.h:259
static void * svm_push_data_heap(svm_region_t *rp)
Definition: svm.h:86
vl_api_registration_t ** vl_clients
vlib/vpp only: vector of client registrations
Definition: api_common.h:296
vl_api_registration_t * vl_mem_api_client_index_to_registration(u32 handle)
Definition: memory_api.c:765
volatile void * user_ctx
Definition: svm_common.h:47
static clib_error_t * vl_api_ring_command(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cli_cmd)
Definition: memory_api.c:862
description fragment has unexpected format
Definition: map.api:433
static u32 vl_msg_api_handle_from_index_and_epoch(u32 index, u32 epoch)
Definition: memory_api.h:54
void vl_api_memclnt_keepalive_reply_t_handler(vl_api_memclnt_keepalive_reply_t *mp)
client answered a ping, stave off the grim reaper...
Definition: memory_api.c:373
void vl_mem_api_dead_client_scan(api_main_t *am, vl_shmem_hdr_t *shm, f64 now)
Definition: memory_api.c:576
vlib_node_registration_t vl_api_clnt_node
(constructor) VLIB_REGISTER_NODE (vl_api_clnt_node)
Definition: vlib_api.c:433
const char * root_path
Chroot path to the shared memory API files.
Definition: api_common.h:359
#define clib_error_return(e, args...)
Definition: error.h:99
svm_region_t * vlib_rp
Current binary api segment descriptor.
Definition: api_common.h:283
struct vl_shmem_hdr_ * shmem_hdr
Binary API shared-memory segment header pointer.
Definition: api_common.h:293
unsigned int u32
Definition: types.h:88
void vl_msg_api_send_shmem(svm_queue_t *q, u8 *elem)
static void vlib_set_queue_signal_callback(vlib_main_t *vm, void(*fp)(vlib_main_t *))
Definition: main.h:463
vl_registration_type_t registration_type
type
Definition: api_common.h:49
char * name
Definition: main.h:170
Definition: cJSON.c:84
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:546
#define foreach_vlib_api_msg
Definition: memory_api.c:414
static void vlib_process_signal_event(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
Definition: node_funcs.h:1015
void vl_msg_pop_heap(void *oldheap)
Definition: cli.c:732
svm_queue_t * vl_input_queue
shared memory only: pointer to client input queue
Definition: api_common.h:62
struct _unformat_input_t unformat_input_t
svm_region_t ** vlib_private_rps
Vector of all mapped shared-VM segments.
Definition: api_common.h:289
#define PREDICT_FALSE(x)
Definition: clib.h:121
uword virtual_size
Definition: svm_common.h:43
void svm_region_init_args(svm_map_region_args_t *a)
Definition: svm.c:871
#define SVM_GLOBAL_REGION_SIZE
Definition: svm_common.h:100
clib_error_t * vlibmemory_init(vlib_main_t *vm)
Definition: memory_api.c:931
volatile u32 queue_signal_pending
Definition: main.h:263
int vl_mem_api_handle_rpc(vlib_main_t *vm, vlib_node_runtime_t *node)
Definition: memory_api.c:712
clib_error_t * map_api_segment_init(vlib_main_t *vm)
Definition: memory_api.c:474
API main structure, used by both vpp and binary API clients.
Definition: api_common.h:227
u64 global_size
size of the global VM region
Definition: api_common.h:320
An API client registration, only in vpp/vlib.
Definition: api_common.h:47
Shared memory connection.
Definition: api_common.h:39
void vl_msg_api_free_w_region(svm_region_t *vlib_rp, void *a)
ring_alloc_t * vl_rings
Definition: memory_shared.h:89
svmdb_client_t * c
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:219
svm_queue_t * vl_api_client_index_to_input_queue(u32 index)
Definition: memory_api.c:794
u64 global_baseva
base virtual address for global VM region
Definition: api_common.h:317
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:317
static clib_error_t * setup_memclnt_exit(vlib_main_t *vm)
Definition: memory_api.c:810
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:380
void vl_api_call_reaper_functions(u32 client_index)
Definition: memory_api.c:235
#define clib_warning(format, args...)
Definition: error.h:59
static void * vl_api_memclnt_delete_t_print(vl_api_memclnt_delete_t *a, void *handle)
Definition: memory_api.c:53
static vl_api_registration_t * vl_api_client_index_to_registration(u32 index)
Definition: api.h:79
svm_queue_t * vl_input_queue
Definition: memory_shared.h:84
string name[64]
Definition: ip.api:44
static CLIB_NOSANITIZE_ADDR void VL_MSG_API_SVM_QUEUE_UNPOISON(const svm_queue_t *q)
Definition: api_common.h:155
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1580
void svm_queue_free(svm_queue_t *q)
Definition: queue.c:91
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:158
static u32 vl_msg_api_handle_get_epoch(u32 index)
Definition: memory_api.h:42
#define pool_put_index(p, i)
Free pool element with given index.
Definition: pool.h:330
#define ASSERT(truth)
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:696
int vl_mem_api_init(const char *region_name)
Definition: memory_api.c:424
#define vec_delete(V, N, M)
Delete N elements starting at element M.
Definition: vec.h:854
static int void_mem_api_handle_msg_i(api_main_t *am, svm_region_t *vlib_rp, vlib_main_t *vm, vlib_node_runtime_t *node, u8 is_private)
Definition: memory_api.c:684
static u8 vl_msg_api_handle_is_valid(u32 handle, u32 restarts)
Definition: memory_api.h:64
static void vl_mem_send_client_keepalive_w_reg(api_main_t *am, f64 now, vl_api_registration_t **regpp, u32 **dead_indices, u32 **confused_indices)
Definition: memory_api.c:532
static void clib_mem_free(void *p)
Definition: mem.h:311
void vl_set_memory_region_name(const char *name)
Definition: memory_api.c:980
u64 svm_get_global_region_base_va()
Definition: svm.c:61
u8 * serialized_message_table_in_shmem
vlib/vpp only: serialized (message, name, crc) table
Definition: api_common.h:299
u8 * vl_api_serialize_message_table(api_main_t *am, u8 *vector)
Definition: api_shared.c:204
#define clib_error_report(e)
Definition: error.h:113
u64 global_pvt_heap_size
size of the global VM private mheap
Definition: api_common.h:326
void vl_api_memclnt_delete_t_handler(vl_api_memclnt_delete_t *mp)
Definition: memory_api.c:254
static void * clib_mem_alloc(uword size)
Definition: mem.h:253
static uword pointer_to_uword(const void *p)
Definition: types.h:131
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
void vl_msg_api_free(void *)
#define MMAP_PAGESIZE
Definition: ssvm.h:42
#define VL_API_EPOCH_MASK
void vl_msg_api_handler_with_vm_node(api_main_t *am, svm_region_t *vlib_rp, void *the_msg, vlib_main_t *vm, vlib_node_runtime_t *node, u8 is_private)
Definition: api_shared.c:557
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
const char * name
Definition: svm_common.h:68
static void memclnt_queue_callback(vlib_main_t *vm)
Definition: memory_api.c:64
u64 uword
Definition: types.h:112
#define clib_unix_warning(format, args...)
Definition: error.h:68
u32 vl_api_registration_pool_index
Index in VLIB&#39;s brain (not shared memory).
Definition: api_common.h:52
u32 index
Definition: flow_types.api:221
void svm_client_scan_this_region_nolock(svm_region_t *rp)
Definition: svm.c:1219
volatile u32 api_queue_nonempty
Definition: main.h:264
struct _svm_queue svm_queue_t
void * vl_mem_api_alloc_as_if_client_w_reg(vl_api_registration_t *reg, int nbytes)
static api_main_t * vlibapi_get_main(void)
Definition: api_common.h:389
u8 * is_mp_safe
Message is mp safe vector.
Definition: api_common.h:250
void vl_msg_api_increment_missing_client_counter(void)
Definition: api_shared.c:55
void * vl_msg_push_heap(void)
Definition: cli.c:725
void * vl_msg_api_alloc_as_if_client(int nbytes)
const char * region_name
Shared VM binary API region name.
Definition: api_common.h:356
void vl_msg_api_barrier_release(void)
Definition: api_shared.c:432
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:106
svm_region_t * vlib_rp
Definition: api_common.h:63
static CLIB_NOSANITIZE_ADDR void VL_MSG_API_UNPOISON(const void *a)
Definition: api_common.h:148