FD.io VPP  v18.10-34-gcce845e
Vector Packet Processing
memif.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2017 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 
19 #define _GNU_SOURCE
20 #include <stdint.h>
21 #include <net/if.h>
22 #include <sys/types.h>
23 #include <fcntl.h>
24 #include <sys/ioctl.h>
25 #include <sys/socket.h>
26 #include <sys/un.h>
27 #include <sys/uio.h>
28 #include <sys/mman.h>
29 #include <sys/prctl.h>
30 #include <sys/eventfd.h>
31 #include <inttypes.h>
32 #include <limits.h>
33 
34 #include <vlib/vlib.h>
35 #include <vlib/unix/unix.h>
36 #include <vppinfra/linux/syscall.h>
37 #include <vnet/plugin/plugin.h>
38 #include <vnet/ethernet/ethernet.h>
39 #include <vpp/app/version.h>
40 #include <memif/memif.h>
41 #include <memif/private.h>
42 
44 
45 static u32
47 {
48  /* nothing for now */
49  return 0;
50 }
51 
52 static void
54 {
55  if (mq->int_clib_file_index != ~0)
56  {
58  mq->int_clib_file_index = ~0;
59  mq->int_fd = -1;
60  }
61  else if (mq->int_fd > -1)
62  {
63  close (mq->int_fd);
64  mq->int_fd = -1;
65  }
66 }
67 
68 void
70 {
71  memif_main_t *mm = &memif_main;
72  vnet_main_t *vnm = vnet_get_main ();
73  memif_region_t *mr;
74  memif_queue_t *mq;
75  int i;
76 
77  if (mif == 0)
78  return;
79 
80  memif_log_debug (mif, "disconnect %u (%v)", mif->dev_instance,
81  err ? err->what : 0);
82 
83  if (err)
84  {
85  clib_error_t *e = 0;
86  mif->local_disc_string = vec_dup (err->what);
87  if (mif->sock && clib_socket_is_connected (mif->sock))
88  e = memif_msg_send_disconnect (mif, err);
89  clib_error_free (e);
90  }
91 
92  /* set interface down */
93  mif->flags &= ~(MEMIF_IF_FLAG_CONNECTED | MEMIF_IF_FLAG_CONNECTING);
94  if (mif->hw_if_index != ~0)
96 
97  /* close connection socket */
98  if (mif->sock && mif->sock->fd)
99  {
101  mif->socket_file_index);
102  hash_unset (msf->dev_instance_by_fd, mif->sock->fd);
103  memif_socket_close (&mif->sock);
104  }
105  else if (mif->sock)
106  {
107  clib_error_t *err;
108  err = clib_socket_close (mif->sock);
109  if (err)
110  {
111  memif_log_err (mif, "%U", format_clib_error, err);
112  clib_error_free (err);
113  }
114  clib_mem_free (mif->sock);
115  }
116 
117  /* *INDENT-OFF* */
118  vec_foreach_index (i, mif->rx_queues)
119  {
120  mq = vec_elt_at_index (mif->rx_queues, i);
121  if (mq->ring)
122  {
123  int rv;
125  if (rv)
126  memif_log_warn (mif,
127  "Unable to unassign interface %d, queue %d: rc=%d",
128  mif->hw_if_index, i, rv);
129  mq->ring = 0;
130  }
131  }
132 
133  /* free tx and rx queues */
134  vec_foreach (mq, mif->rx_queues)
136  vec_free (mif->rx_queues);
137 
138  vec_foreach (mq, mif->tx_queues)
140  vec_free (mif->tx_queues);
141 
142  /* free memory regions */
143  vec_foreach (mr, mif->regions)
144  {
145  int rv;
146  if (mr->is_external)
147  continue;
148  if ((rv = munmap (mr->shm, mr->region_size)))
149  memif_log_err (mif, "munmap failed, rv = %d", rv);
150  if (mr->fd > -1)
151  close (mr->fd);
152  }
153  /* *INDENT-ON* */
154  vec_free (mif->regions);
155  vec_free (mif->remote_name);
156  vec_free (mif->remote_if_name);
157  clib_fifo_free (mif->msg_queue);
158 }
159 
160 static clib_error_t *
162 {
163  memif_main_t *mm = &memif_main;
164  vnet_main_t *vnm = vnet_get_main ();
165  u16 qid = uf->private_data & 0xFFFF;
166  memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data >> 16);
167  memif_queue_t *mq = vec_elt_at_index (mif->rx_queues, qid);
168  u64 b;
169  ssize_t size;
170 
171  size = read (uf->file_descriptor, &b, sizeof (b));
172  if (size < 0)
173  {
174  memif_log_debug (mif, "Failed to read form socket");
175  return 0;
176  }
177 
179  mq->int_count++;
180 
181  return 0;
182 }
183 
184 
185 clib_error_t *
187 {
188  vnet_main_t *vnm = vnet_get_main ();
189  clib_file_t template = { 0 };
190  memif_region_t *mr;
191  int i;
192  clib_error_t *err = NULL;
193 
194  memif_log_debug (mif, "connect %u", mif->dev_instance);
195 
198 
199  /* *INDENT-OFF* */
200  vec_foreach (mr, mif->regions)
201  {
202  if (mr->shm)
203  continue;
204 
205  if (mr->fd < 0)
206  {
207  err = clib_error_return (0, "no memory region fd");
208  goto error;
209  }
210 
211  if ((mr->shm = mmap (NULL, mr->region_size, PROT_READ | PROT_WRITE,
212  MAP_SHARED, mr->fd, 0)) == MAP_FAILED)
213  {
214  err = clib_error_return_unix (0, "mmap");
215  goto error;
216  }
217  }
218  /* *INDENT-ON* */
219 
220  template.read_function = memif_int_fd_read_ready;
221 
222  /* *INDENT-OFF* */
223  vec_foreach_index (i, mif->tx_queues)
224  {
225  memif_queue_t *mq = vec_elt_at_index (mif->tx_queues, i);
226 
227  mq->ring = mif->regions[mq->region].shm + mq->offset;
228  if (mq->ring->cookie != MEMIF_COOKIE)
229  {
230  err = clib_error_return (0, "wrong cookie on tx ring %u", i);
231  goto error;
232  }
233  }
234 
235  vec_foreach_index (i, mif->rx_queues)
236  {
237  memif_queue_t *mq = vec_elt_at_index (mif->rx_queues, i);
238  int rv;
239 
240  mq->ring = mif->regions[mq->region].shm + mq->offset;
241  if (mq->ring->cookie != MEMIF_COOKIE)
242  {
243  err = clib_error_return (0, "wrong cookie on tx ring %u", i);
244  goto error;
245  }
246 
247  if (mq->int_fd > -1)
248  {
249  template.file_descriptor = mq->int_fd;
250  template.private_data = (mif->dev_instance << 16) | (i & 0xFFFF);
251  template.description = format (0, "%U rx %u int",
253  mif->dev_instance, i);
254  memif_file_add (&mq->int_clib_file_index, &template);
255  }
257  rv = vnet_hw_interface_set_rx_mode (vnm, mif->hw_if_index, i,
259  if (rv)
261  (mif, "Warning: unable to set rx mode for interface %d queue %d: "
262  "rc=%d", mif->hw_if_index, i, rv);
263  else
264  {
266  vnet_hw_interface_get_rx_mode (vnm, mif->hw_if_index, i, &rxmode);
267 
268  if (rxmode == VNET_HW_INTERFACE_RX_MODE_POLLING)
270  else
272  }
273  }
274  /* *INDENT-ON* */
275 
276  mif->flags &= ~MEMIF_IF_FLAG_CONNECTING;
277  mif->flags |= MEMIF_IF_FLAG_CONNECTED;
278 
281  return 0;
282 
283 error:
284  memif_log_err (mif, "%U", format_clib_error, err);
285  return err;
286 }
287 
290 {
291  if (vec_len (mif->regions) == 0)
292  return NULL;
293  void *p = mif->regions[0].shm;
294  int ring_size =
295  sizeof (memif_ring_t) +
296  sizeof (memif_desc_t) * (1 << mif->run.log2_ring_size);
297  p += (ring_num + type * mif->run.num_s2m_rings) * ring_size;
298 
299  return (memif_ring_t *) p;
300 }
301 
302 clib_error_t *
304 {
306  memif_ring_t *ring = NULL;
307  int i, j;
308  u64 buffer_offset;
309  memif_region_t *r;
310  clib_mem_vm_alloc_t alloc = { 0 };
311  clib_error_t *err;
312 
313  ASSERT (vec_len (mif->regions) == 0);
315 
316  buffer_offset = (mif->run.num_s2m_rings + mif->run.num_m2s_rings) *
317  (sizeof (memif_ring_t) +
318  sizeof (memif_desc_t) * (1 << mif->run.log2_ring_size));
319 
320  r->region_size = buffer_offset;
321 
322  if ((mif->flags & MEMIF_IF_FLAG_ZERO_COPY) == 0)
323  r->region_size += mif->run.buffer_size * (1 << mif->run.log2_ring_size) *
324  (mif->run.num_s2m_rings + mif->run.num_m2s_rings);
325 
326  alloc.name = "memif region";
327  alloc.size = r->region_size;
328  alloc.flags = CLIB_MEM_VM_F_SHARED;
329 
330  err = clib_mem_vm_ext_alloc (&alloc);
331  if (err)
332  goto error;
333 
334  r->fd = alloc.fd;
335  r->shm = alloc.addr;
336 
337  if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
338  {
339  vlib_buffer_pool_t *bp;
340  /* *INDENT-OFF* */
342  {
346  r->fd = pr->fd;
347  r->region_size = pr->size;
348  r->shm = pr->mem;
349  r->is_external = 1;
350  }
351  /* *INDENT-ON* */
352  }
353 
354  for (i = 0; i < mif->run.num_s2m_rings; i++)
355  {
356  ring = memif_get_ring (mif, MEMIF_RING_S2M, i);
357  ring->head = ring->tail = 0;
358  ring->cookie = MEMIF_COOKIE;
359 
360  if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
361  continue;
362 
363  for (j = 0; j < (1 << mif->run.log2_ring_size); j++)
364  {
365  u16 slot = i * (1 << mif->run.log2_ring_size) + j;
366  ring->desc[j].region = 0;
367  ring->desc[j].offset =
368  buffer_offset + (u32) (slot * mif->run.buffer_size);
369  ring->desc[j].length = mif->run.buffer_size;
370  }
371  }
372  for (i = 0; i < mif->run.num_m2s_rings; i++)
373  {
374  ring = memif_get_ring (mif, MEMIF_RING_M2S, i);
375  ring->head = ring->tail = 0;
376  ring->cookie = MEMIF_COOKIE;
377 
378  if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
379  continue;
380 
381  for (j = 0; j < (1 << mif->run.log2_ring_size); j++)
382  {
383  u16 slot =
384  (i + mif->run.num_s2m_rings) * (1 << mif->run.log2_ring_size) + j;
385  ring->desc[j].region = 0;
386  ring->desc[j].offset =
387  buffer_offset + (u32) (slot * mif->run.buffer_size);
388  ring->desc[j].length = mif->run.buffer_size;
389  }
390  }
391 
392  ASSERT (mif->tx_queues == 0);
395 
396  /* *INDENT-OFF* */
397  vec_foreach_index (i, mif->tx_queues)
398  {
399  memif_queue_t *mq = vec_elt_at_index (mif->tx_queues, i);
400  if ((mq->int_fd = eventfd (0, EFD_NONBLOCK)) < 0)
401  {
402  err = clib_error_return_unix (0, "eventfd[tx queue %u]", i);
403  goto error;
404  }
405  mq->int_clib_file_index = ~0;
406  mq->ring = memif_get_ring (mif, MEMIF_RING_S2M, i);
407  mq->log2_ring_size = mif->cfg.log2_ring_size;
408  mq->region = 0;
409  mq->offset = (void *) mq->ring - (void *) mif->regions[mq->region].shm;
410  mq->last_head = 0;
411  mq->type = MEMIF_RING_S2M;
412  if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
415  }
416  /* *INDENT-ON* */
417 
418  ASSERT (mif->rx_queues == 0);
421 
422  /* *INDENT-OFF* */
423  vec_foreach_index (i, mif->rx_queues)
424  {
425  memif_queue_t *mq = vec_elt_at_index (mif->rx_queues, i);
426  if ((mq->int_fd = eventfd (0, EFD_NONBLOCK)) < 0)
427  {
428  err = clib_error_return_unix (0, "eventfd[rx queue %u]", i);
429  goto error;
430  }
431  mq->int_clib_file_index = ~0;
432  mq->ring = memif_get_ring (mif, MEMIF_RING_M2S, i);
433  mq->log2_ring_size = mif->cfg.log2_ring_size;
434  mq->region = 0;
435  mq->offset = (void *) mq->ring - (void *) mif->regions[mq->region].shm;
436  mq->last_head = 0;
437  mq->type = MEMIF_RING_M2S;
438  if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
441  }
442  /* *INDENT-ON* */
443 
444  return 0;
445 
446 error:
447  memif_log_err (mif, "%U", format_clib_error, err);
448  return err;
449 }
450 
451 static uword
453 {
454  memif_main_t *mm = &memif_main;
455  memif_if_t *mif;
456  clib_socket_t *sock;
457  uword *event_data = 0, event_type;
458  u8 enabled = 0;
459  f64 start_time, last_run_duration = 0, now;
460  clib_error_t *err;
461 
462  sock = clib_mem_alloc (sizeof (clib_socket_t));
463  memset (sock, 0, sizeof (clib_socket_t));
464 
465  while (1)
466  {
467  if (enabled)
469  last_run_duration);
470  else
472 
473  event_type = vlib_process_get_events (vm, &event_data);
474  vec_reset_length (event_data);
475 
476  switch (event_type)
477  {
478  case ~0:
479  break;
481  enabled = 1;
482  break;
484  enabled = 0;
485  continue;
486  default:
487  ASSERT (0);
488  }
489 
490  last_run_duration = start_time = vlib_time_now (vm);
491  /* *INDENT-OFF* */
492  pool_foreach (mif, mm->interfaces,
493  ({
494  memif_socket_file_t * msf = vec_elt_at_index (mm->socket_files, mif->socket_file_index);
495  /* Allow no more than 10us without a pause */
496  now = vlib_time_now (vm);
497  if (now > start_time + 10e-6)
498  {
499  vlib_process_suspend (vm, 100e-6); /* suspend for 100 us */
500  start_time = vlib_time_now (vm);
501  }
502 
503  if ((mif->flags & MEMIF_IF_FLAG_ADMIN_UP) == 0)
504  continue;
505 
506  if (mif->flags & MEMIF_IF_FLAG_CONNECTING)
507  continue;
508 
509  if (mif->flags & MEMIF_IF_FLAG_CONNECTED)
510  continue;
511 
512  if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE)
513  {
514  memset (sock, 0, sizeof(clib_socket_t));
515  sock->config = (char *) msf->filename;
516  sock->flags = CLIB_SOCKET_F_IS_CLIENT| CLIB_SOCKET_F_SEQPACKET;
517 
518  if ((err = clib_socket_init (sock)))
519  {
520  clib_error_free (err);
521  }
522  else
523  {
524  clib_file_t t = { 0 };
525 
526  t.read_function = memif_slave_conn_fd_read_ready;
527  t.write_function = memif_slave_conn_fd_write_ready;
528  t.error_function = memif_slave_conn_fd_error;
529  t.file_descriptor = sock->fd;
530  t.private_data = mif->dev_instance;
531  memif_file_add (&sock->private_data, &t);
532  t.description = format (0, "%U ctl",
534  mif->dev_instance);
535  hash_set (msf->dev_instance_by_fd, sock->fd, mif->dev_instance);
536 
537  mif->flags |= MEMIF_IF_FLAG_CONNECTING;
538  mif->sock = sock;
539  sock = clib_mem_alloc (sizeof(clib_socket_t));
540  }
541  }
542  }));
543  /* *INDENT-ON* */
544  last_run_duration = vlib_time_now (vm) - last_run_duration;
545  }
546  return 0;
547 }
548 
549 /* *INDENT-OFF* */
551  .function = memif_process,
552  .type = VLIB_NODE_TYPE_PROCESS,
553  .name = "memif-process",
554 };
555 /* *INDENT-ON* */
556 
557 static int
558 memif_add_socket_file (u32 sock_id, u8 * socket_filename)
559 {
560  memif_main_t *mm = &memif_main;
561  uword *p;
562  memif_socket_file_t *msf;
563 
564  p = hash_get (mm->socket_file_index_by_sock_id, sock_id);
565  if (p)
566  {
567  msf = pool_elt_at_index (mm->socket_files, *p);
568  if (strcmp ((char *) msf->filename, (char *) socket_filename) == 0)
569  {
570  /* Silently accept identical "add". */
571  return 0;
572  }
573 
574  /* But don't allow a direct add of a different filename. */
575  return VNET_API_ERROR_ENTRY_ALREADY_EXISTS;
576  }
577 
578  pool_get (mm->socket_files, msf);
579  memset (msf, 0, sizeof (memif_socket_file_t));
580 
581  msf->filename = socket_filename;
582  msf->socket_id = sock_id;
583 
585  msf - mm->socket_files);
586 
587  return 0;
588 }
589 
590 static int
592 {
593  memif_main_t *mm = &memif_main;
594  uword *p;
595  memif_socket_file_t *msf;
596 
597  p = hash_get (mm->socket_file_index_by_sock_id, sock_id);
598  if (!p)
599  {
600  /* Don't delete non-existent entries. */
601  return VNET_API_ERROR_INVALID_ARGUMENT;
602  }
603 
604  msf = pool_elt_at_index (mm->socket_files, *p);
605  if (msf->ref_cnt > 0)
606  {
607  return VNET_API_ERROR_UNEXPECTED_INTF_STATE;
608  }
609 
610  vec_free (msf->filename);
611  pool_put (mm->socket_files, msf);
612 
614 
615  return 0;
616 }
617 
618 int
619 memif_socket_filename_add_del (u8 is_add, u32 sock_id, u8 * sock_filename)
620 {
621  char *dir = 0, *tmp;
622  u32 idx = 0;
623 
624  /* allow adding socket id 0 */
625  if ((sock_id == 0 && is_add == 0) || sock_id == ~0)
626  {
627  return VNET_API_ERROR_INVALID_ARGUMENT;
628  }
629 
630  if (is_add == 0)
631  {
632  return memif_delete_socket_file (sock_id);
633  }
634 
635  if (sock_filename == 0 || sock_filename[0] == 0)
636  {
637  return VNET_API_ERROR_INVALID_ARGUMENT;
638  }
639 
640  if (sock_filename[0] != '/')
641  {
642  clib_error_t *error;
643 
644  /* copy runtime dir path */
646  strlen (vlib_unix_get_runtime_dir ()));
647  vec_add1 (dir, '/');
648 
649  /* if sock_filename contains dirs, add them to path */
650  tmp = strrchr ((char *) sock_filename, '/');
651  if (tmp)
652  {
653  idx = tmp - (char *) sock_filename;
654  vec_add (dir, sock_filename, idx);
655  }
656 
657  vec_add1 (dir, '\0');
658  /* create socket dir */
659  error = vlib_unix_recursive_mkdir (dir);
660  if (error)
661  {
662  clib_error_free (error);
663  return VNET_API_ERROR_SYSCALL_ERROR_1;
664  }
665 
666  sock_filename = format (0, "%s/%s%c", vlib_unix_get_runtime_dir (),
667  sock_filename, 0);
668  }
669  else
670  {
671  sock_filename = vec_dup (sock_filename);
672 
673  /* check if directory exists */
674  tmp = strrchr ((char *) sock_filename, '/');
675  if (tmp)
676  {
677  idx = tmp - (char *) sock_filename;
678  vec_add (dir, sock_filename, idx);
679  vec_add1 (dir, '\0');
680  }
681 
682  /* check dir existance and access rights for effective user/group IDs */
683  if ((dir == NULL)
684  ||
685  (faccessat ( /* ignored */ -1, dir, F_OK | R_OK | W_OK, AT_EACCESS)
686  < 0))
687  {
688  vec_free (dir);
689  return VNET_API_ERROR_INVALID_ARGUMENT;
690  }
691  }
692  vec_free (dir);
693 
694  return memif_add_socket_file (sock_id, sock_filename);
695 }
696 
697 int
699 {
700  vnet_main_t *vnm = vnet_get_main ();
701  memif_main_t *mm = &memif_main;
702  memif_socket_file_t *msf =
704  clib_error_t *err;
705 
706  mif->flags |= MEMIF_IF_FLAG_DELETING;
709 
710  /* bring down the interface */
713 
714  err = clib_error_return (0, "interface deleted");
715  memif_disconnect (mif, err);
716  clib_error_free (err);
717 
718  if (mif->hw_if_index != ~0)
719  {
720  /* remove the interface */
721  if (mif->mode == MEMIF_INTERFACE_MODE_IP)
723  else
725  mif->hw_if_index = ~0;
726  }
727 
728  /* free interface data structures */
729  clib_spinlock_free (&mif->lockp);
730  mhash_unset (&msf->dev_instance_by_id, &mif->id, 0);
731 
732  /* remove socket file */
733  if (--(msf->ref_cnt) == 0)
734  {
735  if (msf->is_listener)
736  {
737  int i;
738  /* *INDENT-OFF* */
741  /* *INDENT-ON* */
742  memif_socket_close (&msf->sock);
743  vec_free (msf->pending_clients);
744  }
747  if (msf->sock)
748  {
749  err = clib_socket_close (msf->sock);
750  if (err)
751  {
752  memif_log_err (mif, "%U", format_clib_error, err);
753  clib_error_free (err);
754  }
755  clib_mem_free (msf->sock);
756  }
757  }
758 
759  memset (mif, 0, sizeof (*mif));
760  pool_put (mm->interfaces, mif);
761 
762  if (pool_elts (mm->interfaces) == 0)
765 
766  return 0;
767 }
768 
769 /* *INDENT-OFF* */
770 VNET_HW_INTERFACE_CLASS (memif_ip_hw_if_class, static) =
771 {
772  .name = "memif-ip",
774 };
775 /* *INDENT-ON* */
776 
777 int
779 {
780  memif_main_t *mm = &memif_main;
782  vnet_main_t *vnm = vnet_get_main ();
783  memif_if_t *mif = 0;
785  clib_error_t *error = 0;
786  int ret = 0;
787  uword *p;
789  memif_socket_file_t *msf = 0;
790  int rv = 0;
791 
793  if (p == 0)
794  {
795  rv = VNET_API_ERROR_INVALID_ARGUMENT;
796  goto done;
797  }
798 
799  msf = vec_elt_at_index (mm->socket_files, p[0]);
800 
801  /* existing socket file can be either master or slave but cannot be both */
802  if (msf->ref_cnt > 0)
803  {
804  if ((!msf->is_listener != !args->is_master))
805  {
806  rv = VNET_API_ERROR_SUBIF_ALREADY_EXISTS;
807  goto done;
808  }
809 
810  p = mhash_get (&msf->dev_instance_by_id, &args->id);
811  if (p)
812  {
813  rv = VNET_API_ERROR_SUBIF_ALREADY_EXISTS;
814  goto done;
815  }
816  }
817 
818  /* Create new socket file */
819  if (msf->ref_cnt == 0)
820  {
821  struct stat file_stat;
822 
823  /* If we are creating listener make sure file doesn't exist or if it
824  * exists thn delete it if it is old socket file */
825  if (args->is_master && (stat ((char *) msf->filename, &file_stat) == 0))
826  {
827  if (S_ISSOCK (file_stat.st_mode))
828  {
829  unlink ((char *) msf->filename);
830  }
831  else
832  {
833  error = clib_error_return (0, "File exists for %s",
834  msf->filename);
835  rv = VNET_API_ERROR_VALUE_EXIST;
836  goto done;
837  }
838  }
839 
840  mhash_init (&msf->dev_instance_by_id, sizeof (uword),
841  sizeof (memif_interface_id_t));
842  msf->dev_instance_by_fd = hash_create (0, sizeof (uword));
843  msf->is_listener = (args->is_master != 0);
844 
845  memif_log_debug (0, "initializing socket file %s", msf->filename);
846  }
847 
848  if (mm->per_thread_data == 0)
849  {
850  int i;
852 
855 
856  fl =
858  for (i = 0; i < tm->n_vlib_mains; i++)
859  {
862  vlib_buffer_t *bt = &ptd->buffer_template;
864  bt->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
866  vnet_buffer (bt)->sw_if_index[VLIB_TX] = (u32) ~ 0;
867 
868  /* initially prealloc copy_ops so we can use
869  _vec_len instead of vec_elen */
871  vec_reset_length (ptd->copy_ops);
873  vec_reset_length (ptd->buffers);
874  }
875  }
876 
877  pool_get (mm->interfaces, mif);
878  memset (mif, 0, sizeof (*mif));
879  mif->dev_instance = mif - mm->interfaces;
880  mif->socket_file_index = msf - mm->socket_files;
881  mif->id = args->id;
882  mif->sw_if_index = mif->hw_if_index = mif->per_interface_next_index = ~0;
883  mif->mode = args->mode;
884  if (args->secret)
885  mif->secret = vec_dup (args->secret);
886 
887  if (tm->n_vlib_mains > 1)
888  clib_spinlock_init (&mif->lockp);
889 
891  {
892 
893  if (!args->hw_addr_set)
894  {
895  f64 now = vlib_time_now (vm);
896  u32 rnd;
897  rnd = (u32) (now * 1e6);
898  rnd = random_u32 (&rnd);
899 
900  memcpy (args->hw_addr + 2, &rnd, sizeof (rnd));
901  args->hw_addr[0] = 2;
902  args->hw_addr[1] = 0xfe;
903  }
905  mif->dev_instance, args->hw_addr,
906  &mif->hw_if_index,
908  }
909  else if (mif->mode == MEMIF_INTERFACE_MODE_IP)
910  {
911  mif->hw_if_index =
913  mif->dev_instance,
914  memif_ip_hw_if_class.index,
915  mif->dev_instance);
916  }
917  else
918  error = clib_error_return (0, "unsupported interface mode");
919 
920  if (error)
921  {
922  ret = VNET_API_ERROR_SYSCALL_ERROR_2;
923  goto error;
924  }
925 
926  sw = vnet_get_hw_sw_interface (vnm, mif->hw_if_index);
927  mif->sw_if_index = sw->sw_if_index;
928 
929  mif->cfg.log2_ring_size = args->log2_ring_size;
930  mif->cfg.buffer_size = args->buffer_size;
931  mif->cfg.num_s2m_rings =
932  args->is_master ? args->rx_queues : args->tx_queues;
933  mif->cfg.num_m2s_rings =
934  args->is_master ? args->tx_queues : args->rx_queues;
935 
936  args->sw_if_index = mif->sw_if_index;
937 
938  /* If this is new one, start listening */
939  if (msf->is_listener && msf->ref_cnt == 0)
940  {
941  struct stat file_stat;
943 
944  ASSERT (msf->sock == 0);
945  msf->sock = s;
946 
947  memset (s, 0, sizeof (clib_socket_t));
948  s->config = (char *) msf->filename;
949  s->flags = CLIB_SOCKET_F_IS_SERVER |
952 
953  if ((error = clib_socket_init (s)))
954  {
955  ret = VNET_API_ERROR_SYSCALL_ERROR_4;
956  goto error;
957  }
958 
959  if (stat ((char *) msf->filename, &file_stat) == -1)
960  {
961  ret = VNET_API_ERROR_SYSCALL_ERROR_8;
962  goto error;
963  }
964 
965  clib_file_t template = { 0 };
967  template.file_descriptor = msf->sock->fd;
968  template.private_data = mif->socket_file_index;
969  template.description = format (0, "memif listener %s", msf->filename);
970  memif_file_add (&msf->sock->private_data, &template);
971  }
972 
973  msf->ref_cnt++;
974 
975  if (args->is_master == 0)
976  {
977  mif->flags |= MEMIF_IF_FLAG_IS_SLAVE;
978  if (args->is_zero_copy)
979  mif->flags |= MEMIF_IF_FLAG_ZERO_COPY;
980  }
981 
982  hw = vnet_get_hw_interface (vnm, mif->hw_if_index);
985  memif_input_node.index);
986 
987  mhash_set (&msf->dev_instance_by_id, &mif->id, mif->dev_instance, 0);
988 
989  if (pool_elts (mm->interfaces) == 1)
990  {
993  }
994  goto done;
995 
996 error:
997  if (mif->hw_if_index != ~0)
998  {
999  if (mif->mode == MEMIF_INTERFACE_MODE_IP)
1001  else
1003  mif->hw_if_index = ~0;
1004  }
1005  memif_delete_if (vm, mif);
1006  if (error)
1007  {
1008  memif_log_err (mif, "%U", format_clib_error, error);
1009  clib_error_free (error);
1010  }
1011  return ret;
1012 
1013 done:
1014  return rv;
1015 }
1016 
1017 static clib_error_t *
1019 {
1020  memif_main_t *mm = &memif_main;
1021 
1022  memset (mm, 0, sizeof (memif_main_t));
1023 
1024  mm->log_class = vlib_log_register_class ("memif_plugin", 0);
1025  memif_log_debug (0, "initialized");
1026 
1027  /* initialize binary API */
1029 
1030  /*
1031  * Pre-stuff socket filename pool with a non-modifieable mapping
1032  * for socket-id 0 to MEMIF_DEFAULT_SOCKET_FILENAME in the
1033  * default run-time directory.
1034  */
1036 
1037  return 0;
1038 }
1039 
1041 
1042 /* *INDENT-OFF* */
1043 VLIB_PLUGIN_REGISTER () = {
1044  .version = VPP_BUILD_VER,
1045  .description = "Packet Memory Interface (experimental)",
1046 };
1047 /* *INDENT-ON* */
1048 
1049 /*
1050  * fd.io coding-style-patch-verification: ON
1051  *
1052  * Local Variables:
1053  * eval: (c-set-style "gnu")
1054  * End:
1055  */
memif_if_t * interfaces
Definition: private.h:238
#define memif_log_err(dev, f,...)
Definition: private.h:55
vlib_log_class_t vlib_log_register_class(char *class, char *subclass)
Definition: log.c:227
vlib_log_class_t log_class
Definition: private.h:247
vlib_physmem_region_index_t physmem_region
Definition: buffer.h:414
vmrglw vmrglh hi
#define vec_foreach_index(var, v)
Iterate over vector indices.
u8 * format_clib_error(u8 *s, va_list *va)
Definition: error.c:191
#define hash_set(h, key, value)
Definition: hash.h:255
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: interface.c:469
static f64 vlib_process_wait_for_event_or_clock(vlib_main_t *vm, f64 dt)
Suspend a cooperative multi-tasking thread Waits for an event, or for the indicated number of seconds...
Definition: node_funcs.h:699
#define hash_unset(h, key)
Definition: hash.h:261
u8 * secret
Definition: private.h:168
static uword * vlib_process_wait_for_event(vlib_main_t *vm)
Definition: node_funcs.h:619
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:323
clib_socket_t ** pending_clients
Definition: private.h:87
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
static uword clib_socket_is_connected(clib_socket_t *sock)
Definition: socket.h:112
unsigned long u64
Definition: types.h:89
memif_socket_file_t * socket_files
Definition: private.h:241
memif_log2_ring_size_t log2_ring_size
Definition: private.h:181
#define NULL
Definition: clib.h:57
uword mhash_unset(mhash_t *h, void *key, uword *old_value)
Definition: mhash.c:353
void * addr
Pointer to allocated memory, set on successful allocation.
Definition: mem.h:382
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:227
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:574
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u32 file_descriptor
Definition: file.h:54
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
static clib_error_t * memif_init(vlib_main_t *vm)
Definition: memif.c:1018
int i
clib_error_t * clib_socket_init(clib_socket_t *s)
Definition: socket.c:383
clib_error_t * memif_msg_send_disconnect(memif_if_t *mif, clib_error_t *err)
Definition: socket.c:190
memif_interface_mode_t mode
Definition: private.h:268
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
#define VNET_HW_INTERFACE_FLAG_LINK_UP
Definition: interface.h:494
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:448
u32 * buffers
Definition: private.h:124
u8 num_m2s_rings
Definition: private.h:183
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:228
unsigned char u8
Definition: types.h:56
#define memif_file_del_by_index(a)
Definition: private.h:77
uint32_t length
Definition: memif.h:152
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
clib_file_function_t * read_function
Definition: file.h:67
double f64
Definition: types.h:142
static void clib_spinlock_free(clib_spinlock_t *p)
Definition: lock.h:64
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
Definition: vec.h:600
#define CLIB_SOCKET_F_IS_SERVER
Definition: socket.h:58
vnet_hw_interface_rx_mode
Definition: interface.h:51
memset(h->entries, 0, sizeof(h->entries[0])*entries)
#define static_always_inline
Definition: clib.h:95
clib_error_t * clib_mem_vm_ext_alloc(clib_mem_vm_alloc_t *a)
Definition: mem.c:80
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:443
u8 * remote_name
Definition: private.h:176
static int memif_add_socket_file(u32 sock_id, u8 *socket_filename)
Definition: memif.c:558
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:163
static uword vlib_process_get_events(vlib_main_t *vm, uword **data_vector)
Return the first event type which has occurred and a vector of per-event data of that type...
Definition: node_funcs.h:542
clib_error_t * memif_plugin_api_hookup(vlib_main_t *vm)
Definition: memif_api.c:408
uword socket_file_index
Definition: private.h:166
static char * vlib_unix_get_runtime_dir(void)
Definition: unix.h:141
uint32_t cookie
Definition: memif.h:166
char * name
Name for memory allocation, set by caller.
Definition: mem.h:379
clib_error_t * memif_init_regions_and_queues(memif_if_t *mif)
Definition: memif.c:303
static_always_inline void vnet_device_input_set_interrupt_pending(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.h:136
u16 buffer_size
Definition: private.h:184
memif_log2_ring_size_t log2_ring_size
Definition: private.h:269
struct memif_if_t::@486 cfg
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
vnet_device_class_t memif_device_class
u32 per_interface_next_index
Definition: private.h:162
static int memif_delete_socket_file(u32 sock_id)
Definition: memif.c:591
#define clib_error_return(e, args...)
Definition: error.h:99
memif_region_offset_t offset
Definition: private.h:120
uword size
Allocation size, set by caller.
Definition: mem.h:380
#define memif_file_add(a, b)
Definition: private.h:66
unsigned int u32
Definition: types.h:88
void * shm
Definition: private.h:102
u32 vnet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, u32 hw_class_index, u32 hw_instance)
Definition: interface.c:703
static u32 memif_eth_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hi, u32 flags)
Definition: memif.c:46
mhash_t dev_instance_by_id
Definition: private.h:92
#define fl(x, y)
static vlib_physmem_region_t * vlib_physmem_get_region(vlib_main_t *vm, u8 index)
Definition: physmem_funcs.h:44
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
#define CLIB_MEM_VM_F_SHARED
Definition: mem.h:362
memif_region_index_t region
Definition: memif.h:151
u16 last_head
Definition: private.h:122
#define hash_get(h, key)
Definition: hash.h:249
memif_copy_op_t * copy_ops
Definition: private.h:222
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:464
uword size
vlib_node_registration_t memif_input_node
(constructor) VLIB_REGISTER_NODE (memif_input_node)
Definition: node.c:889
memif_desc_t desc[0]
Definition: memif.h:173
static void vlib_process_signal_event(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
Definition: node_funcs.h:960
int memif_delete_if(vlib_main_t *vm, memif_if_t *mif)
Definition: memif.c:698
int fd
File descriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set.
Definition: mem.h:383
uword dev_instance
Definition: private.h:159
static clib_error_t * clib_socket_close(clib_socket_t *sock)
Definition: socket.h:175
clib_spinlock_t lockp
Definition: private.h:154
unsigned short u16
Definition: types.h:57
#define clib_error_return_unix(e, args...)
Definition: error.h:102
int vnet_hw_interface_get_rx_mode(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, vnet_hw_interface_rx_mode *mode)
Definition: devices.c:312
#define hash_free(h)
Definition: hash.h:310
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
Definition: buffer.h:440
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:274
#define vec_dup(V)
Return copy of vector (no header, no alignment)
Definition: vec.h:373
vlib_buffer_pool_t * buffer_pools
Definition: buffer.h:434
memif_interface_id_t id
Definition: private.h:263
int memif_create_if(vlib_main_t *vm, memif_create_if_args_t *args)
Definition: memif.c:778
uword int_clib_file_index
Definition: private.h:128
#define VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE
Definition: interface.h:532
VNET_HW_INTERFACE_CLASS(memif_ip_hw_if_class, static)
static uword mhash_set(mhash_t *h, void *key, uword new_value, uword *old_value)
Definition: mhash.h:117
memif_queue_t * tx_queues
Definition: private.h:173
u32 flags
Definition: vhost_user.h:115
clib_error_t * memif_connect(memif_if_t *mif)
Definition: memif.c:186
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:155
void mhash_init(mhash_t *h, uword n_value_bytes, uword n_key_bytes)
Definition: mhash.c:168
vlib_main_t * vm
Definition: buffer.c:294
u8 * local_disc_string
Definition: private.h:196
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
int memif_socket_filename_add_del(u8 is_add, u32 sock_id, u8 *sock_filename)
Definition: memif.c:619
uint32_t memif_interface_id_t
Definition: memif.h:64
void memif_disconnect(memif_if_t *mif, clib_error_t *err)
Definition: memif.c:69
u32 flags
vm allocation flags: CLIB_MEM_VM_F_SHARED: request shared memory, file descriptor will be provided ...
Definition: mem.h:368
memif_ring_type_t type
Definition: private.h:132
memif_region_t * regions
Definition: private.h:170
#define hash_create(elts, value_bytes)
Definition: hash.h:696
#define ASSERT(truth)
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
Definition: devices.c:138
static uword * mhash_get(mhash_t *h, const void *key)
Definition: mhash.h:110
static void memif_queue_intfd_close(memif_queue_t *mq)
Definition: memif.c:53
static void mhash_free(mhash_t *h)
Definition: mhash.h:149
static clib_error_t * memif_int_fd_read_ready(clib_file_t *uf)
Definition: memif.c:161
u32 flags
Definition: private.h:155
memif_ring_t * ring
Definition: private.h:117
static void clib_mem_free(void *p)
Definition: mem.h:205
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:277
vlib_buffer_t buffer_template
Definition: private.h:226
u32 hw_if_index
Definition: private.h:157
struct _socket_t clib_socket_t
clib_error_t * memif_slave_conn_fd_write_ready(clib_file_t *uf)
Definition: socket.c:588
#define clib_fifo_free(f)
Definition: fifo.h:257
#define MEMIF_RING_FLAG_MASK_INT
Definition: memif.h:168
static void * clib_mem_alloc(uword size)
Definition: mem.h:132
clib_error_t * memif_slave_conn_fd_error(clib_file_t *uf)
Definition: socket.c:596
#define CLIB_SOCKET_F_SEQPACKET
Definition: socket.h:63
clib_error_t * memif_slave_conn_fd_read_ready(clib_file_t *uf)
Definition: socket.c:545
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
u64 int_count
Definition: private.h:129
void vnet_delete_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:913
memif_region_offset_t offset
Definition: memif.h:153
uword * dev_instance_by_fd
Definition: private.h:95
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:155
u8 num_s2m_rings
Definition: private.h:182
VLIB_PLUGIN_REGISTER()
Definition: defs.h:47
clib_error_t * vlib_unix_recursive_mkdir(char *path)
Definition: util.c:103
static_always_inline memif_ring_t * memif_get_ring(memif_if_t *mif, memif_ring_type_t type, u16 ring_num)
Definition: memif.c:289
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define MEMIF_DEFAULT_SOCKET_FILENAME
Definition: private.h:21
#define CLIB_SOCKET_F_PASSCRED
Definition: socket.h:64
#define memif_log_warn(dev, f,...)
Definition: private.h:44
u64 uword
Definition: types.h:112
a point 2 point interface
Definition: interface.h:381
void memif_socket_close(clib_socket_t **sock)
Definition: socket.c:43
#define clib_error_free(e)
Definition: error.h:86
u8 * remote_if_name
Definition: private.h:177
memif_interface_id_t id
Definition: private.h:156
memif_log2_ring_size_t log2_ring_size
Definition: private.h:118
int vnet_hw_interface_unassign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.c:187
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
#define vnet_buffer(b)
Definition: buffer.h:344
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
struct memif_if_t::@487 run
uint16_t flags
Definition: memif.h:167
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
memif_per_thread_data_t * per_thread_data
Definition: private.h:245
static vlib_node_registration_t memif_process_node
(constructor) VLIB_REGISTER_NODE (memif_process_node)
Definition: memif.c:550
vlib_buffer_main_t buffer_main
Definition: buffer.c:52
#define memif_log_debug(dev, f,...)
Definition: private.h:33
u8 * remote_disc_string
Definition: private.h:197
memif_ring_type_t
Definition: memif.h:47
#define vec_foreach(var, vec)
Vector iterator.
volatile uint16_t head
Definition: memif.h:169
uword private_data
Definition: file.h:64
clib_error_t * vnet_sw_interface_set_flags(vnet_main_t *vnm, u32 sw_if_index, u32 flags)
Definition: interface.c:477
Definition: file.h:51
clib_socket_t * sock
Definition: private.h:165
memif_queue_t * rx_queues
Definition: private.h:172
int vnet_hw_interface_set_rx_mode(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, vnet_hw_interface_rx_mode mode)
Definition: devices.c:252
clib_error_t * memif_conn_fd_accept_ready(clib_file_t *uf)
Definition: socket.c:649
uword * socket_file_index_by_sock_id
Definition: private.h:242
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer_funcs.h:675
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
#define CLIB_SOCKET_F_ALLOW_GROUP_WRITE
Definition: socket.h:62
memif_msg_fifo_elt_t * msg_queue
Definition: private.h:167
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:116
memif_main_t memif_main
Definition: memif.c:43
memif_region_index_t region
Definition: private.h:119
clib_socket_t * sock
Definition: private.h:86
#define MEMIF_COOKIE
Definition: memif.h:25
u32 sw_if_index
Definition: private.h:158
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: devices.h:79
volatile uint16_t tail
Definition: memif.h:171
memif_interface_mode_t mode
Definition: private.h:160
static uword memif_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: memif.c:452
u8 * format_memif_device_name(u8 *s, va_list *args)
Definition: device.c:51
memif_region_size_t region_size
Definition: private.h:103
static uword pool_elts(void *v)
Number of active elements in a pool.
Definition: pool.h:128