FD.io VPP  v18.04-17-g3a0d853
Vector Packet Processing
memif.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2017 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 
19 #define _GNU_SOURCE
20 #include <stdint.h>
21 #include <net/if.h>
22 #include <sys/types.h>
23 #include <fcntl.h>
24 #include <sys/ioctl.h>
25 #include <sys/socket.h>
26 #include <sys/un.h>
27 #include <sys/uio.h>
28 #include <sys/mman.h>
29 #include <sys/prctl.h>
30 #include <sys/eventfd.h>
31 #include <inttypes.h>
32 #include <limits.h>
33 
34 #include <vlib/vlib.h>
35 #include <vlib/unix/unix.h>
36 #include <vppinfra/linux/syscall.h>
37 #include <vnet/plugin/plugin.h>
38 #include <vnet/ethernet/ethernet.h>
39 #include <vpp/app/version.h>
40 #include <memif/memif.h>
41 #include <memif/private.h>
42 
44 
45 static u32
47 {
48  /* nothing for now */
49  return 0;
50 }
51 
52 static void
54 {
55  if (mq->int_clib_file_index != ~0)
56  {
58  mq->int_clib_file_index = ~0;
59  mq->int_fd = -1;
60  }
61  else if (mq->int_fd > -1)
62  {
63  close (mq->int_fd);
64  mq->int_fd = -1;
65  }
66 }
67 
68 void
70 {
71  memif_main_t *mm = &memif_main;
72  vnet_main_t *vnm = vnet_get_main ();
73  memif_region_t *mr;
74  memif_queue_t *mq;
75  int i;
76 
77  if (mif == 0)
78  return;
79 
80  DBG ("disconnect %u (%v)", mif->dev_instance, err ? err->what : 0);
81 
82  if (err)
83  {
84  clib_error_t *e = 0;
85  mif->local_disc_string = vec_dup (err->what);
86  if (mif->sock && clib_socket_is_connected (mif->sock))
87  e = memif_msg_send_disconnect (mif, err);
88  clib_error_free (e);
89  }
90 
91  /* set interface down */
92  mif->flags &= ~(MEMIF_IF_FLAG_CONNECTED | MEMIF_IF_FLAG_CONNECTING);
93  if (mif->hw_if_index != ~0)
95 
96  /* close connection socket */
97  if (mif->sock && mif->sock->fd)
98  {
100  mif->socket_file_index);
101  hash_unset (msf->dev_instance_by_fd, mif->sock->fd);
102  memif_socket_close (&mif->sock);
103  }
104  else if (mif->sock)
105  {
106  clib_error_t *err;
107  err = clib_socket_close (mif->sock);
108  if (err)
109  clib_error_report (err);
110  clib_mem_free (mif->sock);
111  }
112 
113  /* *INDENT-OFF* */
114  vec_foreach_index (i, mif->rx_queues)
115  {
116  mq = vec_elt_at_index (mif->rx_queues, i);
117  if (mq->ring)
118  {
119  int rv;
121  if (rv)
122  DBG ("Warning: unable to unassign interface %d, "
123  "queue %d: rc=%d", mif->hw_if_index, i, rv);
124  mq->ring = 0;
125  }
126  }
127 
128  /* free tx and rx queues */
129  vec_foreach (mq, mif->rx_queues)
131  vec_free (mif->rx_queues);
132 
133  vec_foreach (mq, mif->tx_queues)
135  vec_free (mif->tx_queues);
136 
137  /* free memory regions */
138  vec_foreach (mr, mif->regions)
139  {
140  int rv;
141  if (mr->is_external)
142  continue;
143  if ((rv = munmap (mr->shm, mr->region_size)))
144  clib_warning ("munmap failed, rv = %d", rv);
145  if (mr->fd > -1)
146  close (mr->fd);
147  }
148  /* *INDENT-ON* */
149  vec_free (mif->regions);
150  vec_free (mif->remote_name);
151  vec_free (mif->remote_if_name);
152  clib_fifo_free (mif->msg_queue);
153 }
154 
155 static clib_error_t *
157 {
158  memif_main_t *mm = &memif_main;
159  vnet_main_t *vnm = vnet_get_main ();
160  u16 qid = uf->private_data & 0xFFFF;
161  memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data >> 16);
162  memif_queue_t *mq = vec_elt_at_index (mif->rx_queues, qid);
163  u64 b;
164  ssize_t size;
165 
166  size = read (uf->file_descriptor, &b, sizeof (b));
167  if (size < 0)
168  {
169  DBG_UNIX_LOG ("Failed to read from socket");
170  return 0;
171  }
172 
174  mq->int_count++;
175 
176  return 0;
177 }
178 
179 
180 clib_error_t *
182 {
183  vnet_main_t *vnm = vnet_get_main ();
184  clib_file_t template = { 0 };
185  memif_region_t *mr;
186  int i;
187 
188  DBG ("connect %u", mif->dev_instance);
189 
192 
193  /* *INDENT-OFF* */
194  vec_foreach (mr, mif->regions)
195  {
196  if (mr->shm)
197  continue;
198 
199  if (mr->fd < 0)
200  clib_error_return (0, "no memory region fd");
201 
202  if ((mr->shm = mmap (NULL, mr->region_size, PROT_READ | PROT_WRITE,
203  MAP_SHARED, mr->fd, 0)) == MAP_FAILED)
204  return clib_error_return_unix (0, "mmap");
205  }
206  /* *INDENT-ON* */
207 
208  template.read_function = memif_int_fd_read_ready;
209 
210  /* *INDENT-OFF* */
211  vec_foreach_index (i, mif->tx_queues)
212  {
213  memif_queue_t *mq = vec_elt_at_index (mif->tx_queues, i);
214 
215  mq->ring = mif->regions[mq->region].shm + mq->offset;
216  if (mq->ring->cookie != MEMIF_COOKIE)
217  return clib_error_return (0, "wrong cookie on tx ring %u", i);
218  }
219 
220  vec_foreach_index (i, mif->rx_queues)
221  {
222  memif_queue_t *mq = vec_elt_at_index (mif->rx_queues, i);
223  int rv;
224 
225  mq->ring = mif->regions[mq->region].shm + mq->offset;
226  if (mq->ring->cookie != MEMIF_COOKIE)
227  return clib_error_return (0, "wrong cookie on tx ring %u", i);
228 
229  if (mq->int_fd > -1)
230  {
231  template.file_descriptor = mq->int_fd;
232  template.private_data = (mif->dev_instance << 16) | (i & 0xFFFF);
233  template.description = format (0, "%U rx %u int",
235  mif->dev_instance, i);
236  memif_file_add (&mq->int_clib_file_index, &template);
237  }
239  rv = vnet_hw_interface_set_rx_mode (vnm, mif->hw_if_index, i,
241  if (rv)
243  ("Warning: unable to set rx mode for interface %d queue %d: "
244  "rc=%d", mif->hw_if_index, i, rv);
245  else
246  {
248  vnet_hw_interface_get_rx_mode (vnm, mif->hw_if_index, i, &rxmode);
249 
250  if (rxmode == VNET_HW_INTERFACE_RX_MODE_POLLING)
252  else
254  }
255  }
256  /* *INDENT-ON* */
257 
258  mif->flags &= ~MEMIF_IF_FLAG_CONNECTING;
259  mif->flags |= MEMIF_IF_FLAG_CONNECTED;
260 
263  return 0;
264 }
265 
268 {
269  if (vec_len (mif->regions) == 0)
270  return NULL;
271  void *p = mif->regions[0].shm;
272  int ring_size =
273  sizeof (memif_ring_t) +
274  sizeof (memif_desc_t) * (1 << mif->run.log2_ring_size);
275  p += (ring_num + type * mif->run.num_s2m_rings) * ring_size;
276 
277  return (memif_ring_t *) p;
278 }
279 
280 clib_error_t *
282 {
284  memif_ring_t *ring = NULL;
285  int i, j;
286  u64 buffer_offset;
287  memif_region_t *r;
288  clib_mem_vm_alloc_t alloc = { 0 };
289  clib_error_t *err;
290 
291  ASSERT (vec_len (mif->regions) == 0);
293 
294  buffer_offset = (mif->run.num_s2m_rings + mif->run.num_m2s_rings) *
295  (sizeof (memif_ring_t) +
296  sizeof (memif_desc_t) * (1 << mif->run.log2_ring_size));
297 
298  r->region_size = buffer_offset;
299 
300  if ((mif->flags & MEMIF_IF_FLAG_ZERO_COPY) == 0)
301  r->region_size += mif->run.buffer_size * (1 << mif->run.log2_ring_size) *
302  (mif->run.num_s2m_rings + mif->run.num_m2s_rings);
303 
304  alloc.name = "memif region";
305  alloc.size = r->region_size;
306  alloc.flags = CLIB_MEM_VM_F_SHARED;
307 
308  err = clib_mem_vm_ext_alloc (&alloc);
309  if (err)
310  return err;
311 
312  r->fd = alloc.fd;
313  r->shm = alloc.addr;
314 
315  if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
316  {
317  vlib_buffer_pool_t *bp;
318  /* *INDENT-OFF* */
320  {
324  r->fd = pr->fd;
325  r->region_size = pr->size;
326  r->shm = pr->mem;
327  r->is_external = 1;
328  }
329  /* *INDENT-ON* */
330  }
331 
332  for (i = 0; i < mif->run.num_s2m_rings; i++)
333  {
334  ring = memif_get_ring (mif, MEMIF_RING_S2M, i);
335  ring->head = ring->tail = 0;
336  ring->cookie = MEMIF_COOKIE;
337 
338  if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
339  continue;
340 
341  for (j = 0; j < (1 << mif->run.log2_ring_size); j++)
342  {
343  u16 slot = i * (1 << mif->run.log2_ring_size) + j;
344  ring->desc[j].region = 0;
345  ring->desc[j].offset =
346  buffer_offset + (u32) (slot * mif->run.buffer_size);
347  ring->desc[j].length = mif->run.buffer_size;
348  }
349  }
350  for (i = 0; i < mif->run.num_m2s_rings; i++)
351  {
352  ring = memif_get_ring (mif, MEMIF_RING_M2S, i);
353  ring->head = ring->tail = 0;
354  ring->cookie = MEMIF_COOKIE;
355 
356  if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
357  continue;
358 
359  for (j = 0; j < (1 << mif->run.log2_ring_size); j++)
360  {
361  u16 slot =
362  (i + mif->run.num_s2m_rings) * (1 << mif->run.log2_ring_size) + j;
363  ring->desc[j].region = 0;
364  ring->desc[j].offset =
365  buffer_offset + (u32) (slot * mif->run.buffer_size);
366  ring->desc[j].length = mif->run.buffer_size;
367  }
368  }
369 
370  ASSERT (mif->tx_queues == 0);
373 
374  /* *INDENT-OFF* */
375  vec_foreach_index (i, mif->tx_queues)
376  {
377  memif_queue_t *mq = vec_elt_at_index (mif->tx_queues, i);
378  if ((mq->int_fd = eventfd (0, EFD_NONBLOCK)) < 0)
379  return clib_error_return_unix (0, "eventfd[tx queue %u]", i);
380  mq->int_clib_file_index = ~0;
381  mq->ring = memif_get_ring (mif, MEMIF_RING_S2M, i);
382  mq->log2_ring_size = mif->cfg.log2_ring_size;
383  mq->region = 0;
384  mq->offset = (void *) mq->ring - (void *) mif->regions[mq->region].shm;
385  mq->last_head = 0;
386  mq->type = MEMIF_RING_S2M;
387  if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
390  }
391  /* *INDENT-ON* */
392 
393  ASSERT (mif->rx_queues == 0);
396 
397  /* *INDENT-OFF* */
398  vec_foreach_index (i, mif->rx_queues)
399  {
400  memif_queue_t *mq = vec_elt_at_index (mif->rx_queues, i);
401  if ((mq->int_fd = eventfd (0, EFD_NONBLOCK)) < 0)
402  return clib_error_return_unix (0, "eventfd[rx queue %u]", i);
403  mq->int_clib_file_index = ~0;
404  mq->ring = memif_get_ring (mif, MEMIF_RING_M2S, i);
405  mq->log2_ring_size = mif->cfg.log2_ring_size;
406  mq->region = 0;
407  mq->offset = (void *) mq->ring - (void *) mif->regions[mq->region].shm;
408  mq->last_head = 0;
409  mq->type = MEMIF_RING_M2S;
410  if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
413  }
414  /* *INDENT-ON* */
415 
416  return 0;
417 }
418 
419 static uword
421 {
422  memif_main_t *mm = &memif_main;
423  memif_if_t *mif;
424  clib_socket_t *sock;
425  uword *event_data = 0, event_type;
426  u8 enabled = 0;
427  f64 start_time, last_run_duration = 0, now;
428  clib_error_t *err;
429 
430  sock = clib_mem_alloc (sizeof (clib_socket_t));
431  memset (sock, 0, sizeof (clib_socket_t));
432 
433  while (1)
434  {
435  if (enabled)
437  last_run_duration);
438  else
440 
441  event_type = vlib_process_get_events (vm, &event_data);
442  vec_reset_length (event_data);
443 
444  switch (event_type)
445  {
446  case ~0:
447  break;
449  enabled = 1;
450  break;
452  enabled = 0;
453  continue;
454  default:
455  ASSERT (0);
456  }
457 
458  last_run_duration = start_time = vlib_time_now (vm);
459  /* *INDENT-OFF* */
460  pool_foreach (mif, mm->interfaces,
461  ({
462  memif_socket_file_t * msf = vec_elt_at_index (mm->socket_files, mif->socket_file_index);
463  /* Allow no more than 10us without a pause */
464  now = vlib_time_now (vm);
465  if (now > start_time + 10e-6)
466  {
467  vlib_process_suspend (vm, 100e-6); /* suspend for 100 us */
468  start_time = vlib_time_now (vm);
469  }
470 
471  if ((mif->flags & MEMIF_IF_FLAG_ADMIN_UP) == 0)
472  continue;
473 
474  if (mif->flags & MEMIF_IF_FLAG_CONNECTING)
475  continue;
476 
477  if (mif->flags & MEMIF_IF_FLAG_CONNECTED)
478  continue;
479 
480  if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE)
481  {
482  memset (sock, 0, sizeof(clib_socket_t));
483  sock->config = (char *) msf->filename;
484  sock->flags = CLIB_SOCKET_F_IS_CLIENT| CLIB_SOCKET_F_SEQPACKET;
485 
486  if ((err = clib_socket_init (sock)))
487  {
488  clib_error_free (err);
489  }
490  else
491  {
492  clib_file_t t = { 0 };
493 
494  t.read_function = memif_slave_conn_fd_read_ready;
495  t.write_function = memif_slave_conn_fd_write_ready;
496  t.error_function = memif_slave_conn_fd_error;
497  t.file_descriptor = sock->fd;
498  t.private_data = mif->dev_instance;
499  memif_file_add (&sock->private_data, &t);
500  t.description = format (0, "%U ctl",
502  mif->dev_instance);
503  hash_set (msf->dev_instance_by_fd, sock->fd, mif->dev_instance);
504 
505  mif->flags |= MEMIF_IF_FLAG_CONNECTING;
506  mif->sock = sock;
507  sock = clib_mem_alloc (sizeof(clib_socket_t));
508  }
509  }
510  }));
511  /* *INDENT-ON* */
512  last_run_duration = vlib_time_now (vm) - last_run_duration;
513  }
514  return 0;
515 }
516 
517 /* *INDENT-OFF* */
519  .function = memif_process,
520  .type = VLIB_NODE_TYPE_PROCESS,
521  .name = "memif-process",
522 };
523 /* *INDENT-ON* */
524 
525 
526 static int
527 memif_add_socket_file (u32 sock_id, u8 * socket_filename)
528 {
529  memif_main_t *mm = &memif_main;
530  uword *p;
531  memif_socket_file_t *msf;
532 
533  p = hash_get (mm->socket_file_index_by_sock_id, sock_id);
534  if (p)
535  {
536  msf = pool_elt_at_index (mm->socket_files, *p);
537  if (strcmp ((char *) msf->filename, (char *) socket_filename) == 0)
538  {
539  /* Silently accept identical "add". */
540  return 0;
541  }
542 
543  /* But don't allow a direct add of a different filename. */
544  return VNET_API_ERROR_ENTRY_ALREADY_EXISTS;
545  }
546 
547  DBG ("creating socket file (uninitialized) %s", msf->filename);
548 
549  pool_get (mm->socket_files, msf);
550  memset (msf, 0, sizeof (memif_socket_file_t));
551 
552  msf->filename = socket_filename;
553  msf->socket_id = sock_id;
554 
556  msf - mm->socket_files);
557 
558  return 0;
559 }
560 
561 static int
563 {
564  memif_main_t *mm = &memif_main;
565  uword *p;
566  memif_socket_file_t *msf;
567 
568  p = hash_get (mm->socket_file_index_by_sock_id, sock_id);
569  if (!p)
570  {
571  /* Don't delete non-existent entries. */
572  return VNET_API_ERROR_INVALID_ARGUMENT;
573  }
574 
575  msf = pool_elt_at_index (mm->socket_files, *p);
576  if (msf->ref_cnt > 0)
577  {
578  return VNET_API_ERROR_UNEXPECTED_INTF_STATE;
579  }
580 
581  vec_free (msf->filename);
582  pool_put (mm->socket_files, msf);
583 
585 
586  return 0;
587 }
588 
589 int
590 memif_socket_filename_add_del (u8 is_add, u32 sock_id, u8 * sock_filename)
591 {
592  if (sock_id == 0 || sock_id == ~0)
593  {
594  return VNET_API_ERROR_INVALID_ARGUMENT;
595  }
596 
597  if (is_add == 0)
598  {
599  return memif_delete_socket_file (sock_id);
600  }
601 
602  if (sock_filename == 0 || sock_filename[0] == 0)
603  {
604  return VNET_API_ERROR_INVALID_ARGUMENT;
605  }
606 
607  if (sock_filename[0] != '/')
608  {
609  clib_error_t *error;
611  if (error)
612  {
613  clib_error_free (error);
614  return VNET_API_ERROR_SYSCALL_ERROR_1;
615  }
616 
617  sock_filename = format (0, "%s/%s%c", vlib_unix_get_runtime_dir (),
618  sock_filename, 0);
619  }
620  else
621  {
622  sock_filename = vec_dup (sock_filename);
623  }
624 
625  return memif_add_socket_file (sock_id, sock_filename);
626 }
627 
628 int
630 {
631  vnet_main_t *vnm = vnet_get_main ();
632  memif_main_t *mm = &memif_main;
633  memif_socket_file_t *msf =
635  clib_error_t *err;
636 
637  mif->flags |= MEMIF_IF_FLAG_DELETING;
640 
641  /* bring down the interface */
644 
645  err = clib_error_return (0, "interface deleted");
646  memif_disconnect (mif, err);
647  clib_error_free (err);
648 
649  if (mif->hw_if_index != ~0)
650  {
651  /* remove the interface */
652  if (mif->mode == MEMIF_INTERFACE_MODE_IP)
654  else
656  mif->hw_if_index = ~0;
657  }
658 
659  /* free interface data structures */
660  clib_spinlock_free (&mif->lockp);
661  mhash_unset (&msf->dev_instance_by_id, &mif->id, 0);
662 
663  /* remove socket file */
664  if (--(msf->ref_cnt) == 0)
665  {
666  if (msf->is_listener)
667  {
668  int i;
669  /* *INDENT-OFF* */
672  /* *INDENT-ON* */
673  memif_socket_close (&msf->sock);
674  vec_free (msf->pending_clients);
675  }
678  if (msf->sock)
679  {
680  err = clib_socket_close (msf->sock);
681  if (err)
682  clib_error_report (err);
683  clib_mem_free (msf->sock);
684  }
685  }
686 
687  memset (mif, 0, sizeof (*mif));
688  pool_put (mm->interfaces, mif);
689 
690  if (pool_elts (mm->interfaces) == 0)
693 
694  return 0;
695 }
696 
697 /* *INDENT-OFF* */
698 VNET_HW_INTERFACE_CLASS (memif_ip_hw_if_class, static) =
699 {
700  .name = "memif-ip",
702 };
703 /* *INDENT-ON* */
704 
705 int
707 {
708  memif_main_t *mm = &memif_main;
710  vnet_main_t *vnm = vnet_get_main ();
711  memif_if_t *mif = 0;
713  clib_error_t *error = 0;
714  int ret = 0;
715  uword *p;
717  memif_socket_file_t *msf = 0;
718  int rv = 0;
719 
721  if (p == 0)
722  {
723  rv = VNET_API_ERROR_INVALID_ARGUMENT;
724  goto done;
725  }
726 
727  msf = vec_elt_at_index (mm->socket_files, p[0]);
728 
729  /* existing socket file can be either master or slave but cannot be both */
730  if (msf->ref_cnt > 0)
731  {
732  if ((!msf->is_listener != !args->is_master))
733  {
734  rv = VNET_API_ERROR_SUBIF_ALREADY_EXISTS;
735  goto done;
736  }
737 
738  p = mhash_get (&msf->dev_instance_by_id, &args->id);
739  if (p)
740  {
741  rv = VNET_API_ERROR_SUBIF_ALREADY_EXISTS;
742  goto done;
743  }
744  }
745 
746  /* Create new socket file */
747  if (msf->ref_cnt == 0)
748  {
749  struct stat file_stat;
750 
751  /* If we are creating listener make sure file doesn't exist or if it
752  * exists thn delete it if it is old socket file */
753  if (args->is_master && (stat ((char *) msf->filename, &file_stat) == 0))
754  {
755  if (S_ISSOCK (file_stat.st_mode))
756  {
757  unlink ((char *) msf->filename);
758  }
759  else
760  {
761  error = clib_error_return (0, "File exists for %s",
762  msf->filename);
763  clib_error_report (error);
764  rv = VNET_API_ERROR_VALUE_EXIST;
765  goto done;
766  }
767  }
768 
769  mhash_init (&msf->dev_instance_by_id, sizeof (uword),
770  sizeof (memif_interface_id_t));
771  msf->dev_instance_by_fd = hash_create (0, sizeof (uword));
772  msf->is_listener = (args->is_master != 0);
773 
774  DBG ("initializing socket file %s", msf->filename);
775  }
776 
777  if (mm->per_thread_data == 0)
778  {
779  int i;
781 
784 
785  fl =
787  for (i = 0; i < tm->n_vlib_mains; i++)
788  {
791  vlib_buffer_t *bt = &ptd->buffer_template;
793  bt->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
795  vnet_buffer (bt)->sw_if_index[VLIB_TX] = (u32) ~ 0;
796 
797  /* initially prealloc copy_ops so we can use
798  _vec_len instead of vec_elen */
800  vec_reset_length (ptd->copy_ops);
802  vec_reset_length (ptd->buffers);
803  }
804  }
805 
806  pool_get (mm->interfaces, mif);
807  memset (mif, 0, sizeof (*mif));
808  mif->dev_instance = mif - mm->interfaces;
809  mif->socket_file_index = msf - mm->socket_files;
810  mif->id = args->id;
811  mif->sw_if_index = mif->hw_if_index = mif->per_interface_next_index = ~0;
812  mif->mode = args->mode;
813  if (args->secret)
814  mif->secret = vec_dup (args->secret);
815 
816  if (tm->n_vlib_mains > 1)
817  clib_spinlock_init (&mif->lockp);
818 
820  {
821 
822  if (!args->hw_addr_set)
823  {
824  f64 now = vlib_time_now (vm);
825  u32 rnd;
826  rnd = (u32) (now * 1e6);
827  rnd = random_u32 (&rnd);
828 
829  memcpy (args->hw_addr + 2, &rnd, sizeof (rnd));
830  args->hw_addr[0] = 2;
831  args->hw_addr[1] = 0xfe;
832  }
834  mif->dev_instance, args->hw_addr,
835  &mif->hw_if_index,
837  }
838  else if (mif->mode == MEMIF_INTERFACE_MODE_IP)
839  {
840  mif->hw_if_index =
842  mif->dev_instance,
843  memif_ip_hw_if_class.index,
844  mif->dev_instance);
845  }
846  else
847  error = clib_error_return (0, "unsupported interface mode");
848 
849  if (error)
850  {
851  clib_error_report (error);
852  ret = VNET_API_ERROR_SYSCALL_ERROR_2;
853  goto error;
854  }
855 
856  sw = vnet_get_hw_sw_interface (vnm, mif->hw_if_index);
857  mif->sw_if_index = sw->sw_if_index;
858 
859  mif->cfg.log2_ring_size = args->log2_ring_size;
860  mif->cfg.buffer_size = args->buffer_size;
861  mif->cfg.num_s2m_rings =
862  args->is_master ? args->rx_queues : args->tx_queues;
863  mif->cfg.num_m2s_rings =
864  args->is_master ? args->tx_queues : args->rx_queues;
865 
866  args->sw_if_index = mif->sw_if_index;
867 
868  /* If this is new one, start listening */
869  if (msf->is_listener && msf->ref_cnt == 0)
870  {
871  struct stat file_stat;
873 
874  ASSERT (msf->sock == 0);
875  msf->sock = s;
876 
877  memset (s, 0, sizeof (clib_socket_t));
878  s->config = (char *) msf->filename;
879  s->flags = CLIB_SOCKET_F_IS_SERVER |
882 
883  if ((error = clib_socket_init (s)))
884  {
885  clib_error_report (error);
886  ret = VNET_API_ERROR_SYSCALL_ERROR_4;
887  goto error;
888  }
889 
890  if (stat ((char *) msf->filename, &file_stat) == -1)
891  {
892  ret = VNET_API_ERROR_SYSCALL_ERROR_8;
893  goto error;
894  }
895 
896  clib_file_t template = { 0 };
898  template.file_descriptor = msf->sock->fd;
899  template.private_data = mif->socket_file_index;
900  template.description = format (0, "memif listener %s", msf->filename);
901  memif_file_add (&msf->sock->private_data, &template);
902  }
903 
904  msf->ref_cnt++;
905 
906  if (args->is_master == 0)
907  {
908  mif->flags |= MEMIF_IF_FLAG_IS_SLAVE;
909  if (args->is_zero_copy)
910  mif->flags |= MEMIF_IF_FLAG_ZERO_COPY;
911  }
912 
913  hw = vnet_get_hw_interface (vnm, mif->hw_if_index);
916  memif_input_node.index);
917 
918  mhash_set (&msf->dev_instance_by_id, &mif->id, mif->dev_instance, 0);
919 
920  if (pool_elts (mm->interfaces) == 1)
921  {
924  }
925  goto done;
926 
927 error:
928  if (mif->hw_if_index != ~0)
929  {
930  if (mif->mode == MEMIF_INTERFACE_MODE_IP)
932  else
934  mif->hw_if_index = ~0;
935  }
936  memif_delete_if (vm, mif);
937  return ret;
938 
939 done:
940  return rv;
941 }
942 
943 
944 static clib_error_t *
946 {
947  memif_main_t *mm = &memif_main;
948  u8 *filename;
949 
950  memset (mm, 0, sizeof (memif_main_t));
951 
952  /* initialize binary API */
954 
955  /*
956  * Pre-stuff socket filename pool with a non-modifieable mapping
957  * for socket-id 0 to MEMIF_DEFAULT_SOCKET_FILENAME in the
958  * default run-time directory.
959  */
960  filename = format (0, "%s/%s%c",
963  memif_add_socket_file (0, filename);
964 
965  return 0;
966 }
967 
969 
970 /* *INDENT-OFF* */
972  .version = VPP_BUILD_VER,
973  .description = "Packet Memory Interface (experimetal)",
974 };
975 /* *INDENT-ON* */
976 
977 /*
978  * fd.io coding-style-patch-verification: ON
979  *
980  * Local Variables:
981  * eval: (c-set-style "gnu")
982  * End:
983  */
#define DBG_UNIX_LOG(...)
memif_if_t * interfaces
Definition: private.h:221
vlib_physmem_region_index_t physmem_region
Definition: buffer.h:420
vmrglw vmrglh hi
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define hash_set(h, key, value)
Definition: hash.h:254
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: interface.c:554
static f64 vlib_process_wait_for_event_or_clock(vlib_main_t *vm, f64 dt)
Suspend a cooperative multi-tasking thread Waits for an event, or for the indicated number of seconds...
Definition: node_funcs.h:699
#define hash_unset(h, key)
Definition: hash.h:260
u8 * secret
Definition: private.h:151
static uword * vlib_process_wait_for_event(vlib_main_t *vm)
Definition: node_funcs.h:619
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:321
clib_socket_t ** pending_clients
Definition: private.h:70
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
static uword clib_socket_is_connected(clib_socket_t *sock)
Definition: socket.h:112
struct memif_if_t::@424 run
memif_socket_file_t * socket_files
Definition: private.h:224
memif_log2_ring_size_t log2_ring_size
Definition: private.h:164
#define NULL
Definition: clib.h:55
uword mhash_unset(mhash_t *h, void *key, uword *old_value)
Definition: mhash.c:353
void * addr
Pointer to allocated memory, set on successful allocation.
Definition: mem.h:346
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:227
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:571
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u32 file_descriptor
Definition: file.h:54
static clib_error_t * memif_init(vlib_main_t *vm)
Definition: memif.c:945
int i
clib_error_t * clib_socket_init(clib_socket_t *s)
Definition: socket.c:376
clib_error_t * memif_msg_send_disconnect(memif_if_t *mif, clib_error_t *err)
Definition: socket.c:190
memif_interface_mode_t mode
Definition: private.h:249
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
#define VNET_HW_INTERFACE_FLAG_LINK_UP
Definition: interface.h:390
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:445
u32 * buffers
Definition: private.h:107
u8 num_m2s_rings
Definition: private.h:166
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:227
#define memif_file_del_by_index(a)
Definition: private.h:62
uint32_t length
Definition: memif.h:152
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
clib_file_function_t * read_function
Definition: file.h:67
static void clib_spinlock_free(clib_spinlock_t *p)
Definition: lock.h:64
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define CLIB_SOCKET_F_IS_SERVER
Definition: socket.h:58
vnet_hw_interface_rx_mode
Definition: interface.h:51
#define static_always_inline
Definition: clib.h:93
clib_error_t * clib_mem_vm_ext_alloc(clib_mem_vm_alloc_t *a)
Definition: mem.c:80
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:440
u8 * remote_name
Definition: private.h:159
static int memif_add_socket_file(u32 sock_id, u8 *socket_filename)
Definition: memif.c:527
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:111
static uword vlib_process_get_events(vlib_main_t *vm, uword **data_vector)
Return the first event type which has occurred and a vector of per-event data of that type...
Definition: node_funcs.h:542
clib_error_t * memif_plugin_api_hookup(vlib_main_t *vm)
Definition: memif_api.c:408
uword socket_file_index
Definition: private.h:149
static char * vlib_unix_get_runtime_dir(void)
Definition: unix.h:138
uint32_t cookie
Definition: memif.h:166
char * name
Name for memory allocation, set by caller.
Definition: mem.h:343
clib_error_t * memif_init_regions_and_queues(memif_if_t *mif)
Definition: memif.c:281
static_always_inline void vnet_device_input_set_interrupt_pending(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.h:136
u16 buffer_size
Definition: private.h:167
memif_log2_ring_size_t log2_ring_size
Definition: private.h:250
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
vnet_device_class_t memif_device_class
u32 per_interface_next_index
Definition: private.h:145
static int memif_delete_socket_file(u32 sock_id)
Definition: memif.c:562
#define clib_error_return(e, args...)
Definition: error.h:99
memif_region_offset_t offset
Definition: private.h:103
unsigned long u64
Definition: types.h:89
uword size
Allocation size, set by caller.
Definition: mem.h:344
#define memif_file_add(a, b)
Definition: private.h:58
u32 vnet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, u32 hw_class_index, u32 hw_instance)
Definition: interface.c:705
static u32 memif_eth_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hi, u32 flags)
Definition: memif.c:46
mhash_t dev_instance_by_id
Definition: private.h:75
#define fl(x, y)
static vlib_physmem_region_t * vlib_physmem_get_region(vlib_main_t *vm, u8 index)
Definition: physmem_funcs.h:44
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
#define CLIB_MEM_VM_F_SHARED
Definition: mem.h:326
memif_region_index_t region
Definition: memif.h:151
u16 last_head
Definition: private.h:105
#define hash_get(h, key)
Definition: hash.h:248
memif_copy_op_t * copy_ops
Definition: private.h:205
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:461
u32 size
vlib_node_registration_t memif_input_node
(constructor) VLIB_REGISTER_NODE (memif_input_node)
Definition: node.c:915
memif_desc_t desc[0]
Definition: memif.h:173
static void vlib_process_signal_event(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
Definition: node_funcs.h:952
int memif_delete_if(vlib_main_t *vm, memif_if_t *mif)
Definition: memif.c:629
int fd
File descriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set.
Definition: mem.h:347
uword dev_instance
Definition: private.h:142
static clib_error_t * clib_socket_close(clib_socket_t *sock)
Definition: socket.h:175
clib_spinlock_t lockp
Definition: private.h:137
#define clib_error_return_unix(e, args...)
Definition: error.h:102
int vnet_hw_interface_get_rx_mode(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, vnet_hw_interface_rx_mode *mode)
Definition: devices.c:312
#define hash_free(h)
Definition: hash.h:309
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
Definition: buffer.h:446
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:273
#define vec_dup(V)
Return copy of vector (no header, no alignment)
Definition: vec.h:370
vlib_buffer_pool_t * buffer_pools
Definition: buffer.h:440
memif_interface_id_t id
Definition: private.h:244
int memif_create_if(vlib_main_t *vm, memif_create_if_args_t *args)
Definition: memif.c:706
uword int_clib_file_index
Definition: private.h:111
#define VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE
Definition: interface.h:428
u8 is_external
Definition: private.h:88
VNET_HW_INTERFACE_CLASS(memif_ip_hw_if_class, static)
static uword mhash_set(mhash_t *h, void *key, uword new_value, uword *old_value)
Definition: mhash.h:117
memif_queue_t * tx_queues
Definition: private.h:156
clib_error_t * memif_connect(memif_if_t *mif)
Definition: memif.c:181
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:143
void mhash_init(mhash_t *h, uword n_value_bytes, uword n_key_bytes)
Definition: mhash.c:168
vlib_main_t * vm
Definition: buffer.c:294
u8 * local_disc_string
Definition: private.h:179
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:336
#define clib_warning(format, args...)
Definition: error.h:59
int memif_socket_filename_add_del(u8 is_add, u32 sock_id, u8 *sock_filename)
Definition: memif.c:590
#define DBG(...)
uint32_t memif_interface_id_t
Definition: memif.h:64
void memif_disconnect(memif_if_t *mif, clib_error_t *err)
Definition: memif.c:69
u32 flags
vm allocation flags: CLIB_MEM_VM_F_SHARED: request shared memory, file descriptor will be provided ...
Definition: mem.h:332
memif_ring_type_t type
Definition: private.h:115
memif_region_t * regions
Definition: private.h:153
#define hash_create(elts, value_bytes)
Definition: hash.h:681
#define ASSERT(truth)
unsigned int u32
Definition: types.h:88
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
Definition: devices.c:138
static uword * mhash_get(mhash_t *h, const void *key)
Definition: mhash.h:110
static void memif_queue_intfd_close(memif_queue_t *mq)
Definition: memif.c:53
static void mhash_free(mhash_t *h)
Definition: mhash.h:149
static clib_error_t * memif_int_fd_read_ready(clib_file_t *uf)
Definition: memif.c:156
u32 flags
Definition: private.h:138
memif_ring_t * ring
Definition: private.h:100
static void clib_mem_free(void *p)
Definition: mem.h:179
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:273
vlib_buffer_t buffer_template
Definition: private.h:209
#define clib_error_report(e)
Definition: error.h:113
u32 hw_if_index
Definition: private.h:140
struct _socket_t clib_socket_t
clib_error_t * memif_slave_conn_fd_write_ready(clib_file_t *uf)
Definition: socket.c:583
#define clib_fifo_free(f)
Definition: fifo.h:257
#define MEMIF_RING_FLAG_MASK_INT
Definition: memif.h:168
static void * clib_mem_alloc(uword size)
Definition: mem.h:112
clib_error_t * memif_slave_conn_fd_error(clib_file_t *uf)
Definition: socket.c:591
#define CLIB_SOCKET_F_SEQPACKET
Definition: socket.h:63
clib_error_t * memif_slave_conn_fd_read_ready(clib_file_t *uf)
Definition: socket.c:540
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
u64 int_count
Definition: private.h:112
void vnet_delete_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:917
u64 uword
Definition: types.h:112
memif_region_offset_t offset
Definition: memif.h:153
uword * dev_instance_by_fd
Definition: private.h:78
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:159
u8 num_s2m_rings
Definition: private.h:165
VLIB_PLUGIN_REGISTER()
struct memif_if_t::@423 cfg
Definition: defs.h:47
clib_error_t * vlib_unix_recursive_mkdir(char *path)
Definition: util.c:103
unsigned short u16
Definition: types.h:57
static_always_inline memif_ring_t * memif_get_ring(memif_if_t *mif, memif_ring_type_t type, u16 ring_num)
Definition: memif.c:267
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
double f64
Definition: types.h:142
unsigned char u8
Definition: types.h:56
#define MEMIF_DEFAULT_SOCKET_FILENAME
Definition: private.h:20
#define CLIB_SOCKET_F_PASSCRED
Definition: socket.h:64
a point 2 point interface
Definition: interface.h:285
void memif_socket_close(clib_socket_t **sock)
Definition: socket.c:43
#define clib_error_free(e)
Definition: error.h:86
void * shm
Definition: private.h:85
u8 * remote_if_name
Definition: private.h:160
memif_interface_id_t id
Definition: private.h:139
memif_log2_ring_size_t log2_ring_size
Definition: private.h:101
int vnet_hw_interface_unassign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.c:187
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
Definition: buffer_funcs.h:915
#define vnet_buffer(b)
Definition: buffer.h:372
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
uint16_t flags
Definition: memif.h:167
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
memif_per_thread_data_t * per_thread_data
Definition: private.h:228
static vlib_node_registration_t memif_process_node
(constructor) VLIB_REGISTER_NODE (memif_process_node)
Definition: memif.c:518
vlib_buffer_main_t buffer_main
Definition: buffer.c:52
u8 * remote_disc_string
Definition: private.h:180
memif_ring_type_t
Definition: memif.h:47
#define vec_foreach(var, vec)
Vector iterator.
volatile uint16_t head
Definition: memif.h:169
uword private_data
Definition: file.h:64
clib_error_t * vnet_sw_interface_set_flags(vnet_main_t *vnm, u32 sw_if_index, u32 flags)
Definition: interface.c:562
Definition: file.h:51
clib_socket_t * sock
Definition: private.h:148
memif_queue_t * rx_queues
Definition: private.h:155
int vnet_hw_interface_set_rx_mode(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, vnet_hw_interface_rx_mode mode)
Definition: devices.c:252
clib_error_t * memif_conn_fd_accept_ready(clib_file_t *uf)
Definition: socket.c:643
uword * socket_file_index_by_sock_id
Definition: private.h:225
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer_funcs.h:519
u32 flags
Definition: vhost-user.h:77
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
#define CLIB_SOCKET_F_ALLOW_GROUP_WRITE
Definition: socket.h:62
memif_msg_fifo_elt_t * msg_queue
Definition: private.h:150
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:111
memif_main_t memif_main
Definition: memif.c:43
memif_region_index_t region
Definition: private.h:102
clib_socket_t * sock
Definition: private.h:69
#define MEMIF_COOKIE
Definition: memif.h:25
u32 sw_if_index
Definition: private.h:141
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: devices.h:79
volatile uint16_t tail
Definition: memif.h:171
memif_interface_mode_t mode
Definition: private.h:143
static uword memif_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: memif.c:420
u8 * format_memif_device_name(u8 *s, va_list *args)
Definition: device.c:51
memif_region_size_t region_size
Definition: private.h:86
static uword pool_elts(void *v)
Number of active elements in a pool.
Definition: pool.h:128