21 #include <sys/ioctl.h> 22 #include <sys/socket.h> 25 #include <sys/types.h> 27 #include <netinet/in.h> 30 #include <linux/if_arp.h> 31 #include <linux/if_tun.h> 86 ssize_t map_sz = (vui->
regions[
i].memory_size +
88 page_sz - 1) & ~(page_sz - 1);
94 vu_log_debug (vui,
"unmap memory region %d addr 0x%lx len 0x%lx " 102 vu_log_err (vui,
"failed to unmap memory region (errno %d)",
124 u32 thread_index = 0;
142 if (thread_index == 0)
145 for (thread_index = 0;
179 vu_log_warn (vui,
"unable to set rx mode for interface %d, " 187 int i, found[2] = { };
193 return found[0] && found[1];
204 is_ready ?
"ready" :
"down");
232 __attribute__ ((unused))
int n;
255 __attribute__ ((unused))
int n;
298 if (qid == 0 || qid == 1)
321 if (vring->
errfd != -1)
323 close (vring->
errfd);
361 int fd, number_of_fds = 0;
363 vhost_user_msg_t msg;
368 struct cmsghdr *cmsg;
385 iov[0].iov_base = (
void *) &msg;
390 mh.msg_control = control;
391 mh.msg_controllen =
sizeof (control);
399 vu_log_debug (vui,
"recvmsg returned error %d %s", errno,
404 vu_log_debug (vui,
"n (%d) != VHOST_USER_MSG_HDR_SZ (%d)",
410 if (mh.msg_flags & MSG_CTRUNC)
416 cmsg = CMSG_FIRSTHDR (&mh);
418 if (cmsg && (cmsg->cmsg_len > 0) && (cmsg->cmsg_level == SOL_SOCKET) &&
419 (cmsg->cmsg_type == SCM_RIGHTS) &&
420 (cmsg->cmsg_len - CMSG_LEN (0) <=
421 VHOST_MEMORY_MAX_NREGIONS *
sizeof (
int)))
423 number_of_fds = (cmsg->cmsg_len - CMSG_LEN (0)) /
sizeof (
int);
428 if ((msg.flags & 7) != 1)
430 vu_log_debug (vui,
"malformed message received. closing socket");
444 else if (rv != msg.size)
446 vu_log_debug (vui,
"message too short (read %dB should be %dB)", rv,
472 msg.size =
sizeof (msg.u64);
473 vu_log_debug (vui,
"if %d msg VHOST_USER_GET_FEATURES - reply " 485 vu_log_debug (vui,
"if %d msg VHOST_USER_SET_FEATURES features " 517 vu_log_debug (vui,
"if %d msg VHOST_USER_SET_MEM_TABLE nregions %d",
520 if ((msg.memory.nregions < 1) ||
521 (msg.memory.nregions > VHOST_MEMORY_MAX_NREGIONS))
523 vu_log_debug (vui,
"number of mem regions must be between 1 and %i",
524 VHOST_MEMORY_MAX_NREGIONS);
528 if (msg.memory.nregions != number_of_fds)
536 for (i = 0; i < msg.memory.nregions; i++)
541 ssize_t map_sz = (msg.memory.regions[
i].memory_size +
542 msg.memory.regions[
i].mmap_offset +
543 page_sz - 1) & ~(page_sz - 1);
545 region_mmap_addr[
i] = mmap (0, map_sz, PROT_READ | PROT_WRITE,
546 MAP_SHARED, fds[i], 0);
547 if (region_mmap_addr[i] == MAP_FAILED)
549 vu_log_err (vui,
"failed to map memory. errno is %d", errno);
550 for (j = 0; j <
i; j++)
551 munmap (region_mmap_addr[j], map_sz);
554 vu_log_debug (vui,
"map memory region %d addr 0 len 0x%lx fd %d " 555 "mapped 0x%lx page_sz 0x%x", i, map_sz, fds[i],
556 region_mmap_addr[i], page_sz);
561 for (i = 0; i < msg.memory.nregions; i++)
564 sizeof (vhost_user_memory_region_t));
598 vu_log_debug (vui,
"if %d msg VHOST_USER_SET_VRING_NUM idx %d num %d",
601 if ((msg.state.num > 32768) ||
602 (msg.state.num == 0) ||
603 ((msg.state.num - 1) & msg.state.num))
609 vu_log_debug (vui,
"if %d msg VHOST_USER_SET_VRING_ADDR idx %d",
612 if (msg.state.index >= VHOST_VRING_MAX_N)
614 vu_log_debug (vui,
"invalid vring index VHOST_USER_SET_VRING_ADDR:" 615 " %d >= %d", msg.state.index, VHOST_VRING_MAX_N);
619 if (msg.size < sizeof (msg.addr))
621 vu_log_debug (vui,
"vhost message is too short (%d < %d)",
622 msg.size, sizeof (msg.addr));
630 if ((desc == NULL) || (used == NULL) || (avail == NULL))
632 vu_log_debug (vui,
"failed to map user memory for hw_if_index %d",
679 vu_log_debug (vui,
"if %d msg VHOST_USER_SET_VRING_CALL %d",
682 q = (
u8) (msg.u64 & 0xFF);
695 if (number_of_fds != 1)
702 template.file_descriptor = fds[0];
703 template.private_data =
712 vu_log_debug (vui,
"if %d msg VHOST_USER_SET_VRING_KICK %d",
715 q = (
u8) (msg.u64 & 0xFF);
727 if (number_of_fds != 1)
734 template.file_descriptor = fds[0];
735 template.private_data =
751 vu_log_debug (vui,
"if %d msg VHOST_USER_SET_VRING_ERR %d",
754 q = (
u8) (msg.u64 & 0xFF);
761 if (number_of_fds != 1)
772 "if %d msg VHOST_USER_SET_VRING_BASE idx %d num 0x%x",
790 msg.state.num & 0x7fff;
793 ! !(msg.state.num & (1 << 15));
816 if (msg.state.index >= VHOST_VRING_MAX_N)
818 vu_log_debug (vui,
"invalid vring index VHOST_USER_GET_VRING_BASE:" 819 " %d >= %d", msg.state.index, VHOST_VRING_MAX_N);
840 msg.size =
sizeof (msg.state);
849 "if %d msg VHOST_USER_GET_VRING_BASE idx %d num 0x%x",
869 if (msg.size != sizeof (msg.log))
871 vu_log_debug (vui,
"invalid msg size for VHOST_USER_SET_LOG_BASE:" 872 " %d instead of %d", msg.size, sizeof (msg.log));
878 vu_log_debug (vui,
"VHOST_USER_PROTOCOL_F_LOG_SHMFD not set but " 879 "VHOST_USER_SET_LOG_BASE received");
887 (msg.log.size + msg.log.offset + page_sz - 1) & ~(page_sz - 1);
889 void *log_base_addr = mmap (0, map_sz, PROT_READ | PROT_WRITE,
892 vu_log_debug (vui,
"map log region addr 0 len 0x%lx off 0x%lx fd %d " 893 "mapped 0x%lx", map_sz, msg.log.offset, fd,
896 if (log_base_addr == MAP_FAILED)
898 vu_log_err (vui,
"failed to map memory. errno is %d", errno);
909 msg.size =
sizeof (msg.u64);
927 msg.size =
sizeof (msg.u64);
928 vu_log_debug (vui,
"if %d msg VHOST_USER_GET_PROTOCOL_FEATURES - " 940 vu_log_debug (vui,
"if %d msg VHOST_USER_SET_PROTOCOL_FEATURES " 948 msg.size =
sizeof (msg.u64);
949 vu_log_debug (vui,
"if %d msg VHOST_USER_GET_QUEUE_NUM - reply %d",
961 vu_log_debug (vui,
"if %d VHOST_USER_SET_VRING_ENABLE: %s queue %d",
962 vui->
hw_if_index, msg.state.num ?
"enable" :
"disable",
964 if (msg.state.index >= VHOST_VRING_MAX_N)
966 vu_log_debug (vui,
"invalid vring idx VHOST_USER_SET_VRING_ENABLE:" 967 " %d >= %d", msg.state.index, VHOST_VRING_MAX_N);
977 vu_log_debug (vui,
"unknown vhost-user message %d received. " 978 "closing socket", msg.request);
1010 int client_fd, client_len;
1011 struct sockaddr_un client;
1018 client_len =
sizeof (client);
1020 (
struct sockaddr *) &client,
1021 (socklen_t *) & client_len);
1028 vu_log_debug (vui,
"Close client socket for vhost interface %d, fd %d",
1033 vu_log_debug (vui,
"New client socket for vhost interface %d, fd %d",
1037 template.file_descriptor = client_fd;
1083 f64 timeout = 3153600000.0 ;
1084 uword event_type, *event_data = 0;
1087 f64 now, poll_time_remaining;
1093 poll_time_remaining =
1102 timeout = poll_time_remaining;
1123 next_timeout = timeout;
1124 for (qid = 0; qid < VHOST_VRING_MAX_N / 2; qid += 2)
1126 vhost_user_vring_t *rxvq = &vui->vrings[qid];
1127 vhost_user_vring_t *txvq = &vui->vrings[qid + 1];
1129 if (txvq->qid == -1)
1131 if (txvq->n_since_last_int)
1133 if (now >= txvq->int_deadline)
1134 vhost_user_send_call (vm, txvq);
1136 next_timeout = txvq->int_deadline - now;
1139 if (rxvq->n_since_last_int)
1141 if (now >= rxvq->int_deadline)
1142 vhost_user_send_call (vm, rxvq);
1144 next_timeout = rxvq->int_deadline - now;
1147 if ((next_timeout < timeout) && (next_timeout > 0.0))
1148 timeout = next_timeout;
1155 clib_warning (
"BUG: unhandled event type %d", event_type);
1162 timeout = 3153600000.0;
1171 .name =
"vhost-user-send-interrupt-process",
1181 struct sockaddr_un sun;
1184 f64 timeout = 3153600000.0 ;
1185 uword *event_data = 0;
1188 sun.sun_family = AF_UNIX;
1203 if (vui->unix_server_index == ~0) {
1204 if (vui->clib_file_index == ~0)
1207 ((sockfd = socket (AF_UNIX, SOCK_STREAM, 0)) < 0))
1213 if (!vui->sock_errno || (vui->sock_errno != errno))
1216 (
"Error: Could not open unix socket for %s",
1217 vui->sock_filename);
1218 vui->sock_errno = errno;
1224 strncpy (sun.sun_path, (char *) vui->sock_filename,
1225 sizeof (sun.sun_path) - 1);
1228 if (fcntl(sockfd, F_SETFL, O_NONBLOCK) < 0)
1229 clib_unix_warning (
"fcntl");
1231 if (connect (sockfd, (struct sockaddr *) &sun,
1232 sizeof (struct sockaddr_un)) == 0)
1235 if (fcntl(sockfd, F_SETFL, 0) < 0)
1236 clib_unix_warning (
"fcntl2");
1238 vui->sock_errno = 0;
1239 template.file_descriptor = sockfd;
1240 template.private_data =
1241 vui - vhost_user_main.vhost_user_interfaces;
1242 vui->clib_file_index = clib_file_add (&file_main, &template);
1249 vui->sock_errno = errno;
1256 socklen_t len = sizeof (error);
1257 int fd = UNIX_GET_FD(vui->clib_file_index);
1259 getsockopt (fd, SOL_SOCKET, SO_ERROR, &error, &len);
1263 vu_log_debug (vui,
"getsockopt returned %d", retval);
1264 vhost_user_if_disconnect (vui);
1278 .name =
"vhost-user-process",
1306 if (txvq->
qid != -1)
1312 vu_log_warn (vui,
"unable to unassign interface %d, " 1347 return VNET_API_ERROR_INVALID_SW_IF_INDEX;
1351 vu_log_debug (vui,
"Deleting vhost-user interface %s (instance %d)",
1358 if (txvq->
qid == -1)
1404 vhost_user_delete_if (vnm, vm, vui->sw_if_index);
1420 struct sockaddr_un
un = { };
1423 if ((fd = socket (AF_UNIX, SOCK_STREAM, 0)) < 0)
1424 return VNET_API_ERROR_SYSCALL_ERROR_1;
1426 un.sun_family = AF_UNIX;
1427 strncpy ((
char *) un.sun_path, (
char *) sock_filename,
1428 sizeof (un.sun_path) - 1);
1431 unlink ((
char *) sock_filename);
1433 if (bind (fd, (
struct sockaddr *) &un,
sizeof (un)) == -1)
1435 rv = VNET_API_ERROR_SYSCALL_ERROR_2;
1439 if (listen (fd, 1) == -1)
1441 rv = VNET_API_ERROR_SYSCALL_ERROR_3;
1495 const char *sock_filename,
1506 if (server_sock_fd != -1)
1510 template.file_descriptor = server_sock_fd;
1543 (FEATURE_VIRTIO_NET_F_HOST_GUEST_TSO_FEATURE_BITS)))
1572 const char *sock_filename,
1576 u8 renumber,
u32 custom_dev_instance,
u8 * hwaddr,
1577 u8 enable_gso,
u8 enable_packed)
1582 int server_sock_fd = -1;
1586 if (sock_filename == NULL || !(strlen (sock_filename) > 0))
1588 return VNET_API_ERROR_INVALID_ARGUMENT;
1599 return VNET_API_ERROR_IF_ALREADY_EXISTS;
1618 feature_mask, &sw_if_idx, enable_gso, enable_packed);
1626 *sw_if_index = sw_if_idx;
1636 const char *sock_filename,
1639 u64 feature_mask,
u8 renumber,
u32 custom_dev_instance,
1640 u8 enable_gso,
u8 enable_packed)
1645 int server_sock_fd = -1;
1654 return VNET_API_ERROR_INVALID_SW_IF_INDEX;
1656 if (sock_filename == NULL || !(strlen (sock_filename) > 0))
1657 return VNET_API_ERROR_INVALID_ARGUMENT;
1666 if (if_index && (*if_index != vui->
if_index))
1667 return VNET_API_ERROR_IF_ALREADY_EXISTS;
1672 &server_sock_fd)) != 0)
1677 sock_filename, feature_mask, &sw_if_idx, enable_gso,
1695 u8 *sock_filename = NULL;
1698 u64 feature_mask = (
u64) ~ (0ULL);
1700 u32 custom_dev_instance = ~0;
1704 u8 enable_gso = 0, enable_packed = 0;
1716 if (
unformat (line_input,
"socket %s", &sock_filename))
1718 else if (
unformat (line_input,
"server"))
1720 else if (
unformat (line_input,
"gso"))
1722 else if (
unformat (line_input,
"packed"))
1724 else if (
unformat (line_input,
"feature-mask 0x%llx", &feature_mask))
1730 else if (
unformat (line_input,
"renumber %d", &custom_dev_instance))
1746 is_server, &sw_if_index, feature_mask,
1747 renumber, custom_dev_instance, hw,
1748 enable_gso, enable_packed)))
1780 if (
unformat (line_input,
"sw_if_index %d", &sw_if_index))
1820 u32 *hw_if_indices = 0;
1831 for (i = 0; i <
vec_len (hw_if_indices); i++)
1852 *out_vuids = r_vuids;
1860 char *fmt = va_arg (*args,
char *);
1863 int idx = va_arg (*args,
int);
1864 u32 *mem_hint = va_arg (*args,
u32 *);
1866 s =
format (s, fmt, idx, desc_table[idx].
addr, desc_table[idx].
len,
1867 desc_table[idx].
flags, desc_table[idx].next,
1876 char *fmt = va_arg (*args,
char *);
1878 int q = va_arg (*args,
int);
1891 vlib_cli_output (vm,
" kickfd %d callfd %d errfd %d\n", kickfd, callfd,
1897 int show_descr,
int show_verbose)
1907 " avail.flags %x avail.idx %d used.flags %x used.idx %d\n",
1916 " slot addr len flags next " 1919 " ===== ================== ===== ====== ===== " 1920 "==================\n");
1925 " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n", vui,
1926 desc_table, j, &mem_hint);
1933 for (idx = 0; idx <
clib_min (20, n_entries); idx++)
1937 "> %-4u 0x%016lx %-5u 0x%04x %-5u 0x%016lx\n", vui,
1938 desc_table, idx, &mem_hint);
1940 if (n_entries >= 20)
1952 char *fmt = va_arg (*args,
char *);
1954 vring_packed_desc_t *desc_table = va_arg (*args, vring_packed_desc_t *);
1955 int idx = va_arg (*args,
int);
1956 u32 *mem_hint = va_arg (*args,
u32 *);
1958 s =
format (s, fmt, idx, desc_table[idx].
addr, desc_table[idx].
len,
1959 desc_table[idx].
flags, desc_table[idx].
id,
1968 char *fmt = va_arg (*args,
char *);
1970 int q = va_arg (*args,
int);
1983 int show_descr,
int show_verbose)
1989 vring_packed_desc_t *desc_table;
1993 " avail_event.flags %x avail_event.off_wrap %u " 1994 "used_event.flags %x used_event.off_wrap %u\n" 1995 " avail wrap counter %u, used wrap counter %u\n",
2004 " slot addr len flags id " 2007 " ===== ================== ===== ====== ===== " 2008 "==================\n");
2013 " %-5u 0x%016lx %-5u 0x%04x %-5u 0x%016lx\n", vui,
2014 desc_table, j, &mem_hint);
2017 n_entries = desc_table[j].len >> 4;
2021 for (idx = 0; idx <
clib_min (20, n_entries); idx++)
2025 "> %-4u 0x%016lx %-5u 0x%04x %-5u 0x%016lx\n", vui,
2026 desc_table, idx, &mem_hint);
2028 if (n_entries >= 20)
2046 u32 hw_if_index, *hw_if_indices = 0;
2052 int show_verbose = 0;
2058 struct feat_struct *feat_entry;
2060 static struct feat_struct feat_array[] = {
2061 #define _(s,b) { .str = #s, .bit = b, }, 2067 #define foreach_protocol_feature \ 2068 _(VHOST_USER_PROTOCOL_F_MQ) \ 2069 _(VHOST_USER_PROTOCOL_F_LOG_SHMFD) 2071 static struct feat_struct proto_feat_array[] = {
2072 #define _(s) { .str = #s, .bit = s}, 2090 vec_add1 (hw_if_indices, hw_if_index);
2094 else if (
unformat (input,
"verbose"))
2103 if (
vec_len (hw_if_indices) == 0)
2112 vlib_cli_output (vm,
" Number of rx virtqueues in interrupt mode: %d",
2116 for (i = 0; i <
vec_len (hw_if_indices); i++)
2129 " features mask (0x%llx): \n" 2130 " features (0x%llx): \n",
2134 feat_entry = (
struct feat_struct *) &feat_array;
2135 while (feat_entry->str)
2137 if (vui->
features & (1ULL << feat_entry->bit))
2145 feat_entry = (
struct feat_struct *) &proto_feat_array;
2146 while (feat_entry->str)
2170 if (txvq->
qid == -1)
2198 " region fd guest_phys_addr memory_size userspace_addr mmap_offset mmap_addr\n");
2200 " ====== ===== ================== ================== ================== ================== ==================\n");
2202 for (j = 0; j < vui->
nregions; j++)
2205 " %d %-5d 0x%016lx 0x%016lx 0x%016lx 0x%016lx 0x%016lx\n",
2207 vui->
regions[j].guest_phys_addr,
2209 vui->
regions[j].userspace_addr,
2219 (q & 1) ?
"RX" :
"TX",
2223 " qsz %d last_avail_idx %d last_used_idx %d\n",
2301 .path =
"create vhost-user",
2302 .short_help =
"create vhost-user socket <socket-filename> [server] " 2303 "[feature-mask <hex>] [hwaddr <mac-addr>] [renumber <dev_instance>] [gso] " 2324 .path =
"delete vhost-user",
2325 .short_help =
"delete vhost-user {<interface> | sw_if_index <sw_idx>}",
2464 .path =
"show vhost-user",
2465 .short_help =
"show vhost-user [<interface> [<interface> [..]]] " 2466 "[[descriptors] [verbose]]",
2483 else if (
unformat (input,
"dont-dump-memory"))
unformat_function_t unformat_vnet_hw_interface
vlib_log_class_t vlib_log_register_class(char *class, char *subclass)
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
#define VRING_EVENT_F_DISABLE
#define vec_foreach_index(var, v)
Iterate over vector indices.
static void clib_file_del(clib_file_main_t *um, clib_file_t *f)
format_function_t format_vnet_hw_if_index_name
#define vu_log_warn(dev, f,...)
static uword random_default_seed(void)
Default random seed (unix/linux user-mode)
#define VHOST_USER_PROTOCOL_F_LOG_SHMFD
static f64 vlib_process_wait_for_event_or_clock(vlib_main_t *vm, f64 dt)
Suspend a cooperative multi-tasking thread Waits for an event, or for the indicated number of seconds...
VNET_HW_INTERFACE_CLASS(vhost_interface_class, static)
vhost_cpu_t * cpus
Per-CPU data for vhost-user.
vl_api_wireguard_peer_flags_t flags
vring_desc_event_t * used_event
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
static void stop_timer(wg_peer_t *peer, u32 timer_id)
vnet_main_t * vnet_get_main(void)
u64 region_guest_addr_hi[VHOST_MEMORY_MAX_NREGIONS]
static clib_error_t * vhost_user_socksvr_accept_ready(clib_file_t *uf)
#define VHOST_VRING_MAX_N
#define clib_memcpy_fast(a, b, c)
uword mhash_unset(mhash_t *h, void *key, uword *old_value)
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
static f64 vlib_time_now(vlib_main_t *vm)
vring_packed_desc_t * packed_desc
static_always_inline void vhost_user_update_iface_state(vhost_user_intf_t *vui)
#define VLIB_BUFFER_PRE_DATA_SIZE
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
vnet_device_class_t vhost_user_device_class
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
int vnet_interface_name_renumber(u32 sw_if_index, u32 new_show_dev_instance)
static clib_error_t * vhost_user_callfd_read_ready(clib_file_t *uf)
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
#define VHOST_USER_EVENT_START_TIMER
static_always_inline void vhost_user_vring_init(vhost_user_intf_t *vui, u32 qid)
unformat_function_t unformat_vnet_sw_interface
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
format_function_t format_vnet_sw_if_index_name
#define VHOST_USER_VRING_NOFD_MASK
static_always_inline void vhost_user_rx_thread_placement(vhost_user_intf_t *vui, u32 qid)
Unassign existing interface/queue to thread mappings and re-assign new interface/queue to thread mapp...
static uword vlib_process_suspend_time_is_zero(f64 dt)
Returns TRUE if a process suspend time is less than 10us.
#define VIRTIO_FEATURE(X)
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
clib_file_function_t * read_function
#define vlib_worker_thread_barrier_sync(X)
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define clib_memcpy(d, s, n)
#define vu_log_debug(dev, f,...)
vnet_hw_interface_rx_mode
#define static_always_inline
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
#define VLIB_INIT_FUNCTION(x)
static uword vlib_process_get_events(vlib_main_t *vm, uword **data_vector)
Return the first event type which has occurred and a vector of per-event data of that type...
static clib_error_t * vhost_user_socket_read(clib_file_t *uf)
#define UNIX_GET_FD(unixfd_idx)
static void vhost_user_show_desc(vlib_main_t *vm, vhost_user_intf_t *vui, int q, int show_descr, int show_verbose)
static_always_inline void vhost_user_thread_placement(vhost_user_intf_t *vui, u32 qid)
static_always_inline void * map_guest_mem(vhost_user_intf_t *vui, uword addr, u32 *hint)
static_always_inline void vnet_device_input_set_interrupt_pending(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
vnet_hw_interface_flags_t flags
static void vhost_user_show_desc_packed(vlib_main_t *vm, vhost_user_intf_t *vui, int q, int show_descr, int show_verbose)
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
clib_file_main_t file_main
static vlib_node_registration_t vhost_user_process_node
(constructor) VLIB_REGISTER_NODE (vhost_user_process_node)
static clib_error_t * vhost_user_kickfd_read_ready(clib_file_t *uf)
static clib_error_t * vhost_user_socket_error(clib_file_t *uf)
void vhost_user_unmap_all(void)
static u8 * format_vhost_user_packed_desc(u8 *s, va_list *args)
static_always_inline void vhost_user_tx_thread_placement(vhost_user_intf_t *vui)
#define VHOST_USER_PROTOCOL_F_MQ
#define VRING_USED_F_NO_NOTIFY
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
int vhost_user_create_if(vnet_main_t *vnm, vlib_main_t *vm, const char *sock_filename, u8 is_server, u32 *sw_if_index, u64 feature_mask, u8 renumber, u32 custom_dev_instance, u8 *hwaddr, u8 enable_gso, u8 enable_packed)
static_always_inline uword vnet_get_device_input_thread_index(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
static void vhost_user_term_if(vhost_user_intf_t *vui)
Disables and reset interface structure.
static void vlib_process_signal_event(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
static_always_inline int vhost_user_intf_ready(vhost_user_intf_t *vui)
Returns whether at least one TX and one RX vring are enabled.
u32 random
Pseudo random iterator.
uword mhash_set_mem(mhash_t *h, void *key, uword *new_value, uword *old_value)
vlib_node_registration_t vhost_user_input_node
(constructor) VLIB_REGISTER_NODE (vhost_user_input_node)
#define clib_error_return_unix(e, args...)
int vnet_hw_interface_get_rx_mode(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, vnet_hw_interface_rx_mode *mode)
#define pool_put(P, E)
Free an object E in pool P.
int vhost_user_delete_if(vnet_main_t *vnm, vlib_main_t *vm, u32 sw_if_index)
format_function_t format_vnet_hw_interface_rx_mode
static u8 * format_vhost_user_desc(u8 *s, va_list *args)
#define VLIB_CONFIG_FUNCTION(x, n,...)
vhost_user_main_t vhost_user_main
#define VHOST_USER_MSG_HDR_SZ
u32 region_mmap_fd[VHOST_MEMORY_MAX_NREGIONS]
int vhost_user_dump_ifs(vnet_main_t *vnm, vlib_main_t *vm, vhost_user_intf_details_t **out_vuids)
vhost_user_memory_region_t regions[VHOST_MEMORY_MAX_NREGIONS]
vlib_log_class_t log_default
vl_api_tunnel_mode_t mode
static_always_inline void vhost_user_update_gso_interface_count(vhost_user_intf_t *vui, u8 add)
#define VHOST_VRING_IDX_RX(qid)
u32 * show_dev_instance_by_real_dev_instance
static_always_inline void vhost_user_vring_close(vhost_user_intf_t *vui, u32 qid)
vhost_user_intf_t * vhost_user_interfaces
#define VLIB_REGISTER_NODE(x,...)
static void mhash_init_c_string(mhash_t *h, uword n_value_bytes)
sll srl srl sll sra u16x4 i
#define vec_free(V)
Free vector's memory (no header).
static void unmap_all_mem_regions(vhost_user_intf_t *vui)
static clib_error_t * vhost_user_config(vlib_main_t *vm, unformat_input_t *input)
#define VLIB_MAIN_LOOP_EXIT_FUNCTION(x)
static_always_inline u64 vhost_user_is_packed_ring_supported(vhost_user_intf_t *vui)
#define VHOST_USER_EVENT_STOP_TIMER
static vnet_hw_interface_t * vnet_get_sup_hw_interface_api_visible_or_null(vnet_main_t *vnm, u32 sw_if_index)
#define clib_warning(format, args...)
static uword vhost_user_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
clib_error_t * vhost_user_delete_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
#define foreach_protocol_feature
int vhost_user_modify_if(vnet_main_t *vnm, vlib_main_t *vm, const char *sock_filename, u8 is_server, u32 sw_if_index, u64 feature_mask, u8 renumber, u32 custom_dev_instance, u8 enable_gso, u8 enable_packed)
static int vhost_user_init_server_sock(const char *sock_filename, int *sock_fd)
Open server unix socket on specified sock_filename.
#define VLIB_CLI_COMMAND(x,...)
uword unformat_ethernet_address(unformat_input_t *input, va_list *args)
clib_error_t * show_vhost_user_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
#define VHOST_VRING_F_LOG
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
static uword * mhash_get(mhash_t *h, const void *key)
#define vu_log_err(dev, f,...)
static uword clib_file_add(clib_file_main_t *um, clib_file_t *template)
static_always_inline void vhost_user_if_disconnect(vhost_user_intf_t *vui)
static void clib_mem_free(void *p)
volatile u32 * vring_locks[VHOST_VRING_MAX_N]
vring_desc_event_t * avail_event
mhash_t if_index_by_sock_name
static clib_error_t * vhost_user_exit(vlib_main_t *vm)
#define clib_error_report(e)
vlib_node_registration_t vhost_user_send_interrupt_node
(constructor) VLIB_REGISTER_NODE (vhost_user_send_interrupt_node)
static uword pointer_to_uword(const void *p)
static void vhost_user_create_ethernet(vnet_main_t *vnm, vlib_main_t *vm, vhost_user_intf_t *vui, u8 *hwaddress)
Create ethernet interface for vhost user interface.
static u8 * format_vhost_user_vring_packed(u8 *s, va_list *args)
static vlib_main_t * vlib_get_main(void)
struct _vlib_node_registration vlib_node_registration_t
void * region_mmap_addr[VHOST_MEMORY_MAX_NREGIONS]
static void vhost_user_vui_init(vnet_main_t *vnm, vhost_user_intf_t *vui, int server_sock_fd, const char *sock_filename, u64 feature_mask, u32 *sw_if_index, u8 enable_gso, u8 enable_packed)
errno_t memcpy_s(void *__restrict__ dest, rsize_t dmax, const void *__restrict__ src, rsize_t n)
copy src to dest, at most n bytes, up to dmax
static long get_huge_page_size(int fd)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
static u8 * format_vhost_user_vring(u8 *s, va_list *args)
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, const u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
vhost_user_vring_t vrings[VHOST_VRING_MAX_N]
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
static void vhost_user_set_interrupt_pending(vhost_user_intf_t *vui, u32 ifq)
#define VHOST_MEMORY_MAX_NREGIONS
clib_error_t * vhost_user_connect_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
void vnet_sw_interface_set_mtu(vnet_main_t *vnm, u32 sw_if_index, u32 mtu)
int vnet_hw_interface_unassign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
static void * clib_mem_alloc_aligned(uword size, uword align)
static u32 random_u32(u32 *seed)
32-bit random number generator
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
static vlib_thread_main_t * vlib_get_thread_main()
u64 region_guest_addr_lo[VHOST_MEMORY_MAX_NREGIONS]
#define vec_foreach(var, vec)
Vector iterator.
int vnet_hw_interface_set_rx_mode(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, vnet_hw_interface_rx_mode mode)
static clib_error_t * vhost_user_init(vlib_main_t *vm)
static_always_inline void * map_user_mem(vhost_user_intf_t *vui, uword addr)
static void vhost_user_show_fds(vlib_main_t *vm, vhost_user_intf_t *vui, int q)
#define CLIB_CACHE_LINE_BYTES
static uword vhost_user_send_interrupt_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
#define FEATURE_VIRTIO_NET_F_HOST_GUEST_TSO_FEATURE_BITS
vl_api_address_union_t un
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
vl_api_interface_index_t sw_if_index
#define VRING_DESC_F_INDIRECT
int dont_dump_vhost_user_memory
#define VRING_DESC_F_AVAIL