FD.io VPP  v21.06-3-gbb25fbf28
Vector Packet Processing
vfio.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <unistd.h>
17 #include <sys/types.h>
18 #include <sys/stat.h>
19 #include <fcntl.h>
20 #include <linux/vfio.h>
21 #include <sys/ioctl.h>
22 
23 #include <vppinfra/linux/sysfs.h>
24 
25 #include <vlib/vlib.h>
26 #include <vlib/unix/unix.h>
27 #include <vlib/pci/pci.h>
28 #include <vlib/linux/vfio.h>
29 #include <vlib/physmem.h>
30 
31 #ifndef VFIO_NOIOMMU_IOMMU
32 #define VFIO_NOIOMMU_IOMMU 8
33 #endif
34 
36 
39 {
42  struct vfio_iommu_type1_dma_map dm = { 0 };
43  uword log2_page_size = vpm->pmalloc_main->def_log2_page_sz;
44  uword physmem_start = pointer_to_uword (vpm->pmalloc_main->base);
45 
46  if (lvm->container_fd == -1)
47  return clib_error_return (0, "No cointainer fd");
48 
49  u32 page_index = vlib_physmem_get_page_index (vm, addr);
50 
51  if (clib_bitmap_get (lvm->physmem_pages_mapped, page_index))
52  {
53  vlib_log_debug (lvm->log_default, "map DMA va:%p page:%u already "
54  "mapped", addr, page_index);
55  return 0;
56  }
57 
58  dm.argsz = sizeof (struct vfio_iommu_type1_dma_map);
59  dm.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
60  dm.vaddr = physmem_start + (page_index << log2_page_size);
61  dm.size = 1ULL << log2_page_size;
62  dm.iova = dm.vaddr;
63  vlib_log_debug (lvm->log_default, "map DMA page:%u va:0x%lx iova:%lx "
64  "size:0x%lx", page_index, dm.vaddr, dm.iova, dm.size);
65 
66  if (ioctl (lvm->container_fd, VFIO_IOMMU_MAP_DMA, &dm) == -1)
67  {
68  vlib_log_err (lvm->log_default, "map DMA page:%u va:0x%lx iova:%lx "
69  "size:0x%lx failed, error %s (errno %d)", page_index,
70  dm.vaddr, dm.iova, dm.size, strerror (errno), errno);
71  return clib_error_return_unix (0, "physmem DMA map failed");
72  }
73 
75  page_index, 1);
76  return 0;
77 }
78 
81 {
83  uword *p;
84 
85  p = hash_get (lvm->iommu_pool_index_by_group, group);
86 
87  return p ? pool_elt_at_index (lvm->iommu_groups, p[0]) : 0;
88 }
89 
90 static clib_error_t *
91 open_vfio_iommu_group (int group, int is_noiommu)
92 {
95  clib_error_t *err = 0;
96  struct vfio_group_status group_status;
97  u8 *s = 0;
98  int fd;
99 
100  if (lvm->container_fd == -1)
101  {
102  if ((fd = open ("/dev/vfio/vfio", O_RDWR)) == -1)
103  return clib_error_return_unix (0, "failed to open VFIO container");
104 
105  if (ioctl (fd, VFIO_GET_API_VERSION) != VFIO_API_VERSION)
106  {
107  close (fd);
108  return clib_error_return_unix (0, "incompatible VFIO version");
109  }
110 
111  lvm->iommu_pool_index_by_group = hash_create (0, sizeof (uword));
112  lvm->container_fd = fd;
113  }
114 
115  g = get_vfio_iommu_group (group);
116  if (g)
117  {
118  g->refcnt++;
119  return 0;
120  }
121  s = format (s, "/dev/vfio/%s%u%c", is_noiommu ? "noiommu-" : "", group, 0);
122  fd = open ((char *) s, O_RDWR);
123  if (fd < 0)
124  return clib_error_return_unix (0, "open '%s'", s);
125 
126  group_status.argsz = sizeof (group_status);
127  if (ioctl (fd, VFIO_GROUP_GET_STATUS, &group_status) < 0)
128  {
129  err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_GET_STATUS) '%s'",
130  s);
131  goto error;
132  }
133 
134  if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE))
135  {
136  err = clib_error_return (0, "iommu group %d is not viable (not all "
137  "devices in this group bound to vfio-pci)",
138  group);
139  goto error;
140  }
141 
142  if (ioctl (fd, VFIO_GROUP_SET_CONTAINER, &lvm->container_fd) < 0)
143  {
144  err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_SET_CONTAINER) '%s'",
145  s);
146  goto error;
147  }
148 
149  if (lvm->iommu_mode == 0)
150  {
151  if (is_noiommu)
153  else
154  lvm->iommu_mode = VFIO_TYPE1_IOMMU;
155 
156  if (ioctl (lvm->container_fd, VFIO_SET_IOMMU, lvm->iommu_mode) < 0)
157  {
158  err = clib_error_return_unix (0, "ioctl(VFIO_SET_IOMMU) "
159  "'/dev/vfio/vfio'");
160  goto error;
161  }
162  }
163 
164 
165  pool_get (lvm->iommu_groups, g);
166  g->fd = fd;
167  g->refcnt = 1;
168  hash_set (lvm->iommu_pool_index_by_group, group, g - lvm->iommu_groups);
169  vec_free (s);
170  return 0;
171 error:
172  close (fd);
173  return err;
174 }
175 
176 clib_error_t *
177 linux_vfio_group_get_device_fd (vlib_pci_addr_t * addr, int *fdp,
178  int *is_noiommu)
179 {
180  clib_error_t *err = 0;
182  u8 *s = 0;
183  int iommu_group;
184  u8 *tmpstr;
185  int fd;
186 
187  *is_noiommu = 0;
188  s =
189  format (s, "/sys/bus/pci/devices/%U/iommu_group%c", format_vlib_pci_addr,
190  addr, 0);
191  tmpstr = clib_sysfs_link_to_name ((char *) s);
192  if (tmpstr)
193  {
194  iommu_group = atoi ((char *) tmpstr);
195  vec_free (tmpstr);
196  }
197  else
198  {
199  err = clib_error_return (0, "Cannot find IOMMU group for PCI device ",
200  "'%U'", format_vlib_pci_addr, addr);
201  goto error;
202  }
203  vec_reset_length (s);
204 
205  s = format (s, "/sys/bus/pci/devices/%U/iommu_group/name%c",
207  err = clib_sysfs_read ((char *) s, "%s", &tmpstr);
208  if (err == 0)
209  {
210  if (strncmp ((char *) tmpstr, "vfio-noiommu", 12) == 0)
211  *is_noiommu = 1;
212 
213  vec_free (tmpstr);
214  }
215  else
216  clib_error_free (err);
217  vec_reset_length (s);
218  if ((err = open_vfio_iommu_group (iommu_group, *is_noiommu)))
219  return err;
220 
221  g = get_vfio_iommu_group (iommu_group);
222 
223  s = format (s, "%U%c", format_vlib_pci_addr, addr, 0);
224  if ((fd = ioctl (g->fd, VFIO_GROUP_GET_DEVICE_FD, (char *) s)) < 0)
225  {
226  err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_GET_DEVICE_FD) '%U'",
228  goto error;
229  }
230  vec_reset_length (s);
231 
232  *fdp = fd;
233 
234 error:
235  vec_free (s);
236  return err;
237 }
238 
239 clib_error_t *
241 {
243 
244  lvm->log_default = vlib_log_register_class ("vfio", 0);
245  lvm->container_fd = -1;
246 
247  return 0;
248 }
249 
250 u8 *
251 format_vfio_region_info (u8 * s, va_list * args)
252 {
253  struct vfio_region_info *r = va_arg (*args, struct vfio_region_info *);
254 
255  s = format (s, "region_info index:%u size:0x%lx offset:0x%lx flags:",
256  r->index, r->size, r->offset);
257 
258  if (r->flags & VFIO_REGION_INFO_FLAG_READ)
259  s = format (s, " rd");
260 
261  if (r->flags & VFIO_REGION_INFO_FLAG_WRITE)
262  s = format (s, " wr");
263 
264  if (r->flags & VFIO_REGION_INFO_FLAG_MMAP)
265  s = format (s, " mmap");
266 
267 #ifdef VFIO_REGION_INFO_FLAG_CAPS
268  if (r->flags & VFIO_REGION_INFO_FLAG_CAPS)
269  s = format (s, " caps");
270 #endif
271 
272  s = format (s, " (0x%x)", r->flags);
273 
274 #ifdef VFIO_REGION_INFO_FLAG_CAPS
275  u32 cap_offset;
276 
277  if ((r->flags & VFIO_REGION_INFO_FLAG_CAPS) == 0)
278  return s;
279 
280  s = format (s, "\n caps:");
281  cap_offset = r->cap_offset;
282 
283  do
284  {
285  struct vfio_info_cap_header *cap = (void *) r + cap_offset;
286 #ifdef VFIO_REGION_INFO_CAP_SPARSE_MMAP
287  if (cap->id == VFIO_REGION_INFO_CAP_SPARSE_MMAP)
288  s = format (s, " sparse-mmap");
289 #endif
290 #ifdef VFIO_REGION_INFO_CAP_TYPE
291  if (cap->id == VFIO_REGION_INFO_CAP_TYPE)
292  s = format (s, " type");
293 #endif
294 #ifdef VFIO_REGION_INFO_CAP_MSIX_MAPPABLE
295  if (cap->id == VFIO_REGION_INFO_CAP_MSIX_MAPPABLE)
296  s = format (s, " msix-mappable");
297 #endif
298  cap_offset = cap->next;
299  }
300  while (cap_offset);
301 #endif
302 
303  return s;
304 }
305 
306 /*
307  * fd.io coding-style-patch-verification: ON
308  *
309  * Local Variables:
310  * eval: (c-set-style "gnu")
311  * End:
312  */
vec_reset_length
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
Definition: vec_bootstrap.h:194
vlib.h
vlib_physmem_get_page_index
static u64 vlib_physmem_get_page_index(vlib_main_t *vm, void *mem)
Definition: physmem_funcs.h:96
clib_sysfs_link_to_name
__clib_export u8 * clib_sysfs_link_to_name(char *link)
Definition: sysfs.c:91
pointer_to_uword
static uword pointer_to_uword(const void *p)
Definition: types.h:131
pool_elt_at_index
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:553
linux_pci_vfio_iommu_group_t::fd
int fd
Definition: vfio.h:22
vlib_log_register_class
vlib_log_class_t vlib_log_register_class(char *class, char *subclass)
Definition: log.c:339
open_vfio_iommu_group
static clib_error_t * open_vfio_iommu_group(int group, int is_noiommu)
Definition: vfio.c:91
linux_pci_vfio_iommu_group_t::refcnt
int refcnt
Definition: vfio.h:23
clib_error_return
#define clib_error_return(e, args...)
Definition: error.h:99
vm
vlib_main_t * vm
X-connect all packets from the HOST to the PHY.
Definition: nat44_ei.c:3047
vlib_physmem_main_t::pmalloc_main
clib_pmalloc_main_t * pmalloc_main
Definition: physmem.h:64
vfio_main
linux_vfio_main_t vfio_main
Definition: vfio.c:35
addr
vhost_vring_addr_t addr
Definition: vhost_user.h:130
r
vnet_hw_if_output_node_runtime_t * r
Definition: interface_output.c:1071
hash_create
#define hash_create(elts, value_bytes)
Definition: hash.h:695
linux_vfio_main_t::iommu_pool_index_by_group
uword * iommu_pool_index_by_group
Definition: vfio.h:37
error
Definition: cJSON.c:88
sysfs.h
vlib_log_err
#define vlib_log_err(...)
Definition: log.h:133
hash_set
#define hash_set(h, key, value)
Definition: hash.h:255
vfio_map_physmem_page
clib_error_t * vfio_map_physmem_page(vlib_main_t *vm, void *addr)
Definition: vfio.c:38
clib_sysfs_read
__clib_export clib_error_t * clib_sysfs_read(char *file_name, char *fmt,...)
Definition: sysfs.c:51
clib_bitmap_get
static uword clib_bitmap_get(uword *ai, uword i)
Gets the ith bit value from a bitmap.
Definition: bitmap.h:197
vlib_main_t::physmem_main
vlib_physmem_main_t physmem_main
Definition: main.h:168
linux_vfio_main_t
Definition: vfio.h:26
uword
u64 uword
Definition: types.h:112
hash_get
#define hash_get(h, key)
Definition: hash.h:249
linux_vfio_main_t::container_fd
int container_fd
Definition: vfio.h:28
pool_get
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:255
vlib_log_debug
#define vlib_log_debug(...)
Definition: log.h:137
clib_bitmap_set
static uword * clib_bitmap_set(uword *ai, uword i, uword value)
Sets the ith bit of a bitmap to new_value Removes trailing zeros from the bitmap.
Definition: bitmap.h:167
format_vfio_region_info
u8 * format_vfio_region_info(u8 *s, va_list *args)
Definition: vfio.c:251
linux_vfio_main_t::log_default
vlib_log_class_t log_default
Definition: vfio.h:42
vec_free
#define vec_free(V)
Free vector's memory (no header).
Definition: vec.h:395
clib_pmalloc_main_t::base
u8 * base
Definition: pmalloc.h:67
format_vlib_pci_addr
format_function_t format_vlib_pci_addr
Definition: pci.h:326
linux_vfio_main_t::iommu_mode
int iommu_mode
Definition: vfio.h:31
format
description fragment has unexpected format
Definition: map.api:433
u32
unsigned int u32
Definition: types.h:88
linux_vfio_main_t::physmem_pages_mapped
clib_bitmap_t * physmem_pages_mapped
Definition: vfio.h:39
pci.h
clib_pmalloc_main_t::def_log2_page_sz
clib_mem_page_sz_t def_log2_page_sz
Definition: pmalloc.h:70
clib_error_return_unix
#define clib_error_return_unix(e, args...)
Definition: error.h:102
linux_vfio_init
clib_error_t * linux_vfio_init(vlib_main_t *vm)
Definition: vfio.c:240
get_vfio_iommu_group
static linux_pci_vfio_iommu_group_t * get_vfio_iommu_group(int group)
Definition: vfio.c:80
physmem.h
vlib_main_t
Definition: main.h:102
u8
unsigned char u8
Definition: types.h:56
clib_error_t
Definition: clib_error.h:21
vfio.h
unix.h
vlib_physmem_main_t
Definition: physmem.h:56
VFIO_NOIOMMU_IOMMU
#define VFIO_NOIOMMU_IOMMU
Definition: vfio.c:32
clib_error_free
#define clib_error_free(e)
Definition: error.h:86
linux_pci_vfio_iommu_group_t
Definition: vfio.h:19
linux_vfio_group_get_device_fd
clib_error_t * linux_vfio_group_get_device_fd(vlib_pci_addr_t *addr, int *fdp, int *is_noiommu)
Definition: vfio.c:177
linux_vfio_main_t::iommu_groups
linux_pci_vfio_iommu_group_t * iommu_groups
Definition: vfio.h:34