Commit | Line | Data |
---|---|---|
d2912cb1 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
ec53500f AW |
2 | /* |
3 | * VFIO-KVM bridge pseudo device | |
4 | * | |
5 | * Copyright (C) 2013 Red Hat, Inc. All rights reserved. | |
6 | * Author: Alex Williamson <alex.williamson@redhat.com> | |
ec53500f AW |
7 | */ |
8 | ||
9 | #include <linux/errno.h> | |
10 | #include <linux/file.h> | |
11 | #include <linux/kvm_host.h> | |
12 | #include <linux/list.h> | |
13 | #include <linux/module.h> | |
14 | #include <linux/mutex.h> | |
15 | #include <linux/slab.h> | |
16 | #include <linux/uaccess.h> | |
17 | #include <linux/vfio.h> | |
3c3c29fd | 18 | #include "vfio.h" |
ec53500f | 19 | |
121f80ba AK |
20 | #ifdef CONFIG_SPAPR_TCE_IOMMU |
21 | #include <asm/kvm_ppc.h> | |
22 | #endif | |
23 | ||
ec53500f AW |
24 | struct kvm_vfio_group { |
25 | struct list_head node; | |
d55d9e7a | 26 | struct file *file; |
819da99a JG |
27 | #ifdef CONFIG_SPAPR_TCE_IOMMU |
28 | struct iommu_group *iommu_group; | |
29 | #endif | |
ec53500f AW |
30 | }; |
31 | ||
32 | struct kvm_vfio { | |
33 | struct list_head group_list; | |
34 | struct mutex lock; | |
e0f0bbc5 | 35 | bool noncoherent; |
ec53500f AW |
36 | }; |
37 | ||
ba70a89f | 38 | static void kvm_vfio_file_set_kvm(struct file *file, struct kvm *kvm) |
2fc1bec1 | 39 | { |
ba70a89f | 40 | void (*fn)(struct file *file, struct kvm *kvm); |
2fc1bec1 | 41 | |
ba70a89f | 42 | fn = symbol_get(vfio_file_set_kvm); |
2fc1bec1 JS |
43 | if (!fn) |
44 | return; | |
45 | ||
ba70a89f | 46 | fn(file, kvm); |
2fc1bec1 | 47 | |
ba70a89f | 48 | symbol_put(vfio_file_set_kvm); |
2fc1bec1 JS |
49 | } |
50 | ||
a905ad04 | 51 | static bool kvm_vfio_file_enforced_coherent(struct file *file) |
9d830d47 | 52 | { |
a905ad04 JG |
53 | bool (*fn)(struct file *file); |
54 | bool ret; | |
9d830d47 | 55 | |
a905ad04 | 56 | fn = symbol_get(vfio_file_enforced_coherent); |
9d830d47 AW |
57 | if (!fn) |
58 | return false; | |
59 | ||
a905ad04 | 60 | ret = fn(file); |
9d830d47 | 61 | |
a905ad04 | 62 | symbol_put(vfio_file_enforced_coherent); |
9d830d47 | 63 | |
a905ad04 | 64 | return ret; |
9d830d47 AW |
65 | } |
66 | ||
4b22ef04 JG |
67 | static bool kvm_vfio_file_is_group(struct file *file) |
68 | { | |
69 | bool (*fn)(struct file *file); | |
70 | bool ret; | |
71 | ||
72 | fn = symbol_get(vfio_file_is_group); | |
73 | if (!fn) | |
74 | return false; | |
75 | ||
76 | ret = fn(file); | |
77 | ||
78 | symbol_put(vfio_file_is_group); | |
79 | ||
80 | return ret; | |
81 | } | |
82 | ||
83 | #ifdef CONFIG_SPAPR_TCE_IOMMU | |
50d63b5b | 84 | static struct iommu_group *kvm_vfio_file_iommu_group(struct file *file) |
121f80ba | 85 | { |
50d63b5b JG |
86 | struct iommu_group *(*fn)(struct file *file); |
87 | struct iommu_group *ret; | |
121f80ba | 88 | |
50d63b5b | 89 | fn = symbol_get(vfio_file_iommu_group); |
121f80ba | 90 | if (!fn) |
50d63b5b | 91 | return NULL; |
121f80ba | 92 | |
50d63b5b | 93 | ret = fn(file); |
121f80ba | 94 | |
50d63b5b | 95 | symbol_put(vfio_file_iommu_group); |
121f80ba AK |
96 | |
97 | return ret; | |
98 | } | |
99 | ||
121f80ba | 100 | static void kvm_spapr_tce_release_vfio_group(struct kvm *kvm, |
50d63b5b | 101 | struct kvm_vfio_group *kvg) |
121f80ba | 102 | { |
819da99a | 103 | if (WARN_ON_ONCE(!kvg->iommu_group)) |
121f80ba AK |
104 | return; |
105 | ||
819da99a JG |
106 | kvm_spapr_tce_release_iommu_group(kvm, kvg->iommu_group); |
107 | iommu_group_put(kvg->iommu_group); | |
108 | kvg->iommu_group = NULL; | |
121f80ba AK |
109 | } |
110 | #endif | |
111 | ||
e0f0bbc5 AW |
112 | /* |
113 | * Groups can use the same or different IOMMU domains. If the same then | |
114 | * adding a new group may change the coherency of groups we've previously | |
115 | * been told about. We don't want to care about any of that so we retest | |
116 | * each group and bail as soon as we find one that's noncoherent. This | |
117 | * means we only ever [un]register_noncoherent_dma once for the whole device. | |
118 | */ | |
119 | static void kvm_vfio_update_coherency(struct kvm_device *dev) | |
120 | { | |
121 | struct kvm_vfio *kv = dev->private; | |
122 | bool noncoherent = false; | |
123 | struct kvm_vfio_group *kvg; | |
124 | ||
125 | mutex_lock(&kv->lock); | |
126 | ||
127 | list_for_each_entry(kvg, &kv->group_list, node) { | |
a905ad04 | 128 | if (!kvm_vfio_file_enforced_coherent(kvg->file)) { |
9d830d47 AW |
129 | noncoherent = true; |
130 | break; | |
131 | } | |
e0f0bbc5 AW |
132 | } |
133 | ||
134 | if (noncoherent != kv->noncoherent) { | |
135 | kv->noncoherent = noncoherent; | |
136 | ||
137 | if (kv->noncoherent) | |
138 | kvm_arch_register_noncoherent_dma(dev->kvm); | |
139 | else | |
140 | kvm_arch_unregister_noncoherent_dma(dev->kvm); | |
141 | } | |
142 | ||
143 | mutex_unlock(&kv->lock); | |
144 | } | |
145 | ||
73b0565f | 146 | static int kvm_vfio_group_add(struct kvm_device *dev, unsigned int fd) |
ec53500f AW |
147 | { |
148 | struct kvm_vfio *kv = dev->private; | |
ec53500f | 149 | struct kvm_vfio_group *kvg; |
d55d9e7a | 150 | struct file *filp; |
ec53500f AW |
151 | int ret; |
152 | ||
d55d9e7a JG |
153 | filp = fget(fd); |
154 | if (!filp) | |
73b0565f | 155 | return -EBADF; |
ec53500f | 156 | |
3e5449d5 | 157 | /* Ensure the FD is a vfio group FD.*/ |
4b22ef04 | 158 | if (!kvm_vfio_file_is_group(filp)) { |
3e5449d5 JG |
159 | ret = -EINVAL; |
160 | goto err_fput; | |
161 | } | |
162 | ||
73b0565f | 163 | mutex_lock(&kv->lock); |
ec53500f | 164 | |
73b0565f | 165 | list_for_each_entry(kvg, &kv->group_list, node) { |
d55d9e7a | 166 | if (kvg->file == filp) { |
73b0565f JG |
167 | ret = -EEXIST; |
168 | goto err_unlock; | |
ec53500f | 169 | } |
73b0565f | 170 | } |
ec53500f | 171 | |
73b0565f JG |
172 | kvg = kzalloc(sizeof(*kvg), GFP_KERNEL_ACCOUNT); |
173 | if (!kvg) { | |
174 | ret = -ENOMEM; | |
175 | goto err_unlock; | |
176 | } | |
ec53500f | 177 | |
d55d9e7a | 178 | kvg->file = filp; |
73b0565f | 179 | list_add_tail(&kvg->node, &kv->group_list); |
5544eb9b | 180 | |
73b0565f | 181 | kvm_arch_start_assignment(dev->kvm); |
ec53500f | 182 | |
73b0565f | 183 | mutex_unlock(&kv->lock); |
2fc1bec1 | 184 | |
ba70a89f | 185 | kvm_vfio_file_set_kvm(kvg->file, dev->kvm); |
73b0565f | 186 | kvm_vfio_update_coherency(dev); |
e0f0bbc5 | 187 | |
73b0565f JG |
188 | return 0; |
189 | err_unlock: | |
190 | mutex_unlock(&kv->lock); | |
3e5449d5 | 191 | err_fput: |
d55d9e7a | 192 | fput(filp); |
73b0565f JG |
193 | return ret; |
194 | } | |
ec53500f | 195 | |
73b0565f JG |
196 | static int kvm_vfio_group_del(struct kvm_device *dev, unsigned int fd) |
197 | { | |
198 | struct kvm_vfio *kv = dev->private; | |
199 | struct kvm_vfio_group *kvg; | |
200 | struct fd f; | |
201 | int ret; | |
ec53500f | 202 | |
73b0565f JG |
203 | f = fdget(fd); |
204 | if (!f.file) | |
205 | return -EBADF; | |
ec53500f | 206 | |
73b0565f | 207 | ret = -ENOENT; |
ec53500f | 208 | |
73b0565f | 209 | mutex_lock(&kv->lock); |
ec53500f | 210 | |
73b0565f | 211 | list_for_each_entry(kvg, &kv->group_list, node) { |
c38ff5b0 | 212 | if (kvg->file != f.file) |
73b0565f | 213 | continue; |
ec53500f | 214 | |
73b0565f JG |
215 | list_del(&kvg->node); |
216 | kvm_arch_end_assignment(dev->kvm); | |
e323369b | 217 | #ifdef CONFIG_SPAPR_TCE_IOMMU |
50d63b5b | 218 | kvm_spapr_tce_release_vfio_group(dev->kvm, kvg); |
e323369b | 219 | #endif |
ba70a89f | 220 | kvm_vfio_file_set_kvm(kvg->file, NULL); |
d55d9e7a | 221 | fput(kvg->file); |
73b0565f JG |
222 | kfree(kvg); |
223 | ret = 0; | |
224 | break; | |
225 | } | |
ec53500f | 226 | |
73b0565f | 227 | mutex_unlock(&kv->lock); |
ec53500f | 228 | |
73b0565f | 229 | fdput(f); |
ec53500f | 230 | |
73b0565f | 231 | kvm_vfio_update_coherency(dev); |
e0f0bbc5 | 232 | |
73b0565f JG |
233 | return ret; |
234 | } | |
121f80ba AK |
235 | |
236 | #ifdef CONFIG_SPAPR_TCE_IOMMU | |
73b0565f JG |
237 | static int kvm_vfio_group_set_spapr_tce(struct kvm_device *dev, |
238 | void __user *arg) | |
239 | { | |
240 | struct kvm_vfio_spapr_tce param; | |
241 | struct kvm_vfio *kv = dev->private; | |
73b0565f JG |
242 | struct kvm_vfio_group *kvg; |
243 | struct fd f; | |
73b0565f | 244 | int ret; |
121f80ba | 245 | |
73b0565f JG |
246 | if (copy_from_user(¶m, arg, sizeof(struct kvm_vfio_spapr_tce))) |
247 | return -EFAULT; | |
121f80ba | 248 | |
73b0565f JG |
249 | f = fdget(param.groupfd); |
250 | if (!f.file) | |
251 | return -EBADF; | |
121f80ba | 252 | |
73b0565f | 253 | ret = -ENOENT; |
121f80ba | 254 | |
73b0565f | 255 | mutex_lock(&kv->lock); |
121f80ba | 256 | |
73b0565f | 257 | list_for_each_entry(kvg, &kv->group_list, node) { |
d55d9e7a | 258 | if (kvg->file != f.file) |
73b0565f | 259 | continue; |
121f80ba | 260 | |
819da99a JG |
261 | if (!kvg->iommu_group) { |
262 | kvg->iommu_group = kvm_vfio_file_iommu_group(kvg->file); | |
263 | if (WARN_ON_ONCE(!kvg->iommu_group)) { | |
264 | ret = -EIO; | |
265 | goto err_fdput; | |
266 | } | |
d55d9e7a JG |
267 | } |
268 | ||
73b0565f | 269 | ret = kvm_spapr_tce_attach_iommu_group(dev->kvm, param.tablefd, |
819da99a | 270 | kvg->iommu_group); |
73b0565f JG |
271 | break; |
272 | } | |
121f80ba | 273 | |
d55d9e7a | 274 | err_fdput: |
6b17ca8e | 275 | mutex_unlock(&kv->lock); |
d55d9e7a | 276 | fdput(f); |
73b0565f JG |
277 | return ret; |
278 | } | |
279 | #endif | |
280 | ||
281 | static int kvm_vfio_set_group(struct kvm_device *dev, long attr, | |
282 | void __user *arg) | |
283 | { | |
284 | int32_t __user *argp = arg; | |
285 | int32_t fd; | |
286 | ||
287 | switch (attr) { | |
288 | case KVM_DEV_VFIO_GROUP_ADD: | |
289 | if (get_user(fd, argp)) | |
290 | return -EFAULT; | |
291 | return kvm_vfio_group_add(dev, fd); | |
292 | ||
293 | case KVM_DEV_VFIO_GROUP_DEL: | |
294 | if (get_user(fd, argp)) | |
295 | return -EFAULT; | |
296 | return kvm_vfio_group_del(dev, fd); | |
297 | ||
298 | #ifdef CONFIG_SPAPR_TCE_IOMMU | |
299 | case KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE: | |
300 | return kvm_vfio_group_set_spapr_tce(dev, arg); | |
301 | #endif | |
ec53500f AW |
302 | } |
303 | ||
304 | return -ENXIO; | |
305 | } | |
306 | ||
307 | static int kvm_vfio_set_attr(struct kvm_device *dev, | |
308 | struct kvm_device_attr *attr) | |
309 | { | |
310 | switch (attr->group) { | |
311 | case KVM_DEV_VFIO_GROUP: | |
73b0565f JG |
312 | return kvm_vfio_set_group(dev, attr->attr, |
313 | u64_to_user_ptr(attr->addr)); | |
ec53500f AW |
314 | } |
315 | ||
316 | return -ENXIO; | |
317 | } | |
318 | ||
319 | static int kvm_vfio_has_attr(struct kvm_device *dev, | |
320 | struct kvm_device_attr *attr) | |
321 | { | |
322 | switch (attr->group) { | |
323 | case KVM_DEV_VFIO_GROUP: | |
324 | switch (attr->attr) { | |
325 | case KVM_DEV_VFIO_GROUP_ADD: | |
326 | case KVM_DEV_VFIO_GROUP_DEL: | |
121f80ba AK |
327 | #ifdef CONFIG_SPAPR_TCE_IOMMU |
328 | case KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE: | |
329 | #endif | |
ec53500f AW |
330 | return 0; |
331 | } | |
332 | ||
333 | break; | |
334 | } | |
335 | ||
336 | return -ENXIO; | |
337 | } | |
338 | ||
51cdc8bc | 339 | static void kvm_vfio_release(struct kvm_device *dev) |
ec53500f AW |
340 | { |
341 | struct kvm_vfio *kv = dev->private; | |
342 | struct kvm_vfio_group *kvg, *tmp; | |
343 | ||
344 | list_for_each_entry_safe(kvg, tmp, &kv->group_list, node) { | |
121f80ba | 345 | #ifdef CONFIG_SPAPR_TCE_IOMMU |
50d63b5b | 346 | kvm_spapr_tce_release_vfio_group(dev->kvm, kvg); |
121f80ba | 347 | #endif |
ba70a89f | 348 | kvm_vfio_file_set_kvm(kvg->file, NULL); |
d55d9e7a | 349 | fput(kvg->file); |
ec53500f AW |
350 | list_del(&kvg->node); |
351 | kfree(kvg); | |
5544eb9b | 352 | kvm_arch_end_assignment(dev->kvm); |
ec53500f AW |
353 | } |
354 | ||
e0f0bbc5 AW |
355 | kvm_vfio_update_coherency(dev); |
356 | ||
ec53500f | 357 | kfree(kv); |
51cdc8bc | 358 | kfree(dev); /* alloc by kvm_ioctl_create_device, free by .release */ |
ec53500f AW |
359 | } |
360 | ||
80ce1639 WD |
361 | static int kvm_vfio_create(struct kvm_device *dev, u32 type); |
362 | ||
363 | static struct kvm_device_ops kvm_vfio_ops = { | |
364 | .name = "kvm-vfio", | |
365 | .create = kvm_vfio_create, | |
51cdc8bc | 366 | .release = kvm_vfio_release, |
80ce1639 WD |
367 | .set_attr = kvm_vfio_set_attr, |
368 | .has_attr = kvm_vfio_has_attr, | |
369 | }; | |
370 | ||
ec53500f AW |
371 | static int kvm_vfio_create(struct kvm_device *dev, u32 type) |
372 | { | |
373 | struct kvm_device *tmp; | |
374 | struct kvm_vfio *kv; | |
375 | ||
376 | /* Only one VFIO "device" per VM */ | |
377 | list_for_each_entry(tmp, &dev->kvm->devices, vm_node) | |
378 | if (tmp->ops == &kvm_vfio_ops) | |
379 | return -EBUSY; | |
380 | ||
b12ce36a | 381 | kv = kzalloc(sizeof(*kv), GFP_KERNEL_ACCOUNT); |
ec53500f AW |
382 | if (!kv) |
383 | return -ENOMEM; | |
384 | ||
385 | INIT_LIST_HEAD(&kv->group_list); | |
386 | mutex_init(&kv->lock); | |
387 | ||
388 | dev->private = kv; | |
389 | ||
390 | return 0; | |
391 | } | |
392 | ||
3c3c29fd | 393 | int kvm_vfio_ops_init(void) |
80ce1639 WD |
394 | { |
395 | return kvm_register_device_ops(&kvm_vfio_ops, KVM_DEV_TYPE_VFIO); | |
396 | } | |
571ee1b6 WL |
397 | |
398 | void kvm_vfio_ops_exit(void) | |
399 | { | |
400 | kvm_unregister_device_ops(KVM_DEV_TYPE_VFIO); | |
401 | } |