Commit | Line | Data |
---|---|---|
d2912cb1 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
ec53500f AW |
2 | /* |
3 | * VFIO-KVM bridge pseudo device | |
4 | * | |
5 | * Copyright (C) 2013 Red Hat, Inc. All rights reserved. | |
6 | * Author: Alex Williamson <alex.williamson@redhat.com> | |
ec53500f AW |
7 | */ |
8 | ||
9 | #include <linux/errno.h> | |
10 | #include <linux/file.h> | |
11 | #include <linux/kvm_host.h> | |
12 | #include <linux/list.h> | |
13 | #include <linux/module.h> | |
14 | #include <linux/mutex.h> | |
15 | #include <linux/slab.h> | |
16 | #include <linux/uaccess.h> | |
17 | #include <linux/vfio.h> | |
3c3c29fd | 18 | #include "vfio.h" |
ec53500f | 19 | |
121f80ba AK |
20 | #ifdef CONFIG_SPAPR_TCE_IOMMU |
21 | #include <asm/kvm_ppc.h> | |
22 | #endif | |
23 | ||
ec53500f AW |
24 | struct kvm_vfio_group { |
25 | struct list_head node; | |
d55d9e7a | 26 | struct file *file; |
ec53500f AW |
27 | }; |
28 | ||
29 | struct kvm_vfio { | |
30 | struct list_head group_list; | |
31 | struct mutex lock; | |
e0f0bbc5 | 32 | bool noncoherent; |
ec53500f AW |
33 | }; |
34 | ||
ba70a89f | 35 | static void kvm_vfio_file_set_kvm(struct file *file, struct kvm *kvm) |
2fc1bec1 | 36 | { |
ba70a89f | 37 | void (*fn)(struct file *file, struct kvm *kvm); |
2fc1bec1 | 38 | |
ba70a89f | 39 | fn = symbol_get(vfio_file_set_kvm); |
2fc1bec1 JS |
40 | if (!fn) |
41 | return; | |
42 | ||
ba70a89f | 43 | fn(file, kvm); |
2fc1bec1 | 44 | |
ba70a89f | 45 | symbol_put(vfio_file_set_kvm); |
2fc1bec1 JS |
46 | } |
47 | ||
a905ad04 | 48 | static bool kvm_vfio_file_enforced_coherent(struct file *file) |
9d830d47 | 49 | { |
a905ad04 JG |
50 | bool (*fn)(struct file *file); |
51 | bool ret; | |
9d830d47 | 52 | |
a905ad04 | 53 | fn = symbol_get(vfio_file_enforced_coherent); |
9d830d47 AW |
54 | if (!fn) |
55 | return false; | |
56 | ||
a905ad04 | 57 | ret = fn(file); |
9d830d47 | 58 | |
a905ad04 | 59 | symbol_put(vfio_file_enforced_coherent); |
9d830d47 | 60 | |
a905ad04 | 61 | return ret; |
9d830d47 AW |
62 | } |
63 | ||
50d63b5b | 64 | static struct iommu_group *kvm_vfio_file_iommu_group(struct file *file) |
121f80ba | 65 | { |
50d63b5b JG |
66 | struct iommu_group *(*fn)(struct file *file); |
67 | struct iommu_group *ret; | |
121f80ba | 68 | |
50d63b5b | 69 | fn = symbol_get(vfio_file_iommu_group); |
121f80ba | 70 | if (!fn) |
50d63b5b | 71 | return NULL; |
121f80ba | 72 | |
50d63b5b | 73 | ret = fn(file); |
121f80ba | 74 | |
50d63b5b | 75 | symbol_put(vfio_file_iommu_group); |
121f80ba AK |
76 | |
77 | return ret; | |
78 | } | |
79 | ||
3e5449d5 | 80 | #ifdef CONFIG_SPAPR_TCE_IOMMU |
121f80ba | 81 | static void kvm_spapr_tce_release_vfio_group(struct kvm *kvm, |
50d63b5b | 82 | struct kvm_vfio_group *kvg) |
121f80ba | 83 | { |
50d63b5b | 84 | struct iommu_group *grp = kvm_vfio_file_iommu_group(kvg->file); |
121f80ba AK |
85 | |
86 | if (WARN_ON_ONCE(!grp)) | |
87 | return; | |
88 | ||
89 | kvm_spapr_tce_release_iommu_group(kvm, grp); | |
121f80ba AK |
90 | } |
91 | #endif | |
92 | ||
e0f0bbc5 AW |
93 | /* |
94 | * Groups can use the same or different IOMMU domains. If the same then | |
95 | * adding a new group may change the coherency of groups we've previously | |
96 | * been told about. We don't want to care about any of that so we retest | |
97 | * each group and bail as soon as we find one that's noncoherent. This | |
98 | * means we only ever [un]register_noncoherent_dma once for the whole device. | |
99 | */ | |
100 | static void kvm_vfio_update_coherency(struct kvm_device *dev) | |
101 | { | |
102 | struct kvm_vfio *kv = dev->private; | |
103 | bool noncoherent = false; | |
104 | struct kvm_vfio_group *kvg; | |
105 | ||
106 | mutex_lock(&kv->lock); | |
107 | ||
108 | list_for_each_entry(kvg, &kv->group_list, node) { | |
a905ad04 | 109 | if (!kvm_vfio_file_enforced_coherent(kvg->file)) { |
9d830d47 AW |
110 | noncoherent = true; |
111 | break; | |
112 | } | |
e0f0bbc5 AW |
113 | } |
114 | ||
115 | if (noncoherent != kv->noncoherent) { | |
116 | kv->noncoherent = noncoherent; | |
117 | ||
118 | if (kv->noncoherent) | |
119 | kvm_arch_register_noncoherent_dma(dev->kvm); | |
120 | else | |
121 | kvm_arch_unregister_noncoherent_dma(dev->kvm); | |
122 | } | |
123 | ||
124 | mutex_unlock(&kv->lock); | |
125 | } | |
126 | ||
73b0565f | 127 | static int kvm_vfio_group_add(struct kvm_device *dev, unsigned int fd) |
ec53500f AW |
128 | { |
129 | struct kvm_vfio *kv = dev->private; | |
ec53500f | 130 | struct kvm_vfio_group *kvg; |
d55d9e7a | 131 | struct file *filp; |
ec53500f AW |
132 | int ret; |
133 | ||
d55d9e7a JG |
134 | filp = fget(fd); |
135 | if (!filp) | |
73b0565f | 136 | return -EBADF; |
ec53500f | 137 | |
3e5449d5 JG |
138 | /* Ensure the FD is a vfio group FD.*/ |
139 | if (!kvm_vfio_file_iommu_group(filp)) { | |
140 | ret = -EINVAL; | |
141 | goto err_fput; | |
142 | } | |
143 | ||
73b0565f | 144 | mutex_lock(&kv->lock); |
ec53500f | 145 | |
73b0565f | 146 | list_for_each_entry(kvg, &kv->group_list, node) { |
d55d9e7a | 147 | if (kvg->file == filp) { |
73b0565f JG |
148 | ret = -EEXIST; |
149 | goto err_unlock; | |
ec53500f | 150 | } |
73b0565f | 151 | } |
ec53500f | 152 | |
73b0565f JG |
153 | kvg = kzalloc(sizeof(*kvg), GFP_KERNEL_ACCOUNT); |
154 | if (!kvg) { | |
155 | ret = -ENOMEM; | |
156 | goto err_unlock; | |
157 | } | |
ec53500f | 158 | |
d55d9e7a | 159 | kvg->file = filp; |
73b0565f | 160 | list_add_tail(&kvg->node, &kv->group_list); |
5544eb9b | 161 | |
73b0565f | 162 | kvm_arch_start_assignment(dev->kvm); |
ec53500f | 163 | |
73b0565f | 164 | mutex_unlock(&kv->lock); |
2fc1bec1 | 165 | |
ba70a89f | 166 | kvm_vfio_file_set_kvm(kvg->file, dev->kvm); |
73b0565f | 167 | kvm_vfio_update_coherency(dev); |
e0f0bbc5 | 168 | |
73b0565f JG |
169 | return 0; |
170 | err_unlock: | |
171 | mutex_unlock(&kv->lock); | |
3e5449d5 | 172 | err_fput: |
d55d9e7a | 173 | fput(filp); |
73b0565f JG |
174 | return ret; |
175 | } | |
ec53500f | 176 | |
73b0565f JG |
177 | static int kvm_vfio_group_del(struct kvm_device *dev, unsigned int fd) |
178 | { | |
179 | struct kvm_vfio *kv = dev->private; | |
180 | struct kvm_vfio_group *kvg; | |
181 | struct fd f; | |
182 | int ret; | |
ec53500f | 183 | |
73b0565f JG |
184 | f = fdget(fd); |
185 | if (!f.file) | |
186 | return -EBADF; | |
ec53500f | 187 | |
73b0565f | 188 | ret = -ENOENT; |
ec53500f | 189 | |
73b0565f | 190 | mutex_lock(&kv->lock); |
ec53500f | 191 | |
73b0565f | 192 | list_for_each_entry(kvg, &kv->group_list, node) { |
c38ff5b0 | 193 | if (kvg->file != f.file) |
73b0565f | 194 | continue; |
ec53500f | 195 | |
73b0565f JG |
196 | list_del(&kvg->node); |
197 | kvm_arch_end_assignment(dev->kvm); | |
e323369b | 198 | #ifdef CONFIG_SPAPR_TCE_IOMMU |
50d63b5b | 199 | kvm_spapr_tce_release_vfio_group(dev->kvm, kvg); |
e323369b | 200 | #endif |
ba70a89f | 201 | kvm_vfio_file_set_kvm(kvg->file, NULL); |
d55d9e7a | 202 | fput(kvg->file); |
73b0565f JG |
203 | kfree(kvg); |
204 | ret = 0; | |
205 | break; | |
206 | } | |
ec53500f | 207 | |
73b0565f | 208 | mutex_unlock(&kv->lock); |
ec53500f | 209 | |
73b0565f | 210 | fdput(f); |
ec53500f | 211 | |
73b0565f | 212 | kvm_vfio_update_coherency(dev); |
e0f0bbc5 | 213 | |
73b0565f JG |
214 | return ret; |
215 | } | |
121f80ba AK |
216 | |
217 | #ifdef CONFIG_SPAPR_TCE_IOMMU | |
73b0565f JG |
218 | static int kvm_vfio_group_set_spapr_tce(struct kvm_device *dev, |
219 | void __user *arg) | |
220 | { | |
221 | struct kvm_vfio_spapr_tce param; | |
222 | struct kvm_vfio *kv = dev->private; | |
73b0565f JG |
223 | struct kvm_vfio_group *kvg; |
224 | struct fd f; | |
73b0565f | 225 | int ret; |
121f80ba | 226 | |
73b0565f JG |
227 | if (copy_from_user(¶m, arg, sizeof(struct kvm_vfio_spapr_tce))) |
228 | return -EFAULT; | |
121f80ba | 229 | |
73b0565f JG |
230 | f = fdget(param.groupfd); |
231 | if (!f.file) | |
232 | return -EBADF; | |
121f80ba | 233 | |
73b0565f | 234 | ret = -ENOENT; |
121f80ba | 235 | |
73b0565f | 236 | mutex_lock(&kv->lock); |
121f80ba | 237 | |
73b0565f | 238 | list_for_each_entry(kvg, &kv->group_list, node) { |
d55d9e7a JG |
239 | struct iommu_group *grp; |
240 | ||
241 | if (kvg->file != f.file) | |
73b0565f | 242 | continue; |
121f80ba | 243 | |
50d63b5b | 244 | grp = kvm_vfio_file_iommu_group(kvg->file); |
d55d9e7a JG |
245 | if (WARN_ON_ONCE(!grp)) { |
246 | ret = -EIO; | |
247 | goto err_fdput; | |
248 | } | |
249 | ||
73b0565f JG |
250 | ret = kvm_spapr_tce_attach_iommu_group(dev->kvm, param.tablefd, |
251 | grp); | |
252 | break; | |
253 | } | |
121f80ba | 254 | |
d55d9e7a | 255 | err_fdput: |
6b17ca8e | 256 | mutex_unlock(&kv->lock); |
d55d9e7a | 257 | fdput(f); |
73b0565f JG |
258 | return ret; |
259 | } | |
260 | #endif | |
261 | ||
262 | static int kvm_vfio_set_group(struct kvm_device *dev, long attr, | |
263 | void __user *arg) | |
264 | { | |
265 | int32_t __user *argp = arg; | |
266 | int32_t fd; | |
267 | ||
268 | switch (attr) { | |
269 | case KVM_DEV_VFIO_GROUP_ADD: | |
270 | if (get_user(fd, argp)) | |
271 | return -EFAULT; | |
272 | return kvm_vfio_group_add(dev, fd); | |
273 | ||
274 | case KVM_DEV_VFIO_GROUP_DEL: | |
275 | if (get_user(fd, argp)) | |
276 | return -EFAULT; | |
277 | return kvm_vfio_group_del(dev, fd); | |
278 | ||
279 | #ifdef CONFIG_SPAPR_TCE_IOMMU | |
280 | case KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE: | |
281 | return kvm_vfio_group_set_spapr_tce(dev, arg); | |
282 | #endif | |
ec53500f AW |
283 | } |
284 | ||
285 | return -ENXIO; | |
286 | } | |
287 | ||
288 | static int kvm_vfio_set_attr(struct kvm_device *dev, | |
289 | struct kvm_device_attr *attr) | |
290 | { | |
291 | switch (attr->group) { | |
292 | case KVM_DEV_VFIO_GROUP: | |
73b0565f JG |
293 | return kvm_vfio_set_group(dev, attr->attr, |
294 | u64_to_user_ptr(attr->addr)); | |
ec53500f AW |
295 | } |
296 | ||
297 | return -ENXIO; | |
298 | } | |
299 | ||
300 | static int kvm_vfio_has_attr(struct kvm_device *dev, | |
301 | struct kvm_device_attr *attr) | |
302 | { | |
303 | switch (attr->group) { | |
304 | case KVM_DEV_VFIO_GROUP: | |
305 | switch (attr->attr) { | |
306 | case KVM_DEV_VFIO_GROUP_ADD: | |
307 | case KVM_DEV_VFIO_GROUP_DEL: | |
121f80ba AK |
308 | #ifdef CONFIG_SPAPR_TCE_IOMMU |
309 | case KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE: | |
310 | #endif | |
ec53500f AW |
311 | return 0; |
312 | } | |
313 | ||
314 | break; | |
315 | } | |
316 | ||
317 | return -ENXIO; | |
318 | } | |
319 | ||
320 | static void kvm_vfio_destroy(struct kvm_device *dev) | |
321 | { | |
322 | struct kvm_vfio *kv = dev->private; | |
323 | struct kvm_vfio_group *kvg, *tmp; | |
324 | ||
325 | list_for_each_entry_safe(kvg, tmp, &kv->group_list, node) { | |
121f80ba | 326 | #ifdef CONFIG_SPAPR_TCE_IOMMU |
50d63b5b | 327 | kvm_spapr_tce_release_vfio_group(dev->kvm, kvg); |
121f80ba | 328 | #endif |
ba70a89f | 329 | kvm_vfio_file_set_kvm(kvg->file, NULL); |
d55d9e7a | 330 | fput(kvg->file); |
ec53500f AW |
331 | list_del(&kvg->node); |
332 | kfree(kvg); | |
5544eb9b | 333 | kvm_arch_end_assignment(dev->kvm); |
ec53500f AW |
334 | } |
335 | ||
e0f0bbc5 AW |
336 | kvm_vfio_update_coherency(dev); |
337 | ||
ec53500f AW |
338 | kfree(kv); |
339 | kfree(dev); /* alloc by kvm_ioctl_create_device, free by .destroy */ | |
340 | } | |
341 | ||
80ce1639 WD |
342 | static int kvm_vfio_create(struct kvm_device *dev, u32 type); |
343 | ||
344 | static struct kvm_device_ops kvm_vfio_ops = { | |
345 | .name = "kvm-vfio", | |
346 | .create = kvm_vfio_create, | |
347 | .destroy = kvm_vfio_destroy, | |
348 | .set_attr = kvm_vfio_set_attr, | |
349 | .has_attr = kvm_vfio_has_attr, | |
350 | }; | |
351 | ||
ec53500f AW |
352 | static int kvm_vfio_create(struct kvm_device *dev, u32 type) |
353 | { | |
354 | struct kvm_device *tmp; | |
355 | struct kvm_vfio *kv; | |
356 | ||
357 | /* Only one VFIO "device" per VM */ | |
358 | list_for_each_entry(tmp, &dev->kvm->devices, vm_node) | |
359 | if (tmp->ops == &kvm_vfio_ops) | |
360 | return -EBUSY; | |
361 | ||
b12ce36a | 362 | kv = kzalloc(sizeof(*kv), GFP_KERNEL_ACCOUNT); |
ec53500f AW |
363 | if (!kv) |
364 | return -ENOMEM; | |
365 | ||
366 | INIT_LIST_HEAD(&kv->group_list); | |
367 | mutex_init(&kv->lock); | |
368 | ||
369 | dev->private = kv; | |
370 | ||
371 | return 0; | |
372 | } | |
373 | ||
3c3c29fd | 374 | int kvm_vfio_ops_init(void) |
80ce1639 WD |
375 | { |
376 | return kvm_register_device_ops(&kvm_vfio_ops, KVM_DEV_TYPE_VFIO); | |
377 | } | |
571ee1b6 WL |
378 | |
379 | void kvm_vfio_ops_exit(void) | |
380 | { | |
381 | kvm_unregister_device_ops(KVM_DEV_TYPE_VFIO); | |
382 | } |