projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'pci-v6.16-fixes-3' of git://git.kernel.org/pub/scm/linux/kernel/git/pci/pci
[linux-block.git] / drivers / s390 / crypto / vfio_ap_ops.c
CommitLineData
65f06713
TK		 1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * Adjunct processor matrix VFIO device driver callbacks.
4 *
5 * Copyright IBM Corp. 2018
6 *
7 * Author(s): Tony Krowiak <akrowiak@linux.ibm.com>
8 * Halil Pasic <pasic@linux.ibm.com>
9 * Pierre Morel <pmorel@linux.ibm.com>
10 */
11#include <linux/string.h>
12#include <linux/vfio.h>
13#include <linux/device.h>
14#include <linux/list.h>
15#include <linux/ctype.h>
258287c9
TK		 16#include <linux/bitops.h>
17#include <linux/kvm_host.h>
18#include <linux/module.h>
68f554b7 19#include <linux/uuid.h>
258287c9 20#include <asm/kvm.h>
65f06713
TK		 21#include <asm/zcrypt.h>
22
23#include "vfio_ap_private.h"
68f554b7 24#include "vfio_ap_debug.h"
65f06713
TK		 25
26#define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough"
27#define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"
28
260f3ea1
TK		 29#define AP_QUEUE_ASSIGNED "assigned"
30#define AP_QUEUE_UNASSIGNED "unassigned"
31#define AP_QUEUE_IN_USE "in use"
32
7cb7636a
TK		 33#define AP_RESET_INTERVAL 20 /* Reset sleep interval (20ms) */
34
f848cba7
TK		 35static int vfio_ap_mdev_reset_queues(struct ap_matrix_mdev *matrix_mdev);
36static int vfio_ap_mdev_reset_qlist(struct list_head *qlist);
6c12a638 37static struct vfio_ap_queue *vfio_ap_find_queue(int apqn);
eb0feefd 38static const struct vfio_device_ops vfio_ap_matrix_dev_ops;
9261f043 39static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q);
ec89b55e 40
b84eb8e0
TK		 41/**
42 * get_update_locks_for_kvm: Acquire the locks required to dynamically update a
43 * KVM guest's APCB in the proper order.
44 *
45 * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB.
46 *
47 * The proper locking order is:
48 * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM
49 * guest's APCB.
50 * 2. kvm->lock: required to update a guest's APCB
51 * 3. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev
52 *
53 * Note: If @kvm is NULL, the KVM lock will not be taken.
54 */
55static inline void get_update_locks_for_kvm(struct kvm *kvm)
ec89b55e 56{
b84eb8e0
TK		 57 mutex_lock(&matrix_dev->guests_lock);
58 if (kvm)
59 mutex_lock(&kvm->lock);
60 mutex_lock(&matrix_dev->mdevs_lock);
61}
ec89b55e 62
b84eb8e0
TK		 63/**
64 * release_update_locks_for_kvm: Release the locks used to dynamically update a
65 * KVM guest's APCB in the proper order.
66 *
67 * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB.
68 *
69 * The proper unlocking order is:
70 * 1. matrix_dev->mdevs_lock
71 * 2. kvm->lock
72 * 3. matrix_dev->guests_lock
73 *
74 * Note: If @kvm is NULL, the KVM lock will not be released.
75 */
76static inline void release_update_locks_for_kvm(struct kvm *kvm)
77{
78 mutex_unlock(&matrix_dev->mdevs_lock);
79 if (kvm)
80 mutex_unlock(&kvm->lock);
81 mutex_unlock(&matrix_dev->guests_lock);
ec89b55e
PM		 82}
83
84/**
8ee13ad9
TK		 85 * get_update_locks_for_mdev: Acquire the locks required to dynamically update a
86 * KVM guest's APCB in the proper order.
87 *
88 * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP
89 * configuration data to use to update a KVM guest's APCB.
ec89b55e 90 *
8ee13ad9
TK		 91 * The proper locking order is:
92 * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM
93 * guest's APCB.
94 * 2. matrix_mdev->kvm->lock: required to update a guest's APCB
95 * 3. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev
ec89b55e 96 *
8ee13ad9
TK		 97 * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM
98 * lock will not be taken.
ec89b55e 99 */
8ee13ad9
TK		 100static inline void get_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev)
101{
102 mutex_lock(&matrix_dev->guests_lock);
103 if (matrix_mdev && matrix_mdev->kvm)
104 mutex_lock(&matrix_mdev->kvm->lock);
105 mutex_lock(&matrix_dev->mdevs_lock);
106}
107
108/**
109 * release_update_locks_for_mdev: Release the locks used to dynamically update a
110 * KVM guest's APCB in the proper order.
111 *
112 * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP
113 * configuration data to use to update a KVM guest's APCB.
114 *
115 * The proper unlocking order is:
116 * 1. matrix_dev->mdevs_lock
117 * 2. matrix_mdev->kvm->lock
118 * 3. matrix_dev->guests_lock
119 *
120 * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM
121 * lock will not be released.
122 */
123static inline void release_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev)
124{
125 mutex_unlock(&matrix_dev->mdevs_lock);
126 if (matrix_mdev && matrix_mdev->kvm)
127 mutex_unlock(&matrix_mdev->kvm->lock);
128 mutex_unlock(&matrix_dev->guests_lock);
129}
130
2c1ee898
TK		 131/**
132 * get_update_locks_by_apqn: Find the mdev to which an APQN is assigned and
133 * acquire the locks required to update the APCB of
134 * the KVM guest to which the mdev is attached.
135 *
136 * @apqn: the APQN of a queue device.
137 *
138 * The proper locking order is:
139 * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM
140 * guest's APCB.
141 * 2. matrix_mdev->kvm->lock: required to update a guest's APCB
142 * 3. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev
143 *
144 * Note: If @apqn is not assigned to a matrix_mdev, the matrix_mdev->kvm->lock
145 * will not be taken.
146 *
147 * Return: the ap_matrix_mdev object to which @apqn is assigned or NULL if @apqn
148 * is not assigned to an ap_matrix_mdev.
149 */
150static struct ap_matrix_mdev *get_update_locks_by_apqn(int apqn)
151{
152 struct ap_matrix_mdev *matrix_mdev;
153
154 mutex_lock(&matrix_dev->guests_lock);
155
156 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
157 if (test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm) &&
158 test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm)) {
159 if (matrix_mdev->kvm)
160 mutex_lock(&matrix_mdev->kvm->lock);
161
162 mutex_lock(&matrix_dev->mdevs_lock);
163
164 return matrix_mdev;
165 }
166 }
167
168 mutex_lock(&matrix_dev->mdevs_lock);
169
170 return NULL;
171}
172
173/**
174 * get_update_locks_for_queue: get the locks required to update the APCB of the
175 * KVM guest to which the matrix mdev linked to a
176 * vfio_ap_queue object is attached.
177 *
178 * @q: a pointer to a vfio_ap_queue object.
179 *
180 * The proper locking order is:
181 * 1. q->matrix_dev->guests_lock: required to use the KVM pointer to update a
182 * KVM guest's APCB.
183 * 2. q->matrix_mdev->kvm->lock: required to update a guest's APCB
184 * 3. matrix_dev->mdevs_lock: required to access data stored in matrix_mdev
185 *
186 * Note: if @queue is not linked to an ap_matrix_mdev object, the KVM lock
187 * will not be taken.
188 */
189static inline void get_update_locks_for_queue(struct vfio_ap_queue *q)
190{
191 mutex_lock(&matrix_dev->guests_lock);
192 if (q->matrix_mdev && q->matrix_mdev->kvm)
193 mutex_lock(&q->matrix_mdev->kvm->lock);
194 mutex_lock(&matrix_dev->mdevs_lock);
195}
196
ec89b55e 197/**
11cb2419
TK		 198 * vfio_ap_mdev_get_queue - retrieve a queue with a specific APQN from a
199 * hash table of queues assigned to a matrix mdev
200 * @matrix_mdev: the matrix mdev
201 * @apqn: The APQN of a queue device
ec89b55e 202 *
11cb2419
TK		 203 * Return: the pointer to the vfio_ap_queue struct representing the queue or
204 * NULL if the queue is not assigned to @matrix_mdev
ec89b55e 205 */
11cb2419 206static struct vfio_ap_queue *vfio_ap_mdev_get_queue(
ec89b55e
PM		 207 struct ap_matrix_mdev *matrix_mdev,
208 int apqn)
209{
210 struct vfio_ap_queue *q;
ec89b55e 211
11cb2419
TK		 212 hash_for_each_possible(matrix_mdev->qtable.queues, q, mdev_qnode,
213 apqn) {
214 if (q && q->apqn == apqn)
215 return q;
216 }
ec89b55e 217
11cb2419 218 return NULL;
ec89b55e
PM		 219}
220
221/**
0c1abe7c 222 * vfio_ap_wait_for_irqclear - clears the IR bit or gives up after 5 tries
ec89b55e
PM		 223 * @apqn: The AP Queue number
224 *
225 * Checks the IRQ bit for the status of this APQN using ap_tapq.
226 * Returns if the ap_tapq function succeeded and the bit is clear.
227 * Returns if ap_tapq function failed with invalid, deconfigured or
228 * checkstopped AP.
229 * Otherwise retries up to 5 times after waiting 20ms.
ec89b55e
PM		 230 */
231static void vfio_ap_wait_for_irqclear(int apqn)
232{
233 struct ap_queue_status status;
234 int retry = 5;
235
236 do {
237 status = ap_tapq(apqn, NULL);
238 switch (status.response_code) {
239 case AP_RESPONSE_NORMAL:
240 case AP_RESPONSE_RESET_IN_PROGRESS:
241 if (!status.irq_enabled)
242 return;
0696178e 243 fallthrough;
ec89b55e
PM		 244 case AP_RESPONSE_BUSY:
245 msleep(20);
246 break;
247 case AP_RESPONSE_Q_NOT_AVAIL:
248 case AP_RESPONSE_DECONFIGURED:
249 case AP_RESPONSE_CHECKSTOPPED:
250 default:
251 WARN_ONCE(1, "%s: tapq rc %02x: %04x\n", __func__,
252 status.response_code, apqn);
253 return;
254 }
255 } while (--retry);
256
257 WARN_ONCE(1, "%s: tapq rc %02x: %04x could not clear IR bit\n",
258 __func__, status.response_code, apqn);
259}
260
261/**
0c1abe7c 262 * vfio_ap_free_aqic_resources - free vfio_ap_queue resources
ec89b55e
PM		 263 * @q: The vfio_ap_queue
264 *
265 * Unregisters the ISC in the GIB when the saved ISC not invalid.
0c1abe7c 266 * Unpins the guest's page holding the NIB when it exists.
3fad3a26 267 * Resets the saved_iova and saved_isc to invalid values.
ec89b55e
PM		 268 */
269static void vfio_ap_free_aqic_resources(struct vfio_ap_queue *q)
270{
6c12a638
TK		 271 if (!q)
272 return;
273 if (q->saved_isc != VFIO_AP_ISC_INVALID &&
274 !WARN_ON(!(q->matrix_mdev && q->matrix_mdev->kvm))) {
ec89b55e 275 kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc);
6c12a638
TK		 276 q->saved_isc = VFIO_AP_ISC_INVALID;
277 }
3fad3a26
NC		 278 if (q->saved_iova && !WARN_ON(!q->matrix_mdev)) {
279 vfio_unpin_pages(&q->matrix_mdev->vdev, q->saved_iova, 1);
280 q->saved_iova = 0;
6c12a638 281 }
ec89b55e
PM		 282}
283
284/**
0c1abe7c 285 * vfio_ap_irq_disable - disables and clears an ap_queue interrupt
ec89b55e
PM		 286 * @q: The vfio_ap_queue
287 *
288 * Uses ap_aqic to disable the interruption and in case of success, reset
289 * in progress or IRQ disable command already proceeded: calls
290 * vfio_ap_wait_for_irqclear() to check for the IRQ bit to be clear
291 * and calls vfio_ap_free_aqic_resources() to free the resources associated
292 * with the AP interrupt handling.
293 *
294 * In the case the AP is busy, or a reset is in progress,
295 * retries after 20ms, up to 5 times.
296 *
297 * Returns if ap_aqic function failed with invalid, deconfigured or
298 * checkstopped AP.
0c1abe7c
RD		 299 *
300 * Return: &struct ap_queue_status
ec89b55e 301 */
6c12a638 302static struct ap_queue_status vfio_ap_irq_disable(struct vfio_ap_queue *q)
ec89b55e 303{
ebf95e88 304 union ap_qirq_ctrl aqic_gisa = { .value = 0 };
ec89b55e
PM		 305 struct ap_queue_status status;
306 int retries = 5;
307
308 do {
10e19d49 309 status = ap_aqic(q->apqn, aqic_gisa, 0);
ec89b55e
PM		 310 switch (status.response_code) {
311 case AP_RESPONSE_OTHERWISE_CHANGED:
312 case AP_RESPONSE_NORMAL:
313 vfio_ap_wait_for_irqclear(q->apqn);
314 goto end_free;
315 case AP_RESPONSE_RESET_IN_PROGRESS:
316 case AP_RESPONSE_BUSY:
317 msleep(20);
318 break;
319 case AP_RESPONSE_Q_NOT_AVAIL:
320 case AP_RESPONSE_DECONFIGURED:
321 case AP_RESPONSE_CHECKSTOPPED:
322 case AP_RESPONSE_INVALID_ADDRESS:
323 default:
324 /* All cases in default means AP not operational */
325 WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__,
326 status.response_code);
327 goto end_free;
328 }
329 } while (retries--);
330
331 WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__,
332 status.response_code);
333end_free:
334 vfio_ap_free_aqic_resources(q);
335 return status;
336}
337
783f0a3c
TK		 338/**
339 * vfio_ap_validate_nib - validate a notification indicator byte (nib) address.
340 *
341 * @vcpu: the object representing the vcpu executing the PQAP(AQIC) instruction.
342 * @nib: the location for storing the nib address.
783f0a3c
TK		 343 *
344 * When the PQAP(AQIC) instruction is executed, general register 2 contains the
345 * address of the notification indicator byte (nib) used for IRQ notification.
3fad3a26 346 * This function parses and validates the nib from gr2.
783f0a3c
TK		 347 *
348 * Return: returns zero if the nib address is a valid; otherwise, returns
349 * -EINVAL.
350 */
3fad3a26 351static int vfio_ap_validate_nib(struct kvm_vcpu *vcpu, dma_addr_t *nib)
783f0a3c
TK		 352{
353 *nib = vcpu->run->s.regs.gprs[2];
783f0a3c 354
a64a6d23
HP		 355 if (!*nib)
356 return -EINVAL;
3fad3a26 357 if (kvm_is_error_hva(gfn_to_hva(vcpu->kvm, *nib >> PAGE_SHIFT)))
783f0a3c
TK		 358 return -EINVAL;
359
360 return 0;
361}
362
9c7260b5
CI		 363/**
364 * ensure_nib_shared() - Ensure the address of the NIB is secure and shared
365 * @addr: the physical (absolute) address of the NIB
366 *
367 * This function checks whether the NIB page, which has been pinned with
368 * vfio_pin_pages(), is a shared page belonging to a secure guest.
369 *
370 * It will call uv_pin_shared() on it; if the page was already pinned shared
371 * (i.e. if the NIB belongs to a secure guest and is shared), then 0
372 * (success) is returned. If the NIB was not shared, vfio_pin_pages() had
373 * exported it and now it does not belong to the secure guest anymore. In
374 * that case, an error is returned.
375 *
376 * Context: the NIB (at physical address @addr) has to be pinned with
377 * vfio_pin_pages() before calling this function.
378 *
379 * Return: 0 in case of success, otherwise an error < 0.
380 */
381static int ensure_nib_shared(unsigned long addr)
f88fb133 382{
f88fb133
TK		 383 /*
384 * The nib has to be located in shared storage since guest and
385 * host access it. vfio_pin_pages() will do a pin shared and
386 * if that fails (possibly because it's not a shared page) it
387 * calls export. We try to do a second pin shared here so that
388 * the UV gives us an error code if we try to pin a non-shared
389 * page.
390 *
391 * If the page is already pinned shared the UV will return a success.
392 */
9c7260b5 393 return uv_pin_shared(addr);
f88fb133
TK		 394}
395
ec89b55e 396/**
0c1abe7c 397 * vfio_ap_irq_enable - Enable Interruption for a APQN
ec89b55e 398 *
ec89b55e 399 * @q: the vfio_ap_queue holding AQIC parameters
5ef4f710 400 * @isc: the guest ISC to register with the GIB interface
783f0a3c
TK		 401 * @vcpu: the vcpu object containing the registers specifying the parameters
402 * passed to the PQAP(AQIC) instruction.
ec89b55e
PM		 403 *
404 * Pin the NIB saved in *q
405 * Register the guest ISC to GIB interface and retrieve the
406 * host ISC to issue the host side PQAP/AQIC
407 *
3746d48c
AK		 408 * status.response_code may be set to AP_RESPONSE_INVALID_ADDRESS in case the
409 * vfio_pin_pages or kvm_s390_gisc_register failed.
ec89b55e
PM		 410 *
411 * Otherwise return the ap_queue_status returned by the ap_aqic(),
412 * all retry handling will be done by the guest.
0c1abe7c
RD		 413 *
414 * Return: &struct ap_queue_status
ec89b55e
PM		 415 */
416static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q,
417 int isc,
783f0a3c 418 struct kvm_vcpu *vcpu)
ec89b55e 419{
ebf95e88 420 union ap_qirq_ctrl aqic_gisa = { .value = 0 };
ec89b55e
PM		 421 struct ap_queue_status status = {};
422 struct kvm_s390_gisa *gisa;
34a255e6 423 struct page *h_page;
783f0a3c 424 int nisc;
ec89b55e 425 struct kvm *kvm;
10e19d49 426 phys_addr_t h_nib;
3fad3a26 427 dma_addr_t nib;
ec89b55e
PM		 428 int ret;
429
783f0a3c 430 /* Verify that the notification indicator byte address is valid */
3fad3a26
NC		 431 if (vfio_ap_validate_nib(vcpu, &nib)) {
432 VFIO_AP_DBF_WARN("%s: invalid NIB address: nib=%pad, apqn=%#04x\n",
433 __func__, &nib, q->apqn);
783f0a3c
TK		 434
435 status.response_code = AP_RESPONSE_INVALID_ADDRESS;
436 return status;
437 }
438
9c7260b5 439 /* The pin will probably be successful even if the NIB was not shared */
3fad3a26 440 ret = vfio_pin_pages(&q->matrix_mdev->vdev, nib, 1,
34a255e6 441 IOMMU_READ | IOMMU_WRITE, &h_page);
ec89b55e
PM		 442 switch (ret) {
443 case 1:
444 break;
445 default:
783f0a3c 446 VFIO_AP_DBF_WARN("%s: vfio_pin_pages failed: rc=%d,"
3fad3a26
NC		 447 "nib=%pad, apqn=%#04x\n",
448 __func__, ret, &nib, q->apqn);
783f0a3c 449
ec89b55e
PM		 450 status.response_code = AP_RESPONSE_INVALID_ADDRESS;
451 return status;
452 }
453
454 kvm = q->matrix_mdev->kvm;
455 gisa = kvm->arch.gisa_int.origin;
456
34a255e6 457 h_nib = page_to_phys(h_page) | (nib & ~PAGE_MASK);
ec89b55e 458 aqic_gisa.gisc = isc;
783f0a3c 459
f88fb133
TK		 460 /* NIB in non-shared storage is a rc 6 for PV guests */
461 if (kvm_s390_pv_cpu_is_protected(vcpu) &&
9c7260b5 462 ensure_nib_shared(h_nib & PAGE_MASK)) {
f88fb133
TK		 463 vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1);
464 status.response_code = AP_RESPONSE_INVALID_ADDRESS;
465 return status;
466 }
467
783f0a3c
TK		 468 nisc = kvm_s390_gisc_register(kvm, isc);
469 if (nisc < 0) {
470 VFIO_AP_DBF_WARN("%s: gisc registration failed: nisc=%d, isc=%d, apqn=%#04x\n",
471 __func__, nisc, isc, q->apqn);
472
7b2d039d 473 vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1);
3746d48c 474 status.response_code = AP_RESPONSE_INVALID_ADDRESS;
783f0a3c
TK		 475 return status;
476 }
477
478 aqic_gisa.isc = nisc;
ec89b55e 479 aqic_gisa.ir = 1;
dbec2800 480 aqic_gisa.gisa = virt_to_phys(gisa) >> 4;
ec89b55e 481
10e19d49 482 status = ap_aqic(q->apqn, aqic_gisa, h_nib);
ec89b55e
PM		 483 switch (status.response_code) {
484 case AP_RESPONSE_NORMAL:
485 /* See if we did clear older IRQ configuration */
486 vfio_ap_free_aqic_resources(q);
3fad3a26 487 q->saved_iova = nib;
ec89b55e
PM		 488 q->saved_isc = isc;
489 break;
490 case AP_RESPONSE_OTHERWISE_CHANGED:
cada938a 491 /* We could not modify IRQ settings: clear new configuration */
c44ce579
TK		 492 ret = kvm_s390_gisc_unregister(kvm, isc);
493 if (ret)
494 VFIO_AP_DBF_WARN("%s: kvm_s390_gisc_unregister: rc=%d isc=%d, apqn=%#04x\n",
495 __func__, ret, isc, q->apqn);
3fad3a26 496 vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1);
ec89b55e
PM		 497 break;
498 default:
499 pr_warn("%s: apqn %04x: response: %02x\n", __func__, q->apqn,
500 status.response_code);
501 vfio_ap_irq_disable(q);
502 break;
503 }
504
783f0a3c
TK		 505 if (status.response_code != AP_RESPONSE_NORMAL) {
506 VFIO_AP_DBF_WARN("%s: PQAP(AQIC) failed with status=%#02x: "
507 "zone=%#x, ir=%#x, gisc=%#x, f=%#x,"
508 "gisa=%#x, isc=%#x, apqn=%#04x\n",
509 __func__, status.response_code,
510 aqic_gisa.zone, aqic_gisa.ir, aqic_gisa.gisc,
511 aqic_gisa.gf, aqic_gisa.gisa, aqic_gisa.isc,
512 q->apqn);
513 }
514
ec89b55e
PM		 515 return status;
516}
517
68f554b7
TK		 518/**
519 * vfio_ap_le_guid_to_be_uuid - convert a little endian guid array into an array
520 * of big endian elements that can be passed by
521 * value to an s390dbf sprintf event function to
522 * format a UUID string.
523 *
524 * @guid: the object containing the little endian guid
525 * @uuid: a six-element array of long values that can be passed by value as
526 * arguments for a formatting string specifying a UUID.
527 *
528 * The S390 Debug Feature (s390dbf) allows the use of "%s" in the sprintf
529 * event functions if the memory for the passed string is available as long as
530 * the debug feature exists. Since a mediated device can be removed at any
531 * time, it's name can not be used because %s passes the reference to the string
532 * in memory and the reference will go stale once the device is removed .
533 *
534 * The s390dbf string formatting function allows a maximum of 9 arguments for a
535 * message to be displayed in the 'sprintf' view. In order to use the bytes
536 * comprising the mediated device's UUID to display the mediated device name,
537 * they will have to be converted into an array whose elements can be passed by
538 * value to sprintf. For example:
539 *
540 * guid array: { 83, 78, 17, 62, bb, f1, f0, 47, 91, 4d, 32, a2, 2e, 3a, 88, 04 }
541 * mdev name: 62177883-f1bb-47f0-914d-32a22e3a8804
542 * array returned: { 62177883, f1bb, 47f0, 914d, 32a2, 2e3a8804 }
543 * formatting string: "%08lx-%04lx-%04lx-%04lx-%02lx%04lx"
544 */
545static void vfio_ap_le_guid_to_be_uuid(guid_t *guid, unsigned long *uuid)
546{
547 /*
548 * The input guid is ordered in little endian, so it needs to be
549 * reordered for displaying a UUID as a string. This specifies the
550 * guid indices in proper order.
551 */
552 uuid[0] = le32_to_cpup((__le32 *)guid);
553 uuid[1] = le16_to_cpup((__le16 *)&guid->b[4]);
554 uuid[2] = le16_to_cpup((__le16 *)&guid->b[6]);
555 uuid[3] = *((__u16 *)&guid->b[8]);
556 uuid[4] = *((__u16 *)&guid->b[10]);
557 uuid[5] = *((__u32 *)&guid->b[12]);
558}
559
ec89b55e 560/**
0c1abe7c 561 * handle_pqap - PQAP instruction callback
ec89b55e
PM		 562 *
563 * @vcpu: The vcpu on which we received the PQAP instruction
564 *
565 * Get the general register contents to initialize internal variables.
566 * REG[0]: APQN
567 * REG[1]: IR and ISC
568 * REG[2]: NIB
569 *
570 * Response.status may be set to following Response Code:
571 * - AP_RESPONSE_Q_NOT_AVAIL: if the queue is not available
572 * - AP_RESPONSE_DECONFIGURED: if the queue is not configured
cada938a 573 * - AP_RESPONSE_NORMAL (0) : in case of success
ec89b55e
PM		 574 * Check vfio_ap_setirq() and vfio_ap_clrirq() for other possible RC.
575 * We take the matrix_dev lock to ensure serialization on queues and
576 * mediated device access.
577 *
0c1abe7c
RD		 578 * Return: 0 if we could handle the request inside KVM.
579 * Otherwise, returns -EOPNOTSUPP to let QEMU handle the fault.
ec89b55e
PM		 580 */
581static int handle_pqap(struct kvm_vcpu *vcpu)
582{
583 uint64_t status;
584 uint16_t apqn;
68f554b7 585 unsigned long uuid[6];
ec89b55e
PM		 586 struct vfio_ap_queue *q;
587 struct ap_queue_status qstatus = {
588 .response_code = AP_RESPONSE_Q_NOT_AVAIL, };
589 struct ap_matrix_mdev *matrix_mdev;
590
68f554b7
TK		 591 apqn = vcpu->run->s.regs.gprs[0] & 0xffff;
592
ec89b55e 593 /* If we do not use the AIV facility just go to userland */
68f554b7
TK		 594 if (!(vcpu->arch.sie_block->eca & ECA_AIV)) {
595 VFIO_AP_DBF_WARN("%s: AIV facility not installed: apqn=0x%04x, eca=0x%04x\n",
596 __func__, apqn, vcpu->arch.sie_block->eca);
597
ec89b55e 598 return -EOPNOTSUPP;
68f554b7 599 }
ec89b55e 600
d0786556
TK		 601 mutex_lock(&matrix_dev->mdevs_lock);
602
68f554b7
TK		 603 if (!vcpu->kvm->arch.crypto.pqap_hook) {
604 VFIO_AP_DBF_WARN("%s: PQAP(AQIC) hook not registered with the vfio_ap driver: apqn=0x%04x\n",
605 __func__, apqn);
d0786556 606
ec89b55e 607 goto out_unlock;
68f554b7
TK		 608 }
609
ec89b55e
PM		 610 matrix_mdev = container_of(vcpu->kvm->arch.crypto.pqap_hook,
611 struct ap_matrix_mdev, pqap_hook);
612
0cc00c8d 613 /* If the there is no guest using the mdev, there is nothing to do */
68f554b7
TK		 614 if (!matrix_mdev->kvm) {
615 vfio_ap_le_guid_to_be_uuid(&matrix_mdev->mdev->uuid, uuid);
616 VFIO_AP_DBF_WARN("%s: mdev %08lx-%04lx-%04lx-%04lx-%04lx%08lx not in use: apqn=0x%04x\n",
617 __func__, uuid[0], uuid[1], uuid[2],
618 uuid[3], uuid[4], uuid[5], apqn);
0cc00c8d 619 goto out_unlock;
68f554b7 620 }
0cc00c8d 621
11cb2419 622 q = vfio_ap_mdev_get_queue(matrix_mdev, apqn);
68f554b7
TK		 623 if (!q) {
624 VFIO_AP_DBF_WARN("%s: Queue %02x.%04x not bound to the vfio_ap driver\n",
625 __func__, AP_QID_CARD(apqn),
626 AP_QID_QUEUE(apqn));
ec89b55e 627 goto out_unlock;
68f554b7 628 }
ec89b55e
PM		 629
630 status = vcpu->run->s.regs.gprs[1];
631
632 /* If IR bit(16) is set we enable the interrupt */
633 if ((status >> (63 - 16)) & 0x01)
783f0a3c 634 qstatus = vfio_ap_irq_enable(q, status & 0x07, vcpu);
ec89b55e
PM		 635 else
636 qstatus = vfio_ap_irq_disable(q);
637
638out_unlock:
639 memcpy(&vcpu->run->s.regs.gprs[1], &qstatus, sizeof(qstatus));
640 vcpu->run->s.regs.gprs[1] >>= 32;
d0786556 641 mutex_unlock(&matrix_dev->mdevs_lock);
ec89b55e
PM		 642 return 0;
643}
644
65f06713
TK		 645static void vfio_ap_matrix_init(struct ap_config_info *info,
646 struct ap_matrix *matrix)
647{
f6047040
HF		 648 matrix->apm_max = info->apxa ? info->na : 63;
649 matrix->aqm_max = info->apxa ? info->nd : 15;
650 matrix->adm_max = info->apxa ? info->nd : 15;
65f06713
TK		 651}
652
07d89045
RR		 653static void signal_guest_ap_cfg_changed(struct ap_matrix_mdev *matrix_mdev)
654{
cbf367d5
RR		 655 if (matrix_mdev->cfg_chg_trigger)
656 eventfd_signal(matrix_mdev->cfg_chg_trigger);
07d89045
RR		 657}
658
51dc562a 659static void vfio_ap_mdev_update_guest_apcb(struct ap_matrix_mdev *matrix_mdev)
48cae940 660{
07d89045 661 if (matrix_mdev->kvm) {
51dc562a
TK		 662 kvm_arch_crypto_set_masks(matrix_mdev->kvm,
663 matrix_mdev->shadow_apcb.apm,
664 matrix_mdev->shadow_apcb.aqm,
665 matrix_mdev->shadow_apcb.adm);
07d89045
RR		 666
667 signal_guest_ap_cfg_changed(matrix_mdev);
668 }
51dc562a
TK		 669}
670
671static bool vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev)
672{
673 DECLARE_BITMAP(prev_shadow_adm, AP_DOMAINS);
674
675 bitmap_copy(prev_shadow_adm, matrix_mdev->shadow_apcb.adm, AP_DOMAINS);
48cae940
TK		 676 bitmap_and(matrix_mdev->shadow_apcb.adm, matrix_mdev->matrix.adm,
677 (unsigned long *)matrix_dev->info.adm, AP_DOMAINS);
51dc562a
TK		 678
679 return !bitmap_equal(prev_shadow_adm, matrix_mdev->shadow_apcb.adm,
680 AP_DOMAINS);
48cae940
TK		 681}
682
a681226c
JH		 683static bool _queue_passable(struct vfio_ap_queue *q)
684{
685 if (!q)
686 return false;
687
688 switch (q->reset_status.response_code) {
689 case AP_RESPONSE_NORMAL:
690 case AP_RESPONSE_DECONFIGURED:
691 case AP_RESPONSE_CHECKSTOPPED:
692 return true;
693 default:
694 return false;
695 }
696}
697
48cae940
TK		 698/*
699 * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev
700 * to ensure no queue devices are passed through to
701 * the guest that are not bound to the vfio_ap
702 * device driver.
703 *
704 * @matrix_mdev: the matrix mdev whose matrix is to be filtered.
f848cba7
TK		 705 * @apm_filtered: a 256-bit bitmap for storing the APIDs filtered from the
706 * guest's AP configuration that are still in the host's AP
707 * configuration.
48cae940
TK		 708 *
709 * Note: If an APQN referencing a queue device that is not bound to the vfio_ap
710 * driver, its APID will be filtered from the guest's APCB. The matrix
711 * structure precludes filtering an individual APQN, so its APID will be
f848cba7
TK		 712 * filtered. Consequently, all queues associated with the adapter that
713 * are in the host's AP configuration must be reset. If queues are
714 * subsequently made available again to the guest, they should re-appear
715 * in a reset state
51dc562a
TK		 716 *
717 * Return: a boolean value indicating whether the KVM guest's APCB was changed
718 * by the filtering or not.
48cae940 719 */
f848cba7
TK		 720static bool vfio_ap_mdev_filter_matrix(struct ap_matrix_mdev *matrix_mdev,
721 unsigned long *apm_filtered)
48cae940 722{
48cae940 723 unsigned long apid, apqi, apqn;
51dc562a
TK		 724 DECLARE_BITMAP(prev_shadow_apm, AP_DEVICES);
725 DECLARE_BITMAP(prev_shadow_aqm, AP_DOMAINS);
48cae940 726
51dc562a
TK		 727 bitmap_copy(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, AP_DEVICES);
728 bitmap_copy(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS);
48cae940 729 vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb);
f848cba7 730 bitmap_clear(apm_filtered, 0, AP_DEVICES);
48cae940
TK		 731
732 /*
733 * Copy the adapters, domains and control domains to the shadow_apcb
734 * from the matrix mdev, but only those that are assigned to the host's
735 * AP configuration.
736 */
737 bitmap_and(matrix_mdev->shadow_apcb.apm, matrix_mdev->matrix.apm,
738 (unsigned long *)matrix_dev->info.apm, AP_DEVICES);
739 bitmap_and(matrix_mdev->shadow_apcb.aqm, matrix_mdev->matrix.aqm,
740 (unsigned long *)matrix_dev->info.aqm, AP_DOMAINS);
741
16fb78cb
TK		 742 for_each_set_bit_inv(apid, matrix_mdev->shadow_apcb.apm, AP_DEVICES) {
743 for_each_set_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm,
744 AP_DOMAINS) {
48cae940
TK		 745 /*
746 * If the APQN is not bound to the vfio_ap device
747 * driver, then we can't assign it to the guest's
748 * AP configuration. The AP architecture won't
749 * allow filtering of a single APQN, so let's filter
750 * the APID since an adapter represents a physical
751 * hardware device.
752 */
753 apqn = AP_MKQID(apid, apqi);
a681226c 754 if (!_queue_passable(vfio_ap_mdev_get_queue(matrix_mdev, apqn))) {
f848cba7
TK		 755 clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm);
756
757 /*
758 * If the adapter was previously plugged into
759 * the guest, let's let the caller know that
760 * the APID was filtered.
761 */
762 if (test_bit_inv(apid, prev_shadow_apm))
763 set_bit_inv(apid, apm_filtered);
764
48cae940
TK		 765 break;
766 }
767 }
768 }
51dc562a
TK		 769
770 return !bitmap_equal(prev_shadow_apm, matrix_mdev->shadow_apcb.apm,
771 AP_DEVICES) ||
772 !bitmap_equal(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm,
773 AP_DOMAINS);
48cae940
TK		 774}
775
7cb5a82e 776static int vfio_ap_mdev_init_dev(struct vfio_device *vdev)
65f06713 777{
7cb5a82e
YL		 778 struct ap_matrix_mdev *matrix_mdev =
779 container_of(vdev, struct ap_matrix_mdev, vdev);
65f06713 780
7cb5a82e 781 matrix_mdev->mdev = to_mdev_device(vdev->dev);
65f06713 782 vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);
1e753732 783 matrix_mdev->pqap_hook = handle_pqap;
49b0109f 784 vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb);
11cb2419 785 hash_init(matrix_mdev->qtable.queues);
65f06713 786
7cb5a82e
YL		 787 return 0;
788}
789
790static int vfio_ap_mdev_probe(struct mdev_device *mdev)
791{
792 struct ap_matrix_mdev *matrix_mdev;
793 int ret;
794
795 matrix_mdev = vfio_alloc_device(ap_matrix_mdev, vdev, &mdev->dev,
796 &vfio_ap_matrix_dev_ops);
797 if (IS_ERR(matrix_mdev))
798 return PTR_ERR(matrix_mdev);
799
c68ea0d0 800 ret = vfio_register_emulated_iommu_dev(&matrix_mdev->vdev);
eb0feefd 801 if (ret)
7cb5a82e 802 goto err_put_vdev;
bf48961f 803 matrix_mdev->req_trigger = NULL;
07d89045 804 matrix_mdev->cfg_chg_trigger = NULL;
eb0feefd 805 dev_set_drvdata(&mdev->dev, matrix_mdev);
2c1ee898
TK		 806 mutex_lock(&matrix_dev->mdevs_lock);
807 list_add(&matrix_mdev->node, &matrix_dev->mdev_list);
808 mutex_unlock(&matrix_dev->mdevs_lock);
65f06713 809 return 0;
eb0feefd 810
7cb5a82e
YL		 811err_put_vdev:
812 vfio_put_device(&matrix_mdev->vdev);
eb0feefd 813 return ret;
65f06713
TK		 814}
815
11cb2419
TK		 816static void vfio_ap_mdev_link_queue(struct ap_matrix_mdev *matrix_mdev,
817 struct vfio_ap_queue *q)
818{
f3e3a400
JH		 819 if (!q || vfio_ap_mdev_get_queue(matrix_mdev, q->apqn))
820 return;
821
822 q->matrix_mdev = matrix_mdev;
823 hash_add(matrix_mdev->qtable.queues, &q->mdev_qnode, q->apqn);
11cb2419
TK		 824}
825
826static void vfio_ap_mdev_link_apqn(struct ap_matrix_mdev *matrix_mdev, int apqn)
827{
828 struct vfio_ap_queue *q;
829
830 q = vfio_ap_find_queue(apqn);
831 vfio_ap_mdev_link_queue(matrix_mdev, q);
832}
833
834static void vfio_ap_unlink_queue_fr_mdev(struct vfio_ap_queue *q)
835{
836 hash_del(&q->mdev_qnode);
837}
838
839static void vfio_ap_unlink_mdev_fr_queue(struct vfio_ap_queue *q)
840{
841 q->matrix_mdev = NULL;
842}
843
11cb2419
TK		 844static void vfio_ap_mdev_unlink_fr_queues(struct ap_matrix_mdev *matrix_mdev)
845{
846 struct vfio_ap_queue *q;
847 unsigned long apid, apqi;
848
849 for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {
850 for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm,
851 AP_DOMAINS) {
852 q = vfio_ap_mdev_get_queue(matrix_mdev,
853 AP_MKQID(apid, apqi));
854 if (q)
855 q->matrix_mdev = NULL;
856 }
857 }
858}
859
eb0feefd 860static void vfio_ap_mdev_remove(struct mdev_device *mdev)
65f06713 861{
eb0feefd
JG		 862 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(&mdev->dev);
863
864 vfio_unregister_group_dev(&matrix_mdev->vdev);
65f06713 865
2c1ee898 866 mutex_lock(&matrix_dev->guests_lock);
d0786556 867 mutex_lock(&matrix_dev->mdevs_lock);
f848cba7 868 vfio_ap_mdev_reset_queues(matrix_mdev);
11cb2419 869 vfio_ap_mdev_unlink_fr_queues(matrix_mdev);
65f06713 870 list_del(&matrix_mdev->node);
d0786556 871 mutex_unlock(&matrix_dev->mdevs_lock);
2c1ee898 872 mutex_unlock(&matrix_dev->guests_lock);
7cb5a82e 873 vfio_put_device(&matrix_mdev->vdev);
65f06713
TK		 874}
875
d33d729a 876#define MDEV_SHARING_ERR "Userspace may not assign queue %02lx.%04lx to mdev: already assigned to %s"
96d152bd 877
d33d729a
AK		 878#define MDEV_IN_USE_ERR "Can not reserve queue %02lx.%04lx for host driver: in use by mdev"
879
880static void vfio_ap_mdev_log_sharing_err(struct ap_matrix_mdev *assignee,
881 struct ap_matrix_mdev *assigned_to,
882 unsigned long *apm, unsigned long *aqm)
883{
884 unsigned long apid, apqi;
885
886 for_each_set_bit_inv(apid, apm, AP_DEVICES) {
887 for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) {
888 dev_warn(mdev_dev(assignee->mdev), MDEV_SHARING_ERR,
889 apid, apqi, dev_name(mdev_dev(assigned_to->mdev)));
890 }
891 }
892}
893
894static void vfio_ap_mdev_log_in_use_err(struct ap_matrix_mdev *assignee,
895 unsigned long *apm, unsigned long *aqm)
96d152bd 896{
e2126a73 897 unsigned long apid, apqi;
96d152bd 898
d33d729a 899 for_each_set_bit_inv(apid, apm, AP_DEVICES) {
e2126a73 900 for_each_set_bit_inv(apqi, aqm, AP_DOMAINS)
d33d729a
AK		 901 dev_warn(mdev_dev(assignee->mdev), MDEV_IN_USE_ERR, apid, apqi);
902 }
96d152bd
TK		 903}
904
905/**
e2126a73 906 * vfio_ap_mdev_verify_no_sharing - verify APQNs are not shared by matrix mdevs
0c1abe7c 907 *
d33d729a
AK		 908 * @assignee: the matrix mdev to which @mdev_apm and @mdev_aqm are being
909 * assigned; or, NULL if this function was called by the AP bus
910 * driver in_use callback to verify none of the APQNs being reserved
911 * for the host device driver are in use by a vfio_ap mediated device
e2126a73
TK		 912 * @mdev_apm: mask indicating the APIDs of the APQNs to be verified
913 * @mdev_aqm: mask indicating the APQIs of the APQNs to be verified
96d152bd 914 *
d33d729a
AK		 915 * Verifies that each APQN derived from the Cartesian product of APIDs
916 * represented by the bits set in @mdev_apm and the APQIs of the bits set in
917 * @mdev_aqm is not assigned to a mediated device other than the mdev to which
918 * the APQN is being assigned (@assignee). AP queue sharing is not allowed.
96d152bd 919 *
e2126a73 920 * Return: 0 if the APQNs are not shared; otherwise return -EADDRINUSE.
96d152bd 921 */
d33d729a
AK		 922static int vfio_ap_mdev_verify_no_sharing(struct ap_matrix_mdev *assignee,
923 unsigned long *mdev_apm,
e2126a73 924 unsigned long *mdev_aqm)
96d152bd 925{
d33d729a 926 struct ap_matrix_mdev *assigned_to;
96d152bd
TK		 927 DECLARE_BITMAP(apm, AP_DEVICES);
928 DECLARE_BITMAP(aqm, AP_DOMAINS);
929
d33d729a 930 list_for_each_entry(assigned_to, &matrix_dev->mdev_list, node) {
e2126a73 931 /*
d33d729a
AK		 932 * If the mdev to which the mdev_apm and mdev_aqm is being
933 * assigned is the same as the mdev being verified
e2126a73 934 */
d33d729a 935 if (assignee == assigned_to)
96d152bd
TK		 936 continue;
937
938 memset(apm, 0, sizeof(apm));
939 memset(aqm, 0, sizeof(aqm));
940
941 /*
942 * We work on full longs, as we can only exclude the leftover
943 * bits in non-inverse order. The leftover is all zeros.
944 */
d33d729a 945 if (!bitmap_and(apm, mdev_apm, assigned_to->matrix.apm, AP_DEVICES))
96d152bd
TK		 946 continue;
947
d33d729a 948 if (!bitmap_and(aqm, mdev_aqm, assigned_to->matrix.aqm, AP_DOMAINS))
96d152bd
TK		 949 continue;
950
d33d729a
AK		 951 if (assignee)
952 vfio_ap_mdev_log_sharing_err(assignee, assigned_to, apm, aqm);
953 else
954 vfio_ap_mdev_log_in_use_err(assigned_to, apm, aqm);
e2126a73 955
96d152bd
TK		 956 return -EADDRINUSE;
957 }
958
959 return 0;
960}
961
3f85d1df
TK		 962/**
963 * vfio_ap_mdev_validate_masks - verify that the APQNs assigned to the mdev are
964 * not reserved for the default zcrypt driver and
965 * are not assigned to another mdev.
966 *
967 * @matrix_mdev: the mdev to which the APQNs being validated are assigned.
968 *
969 * Return: One of the following values:
970 * o the error returned from the ap_apqn_in_matrix_owned_by_def_drv() function,
971 * most likely -EBUSY indicating the ap_perms_mutex lock is already held.
972 * o EADDRNOTAVAIL if an APQN assigned to @matrix_mdev is reserved for the
973 * zcrypt default driver.
974 * o EADDRINUSE if an APQN assigned to @matrix_mdev is assigned to another mdev
975 * o A zero indicating validation succeeded.
976 */
e2126a73
TK		 977static int vfio_ap_mdev_validate_masks(struct ap_matrix_mdev *matrix_mdev)
978{
979 if (ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm,
980 matrix_mdev->matrix.aqm))
981 return -EADDRNOTAVAIL;
982
d33d729a
AK		 983 return vfio_ap_mdev_verify_no_sharing(matrix_mdev,
984 matrix_mdev->matrix.apm,
e2126a73
TK		 985 matrix_mdev->matrix.aqm);
986}
987
11cb2419
TK		 988static void vfio_ap_mdev_link_adapter(struct ap_matrix_mdev *matrix_mdev,
989 unsigned long apid)
990{
991 unsigned long apqi;
992
993 for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS)
994 vfio_ap_mdev_link_apqn(matrix_mdev,
995 AP_MKQID(apid, apqi));
996}
997
f009cfa4
TK		 998static void collect_queues_to_reset(struct ap_matrix_mdev *matrix_mdev,
999 unsigned long apid,
1000 struct list_head *qlist)
1001{
1002 struct vfio_ap_queue *q;
1003 unsigned long apqi;
1004
1005 for_each_set_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS) {
1006 q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi));
1007 if (q)
1008 list_add_tail(&q->reset_qnode, qlist);
1009 }
1010}
1011
1012static void reset_queues_for_apid(struct ap_matrix_mdev *matrix_mdev,
1013 unsigned long apid)
1014{
1015 struct list_head qlist;
1016
1017 INIT_LIST_HEAD(&qlist);
1018 collect_queues_to_reset(matrix_mdev, apid, &qlist);
1019 vfio_ap_mdev_reset_qlist(&qlist);
1020}
1021
f848cba7
TK		 1022static int reset_queues_for_apids(struct ap_matrix_mdev *matrix_mdev,
1023 unsigned long *apm_reset)
1024{
f848cba7 1025 struct list_head qlist;
f009cfa4 1026 unsigned long apid;
f848cba7
TK		 1027
1028 if (bitmap_empty(apm_reset, AP_DEVICES))
1029 return 0;
1030
1031 INIT_LIST_HEAD(&qlist);
1032
f009cfa4
TK		 1033 for_each_set_bit_inv(apid, apm_reset, AP_DEVICES)
1034 collect_queues_to_reset(matrix_mdev, apid, &qlist);
f848cba7 1035
f009cfa4 1036 return vfio_ap_mdev_reset_qlist(&qlist);
f848cba7
TK		 1037}
1038
96d152bd 1039/**
0c1abe7c
RD		 1040 * assign_adapter_store - parses the APID from @buf and sets the
1041 * corresponding bit in the mediated matrix device's APM
96d152bd
TK		 1042 *
1043 * @dev: the matrix device
1044 * @attr: the mediated matrix device's assign_adapter attribute
1045 * @buf: a buffer containing the AP adapter number (APID) to
1046 * be assigned
1047 * @count: the number of bytes in @buf
1048 *
0c1abe7c 1049 * Return: the number of bytes processed if the APID is valid; otherwise,
96d152bd
TK		 1050 * returns one of the following errors:
1051 *
1052 * 1. -EINVAL
1053 * The APID is not a valid number
1054 *
1055 * 2. -ENODEV
1056 * The APID exceeds the maximum value configured for the system
1057 *
1058 * 3. -EADDRNOTAVAIL
1059 * An APQN derived from the cross product of the APID being assigned
1060 * and the APQIs previously assigned is not bound to the vfio_ap device
1061 * driver; or, if no APQIs have yet been assigned, the APID is not
1062 * contained in an APQN bound to the vfio_ap device driver.
1063 *
1064 * 4. -EADDRINUSE
1065 * An APQN derived from the cross product of the APID being assigned
1066 * and the APQIs previously assigned is being used by another mediated
1067 * matrix device
3f85d1df
TK		 1068 *
1069 * 5. -EAGAIN
1070 * A lock required to validate the mdev's AP configuration could not
1071 * be obtained.
96d152bd
TK		 1072 */
1073static ssize_t assign_adapter_store(struct device *dev,
1074 struct device_attribute *attr,
1075 const char *buf, size_t count)
1076{
1077 int ret;
1078 unsigned long apid;
f848cba7 1079 DECLARE_BITMAP(apm_filtered, AP_DEVICES);
eb0feefd 1080 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
96d152bd 1081
3f85d1df 1082 mutex_lock(&ap_perms_mutex);
8ee13ad9 1083 get_update_locks_for_mdev(matrix_mdev);
258287c9 1084
96d152bd
TK		 1085 ret = kstrtoul(buf, 0, &apid);
1086 if (ret)
0cc00c8d 1087 goto done;
96d152bd 1088
0cc00c8d
TK		 1089 if (apid > matrix_mdev->matrix.apm_max) {
1090 ret = -ENODEV;
1091 goto done;
1092 }
96d152bd 1093
1918f2b2
TK		 1094 if (test_bit_inv(apid, matrix_mdev->matrix.apm)) {
1095 ret = count;
1096 goto done;
1097 }
1098
96d152bd
TK		 1099 set_bit_inv(apid, matrix_mdev->matrix.apm);
1100
e2126a73
TK		 1101 ret = vfio_ap_mdev_validate_masks(matrix_mdev);
1102 if (ret) {
1103 clear_bit_inv(apid, matrix_mdev->matrix.apm);
96d152bd 1104 goto done;
e2126a73 1105 }
96d152bd 1106
11cb2419 1107 vfio_ap_mdev_link_adapter(matrix_mdev, apid);
96d152bd 1108
f848cba7 1109 if (vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered)) {
51dc562a 1110 vfio_ap_mdev_update_guest_apcb(matrix_mdev);
f848cba7
TK		 1111 reset_queues_for_apids(matrix_mdev, apm_filtered);
1112 }
96d152bd
TK		 1113
1114 ret = count;
96d152bd 1115done:
8ee13ad9 1116 release_update_locks_for_mdev(matrix_mdev);
3f85d1df 1117 mutex_unlock(&ap_perms_mutex);
96d152bd
TK		 1118
1119 return ret;
1120}
1121static DEVICE_ATTR_WO(assign_adapter);
1122
70aeefe5
TK		 1123static struct vfio_ap_queue
1124*vfio_ap_unlink_apqn_fr_mdev(struct ap_matrix_mdev *matrix_mdev,
1125 unsigned long apid, unsigned long apqi)
1126{
1127 struct vfio_ap_queue *q = NULL;
1128
1129 q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi));
1130 /* If the queue is assigned to the matrix mdev, unlink it. */
1131 if (q)
1132 vfio_ap_unlink_queue_fr_mdev(q);
1133
1134 return q;
1135}
1136
1137/**
1138 * vfio_ap_mdev_unlink_adapter - unlink all queues associated with unassigned
1139 * adapter from the matrix mdev to which the
1140 * adapter was assigned.
1141 * @matrix_mdev: the matrix mediated device to which the adapter was assigned.
1142 * @apid: the APID of the unassigned adapter.
f848cba7
TK		 1143 * @qlist: list for storing queues associated with unassigned adapter that
1144 * need to be reset.
70aeefe5 1145 */
11cb2419 1146static void vfio_ap_mdev_unlink_adapter(struct ap_matrix_mdev *matrix_mdev,
70aeefe5 1147 unsigned long apid,
f848cba7 1148 struct list_head *qlist)
11cb2419
TK		 1149{
1150 unsigned long apqi;
1151 struct vfio_ap_queue *q;
1152
1153 for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) {
70aeefe5 1154 q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi);
11cb2419 1155
f848cba7 1156 if (q && qlist) {
70aeefe5
TK		 1157 if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
1158 test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm))
f848cba7 1159 list_add_tail(&q->reset_qnode, qlist);
70aeefe5 1160 }
11cb2419
TK		 1161 }
1162}
1163
8fb456bc
JH		 1164static void vfio_ap_mdev_hot_unplug_adapters(struct ap_matrix_mdev *matrix_mdev,
1165 unsigned long *apids)
70aeefe5 1166{
f848cba7
TK		 1167 struct vfio_ap_queue *q, *tmpq;
1168 struct list_head qlist;
8fb456bc
JH		 1169 unsigned long apid;
1170 bool apcb_update = false;
70aeefe5 1171
f848cba7 1172 INIT_LIST_HEAD(&qlist);
70aeefe5 1173
8fb456bc
JH		 1174 for_each_set_bit_inv(apid, apids, AP_DEVICES) {
1175 vfio_ap_mdev_unlink_adapter(matrix_mdev, apid, &qlist);
1176
1177 if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm)) {
1178 clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm);
1179 apcb_update = true;
1180 }
70aeefe5
TK		 1181 }
1182
8fb456bc
JH		 1183 /* Only update apcb if needed to avoid impacting guest */
1184 if (apcb_update)
1185 vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1186
f848cba7 1187 vfio_ap_mdev_reset_qlist(&qlist);
70aeefe5 1188
f848cba7 1189 list_for_each_entry_safe(q, tmpq, &qlist, reset_qnode) {
70aeefe5 1190 vfio_ap_unlink_mdev_fr_queue(q);
f848cba7 1191 list_del(&q->reset_qnode);
70aeefe5 1192 }
70aeefe5
TK		 1193}
1194
8fb456bc
JH		 1195static void vfio_ap_mdev_hot_unplug_adapter(struct ap_matrix_mdev *matrix_mdev,
1196 unsigned long apid)
1197{
1198 DECLARE_BITMAP(apids, AP_DEVICES);
1199
1200 bitmap_zero(apids, AP_DEVICES);
1201 set_bit_inv(apid, apids);
1202 vfio_ap_mdev_hot_unplug_adapters(matrix_mdev, apids);
1203}
1204
96d152bd 1205/**
0c1abe7c
RD		 1206 * unassign_adapter_store - parses the APID from @buf and clears the
1207 * corresponding bit in the mediated matrix device's APM
96d152bd
TK		 1208 *
1209 * @dev: the matrix device
1210 * @attr: the mediated matrix device's unassign_adapter attribute
1211 * @buf: a buffer containing the adapter number (APID) to be unassigned
1212 * @count: the number of bytes in @buf
1213 *
0c1abe7c 1214 * Return: the number of bytes processed if the APID is valid; otherwise,
96d152bd
TK		 1215 * returns one of the following errors:
1216 * -EINVAL if the APID is not a number
1217 * -ENODEV if the APID it exceeds the maximum value configured for the
1218 * system
1219 */
1220static ssize_t unassign_adapter_store(struct device *dev,
1221 struct device_attribute *attr,
1222 const char *buf, size_t count)
1223{
1224 int ret;
1225 unsigned long apid;
eb0feefd 1226 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
96d152bd 1227
8ee13ad9 1228 get_update_locks_for_mdev(matrix_mdev);
258287c9 1229
96d152bd
TK		 1230 ret = kstrtoul(buf, 0, &apid);
1231 if (ret)
0cc00c8d 1232 goto done;
96d152bd 1233
0cc00c8d
TK		 1234 if (apid > matrix_mdev->matrix.apm_max) {
1235 ret = -ENODEV;
1236 goto done;
1237 }
96d152bd 1238
1918f2b2
TK		 1239 if (!test_bit_inv(apid, matrix_mdev->matrix.apm)) {
1240 ret = count;
1241 goto done;
1242 }
1243
96d152bd 1244 clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);
70aeefe5 1245 vfio_ap_mdev_hot_unplug_adapter(matrix_mdev, apid);
0cc00c8d
TK		 1246 ret = count;
1247done:
8ee13ad9 1248 release_update_locks_for_mdev(matrix_mdev);
0cc00c8d 1249 return ret;
96d152bd 1250}
46623ab3 1251static DEVICE_ATTR_WO(unassign_adapter);
96d152bd 1252
11cb2419
TK		 1253static void vfio_ap_mdev_link_domain(struct ap_matrix_mdev *matrix_mdev,
1254 unsigned long apqi)
3211da0c 1255{
3211da0c 1256 unsigned long apid;
3211da0c 1257
11cb2419
TK		 1258 for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES)
1259 vfio_ap_mdev_link_apqn(matrix_mdev,
1260 AP_MKQID(apid, apqi));
3211da0c
TK		 1261}
1262
1263/**
0c1abe7c
RD		 1264 * assign_domain_store - parses the APQI from @buf and sets the
1265 * corresponding bit in the mediated matrix device's AQM
1266 *
3211da0c
TK		 1267 * @dev: the matrix device
1268 * @attr: the mediated matrix device's assign_domain attribute
1269 * @buf: a buffer containing the AP queue index (APQI) of the domain to
1270 * be assigned
1271 * @count: the number of bytes in @buf
1272 *
0c1abe7c 1273 * Return: the number of bytes processed if the APQI is valid; otherwise returns
3211da0c
TK		 1274 * one of the following errors:
1275 *
1276 * 1. -EINVAL
1277 * The APQI is not a valid number
1278 *
1279 * 2. -ENODEV
1280 * The APQI exceeds the maximum value configured for the system
1281 *
1282 * 3. -EADDRNOTAVAIL
1283 * An APQN derived from the cross product of the APQI being assigned
1284 * and the APIDs previously assigned is not bound to the vfio_ap device
1285 * driver; or, if no APIDs have yet been assigned, the APQI is not
1286 * contained in an APQN bound to the vfio_ap device driver.
1287 *
1288 * 4. -EADDRINUSE
1289 * An APQN derived from the cross product of the APQI being assigned
1290 * and the APIDs previously assigned is being used by another mediated
1291 * matrix device
3f85d1df
TK		 1292 *
1293 * 5. -EAGAIN
1294 * The lock required to validate the mdev's AP configuration could not
1295 * be obtained.
3211da0c
TK		 1296 */
1297static ssize_t assign_domain_store(struct device *dev,
1298 struct device_attribute *attr,
1299 const char *buf, size_t count)
1300{
1301 int ret;
1302 unsigned long apqi;
f848cba7 1303 DECLARE_BITMAP(apm_filtered, AP_DEVICES);
eb0feefd 1304 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
3211da0c 1305
3f85d1df 1306 mutex_lock(&ap_perms_mutex);
8ee13ad9 1307 get_update_locks_for_mdev(matrix_mdev);
258287c9 1308
3211da0c
TK		 1309 ret = kstrtoul(buf, 0, &apqi);
1310 if (ret)
0cc00c8d 1311 goto done;
51dc562a
TK		 1312
1313 if (apqi > matrix_mdev->matrix.aqm_max) {
0cc00c8d
TK		 1314 ret = -ENODEV;
1315 goto done;
1316 }
3211da0c 1317
1918f2b2
TK		 1318 if (test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {
1319 ret = count;
1320 goto done;
1321 }
1322
3211da0c
TK		 1323 set_bit_inv(apqi, matrix_mdev->matrix.aqm);
1324
e2126a73
TK		 1325 ret = vfio_ap_mdev_validate_masks(matrix_mdev);
1326 if (ret) {
1327 clear_bit_inv(apqi, matrix_mdev->matrix.aqm);
3211da0c 1328 goto done;
e2126a73 1329 }
3211da0c 1330
11cb2419 1331 vfio_ap_mdev_link_domain(matrix_mdev, apqi);
3211da0c 1332
f848cba7 1333 if (vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered)) {
51dc562a 1334 vfio_ap_mdev_update_guest_apcb(matrix_mdev);
f848cba7
TK		 1335 reset_queues_for_apids(matrix_mdev, apm_filtered);
1336 }
3211da0c
TK		 1337
1338 ret = count;
3211da0c 1339done:
8ee13ad9 1340 release_update_locks_for_mdev(matrix_mdev);
3f85d1df 1341 mutex_unlock(&ap_perms_mutex);
3211da0c
TK		 1342
1343 return ret;
1344}
46623ab3 1345static DEVICE_ATTR_WO(assign_domain);
3211da0c 1346
11cb2419 1347static void vfio_ap_mdev_unlink_domain(struct ap_matrix_mdev *matrix_mdev,
70aeefe5 1348 unsigned long apqi,
f848cba7 1349 struct list_head *qlist)
11cb2419
TK		 1350{
1351 unsigned long apid;
1352 struct vfio_ap_queue *q;
1353
1354 for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {
70aeefe5
TK		 1355 q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi);
1356
f848cba7 1357 if (q && qlist) {
70aeefe5
TK		 1358 if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
1359 test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm))
f848cba7 1360 list_add_tail(&q->reset_qnode, qlist);
70aeefe5
TK		 1361 }
1362 }
1363}
1364
8fb456bc
JH		 1365static void vfio_ap_mdev_hot_unplug_domains(struct ap_matrix_mdev *matrix_mdev,
1366 unsigned long *apqis)
70aeefe5 1367{
f848cba7
TK		 1368 struct vfio_ap_queue *q, *tmpq;
1369 struct list_head qlist;
8fb456bc
JH		 1370 unsigned long apqi;
1371 bool apcb_update = false;
70aeefe5 1372
f848cba7 1373 INIT_LIST_HEAD(&qlist);
11cb2419 1374
8fb456bc
JH		 1375 for_each_set_bit_inv(apqi, apqis, AP_DOMAINS) {
1376 vfio_ap_mdev_unlink_domain(matrix_mdev, apqi, &qlist);
1377
1378 if (test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) {
1379 clear_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm);
1380 apcb_update = true;
1381 }
11cb2419 1382 }
70aeefe5 1383
8fb456bc
JH		 1384 /* Only update apcb if needed to avoid impacting guest */
1385 if (apcb_update)
1386 vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1387
f848cba7 1388 vfio_ap_mdev_reset_qlist(&qlist);
70aeefe5 1389
f848cba7 1390 list_for_each_entry_safe(q, tmpq, &qlist, reset_qnode) {
70aeefe5 1391 vfio_ap_unlink_mdev_fr_queue(q);
f848cba7 1392 list_del(&q->reset_qnode);
70aeefe5 1393 }
11cb2419 1394}
3211da0c 1395
8fb456bc
JH		 1396static void vfio_ap_mdev_hot_unplug_domain(struct ap_matrix_mdev *matrix_mdev,
1397 unsigned long apqi)
1398{
1399 DECLARE_BITMAP(apqis, AP_DOMAINS);
1400
1401 bitmap_zero(apqis, AP_DEVICES);
1402 set_bit_inv(apqi, apqis);
1403 vfio_ap_mdev_hot_unplug_domains(matrix_mdev, apqis);
1404}
1405
3211da0c 1406/**
0c1abe7c
RD		 1407 * unassign_domain_store - parses the APQI from @buf and clears the
1408 * corresponding bit in the mediated matrix device's AQM
3211da0c
TK		 1409 *
1410 * @dev: the matrix device
1411 * @attr: the mediated matrix device's unassign_domain attribute
1412 * @buf: a buffer containing the AP queue index (APQI) of the domain to
1413 * be unassigned
1414 * @count: the number of bytes in @buf
1415 *
0c1abe7c 1416 * Return: the number of bytes processed if the APQI is valid; otherwise,
3211da0c
TK		 1417 * returns one of the following errors:
1418 * -EINVAL if the APQI is not a number
1419 * -ENODEV if the APQI exceeds the maximum value configured for the system
1420 */
1421static ssize_t unassign_domain_store(struct device *dev,
1422 struct device_attribute *attr,
1423 const char *buf, size_t count)
1424{
1425 int ret;
1426 unsigned long apqi;
eb0feefd 1427 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
3211da0c 1428
8ee13ad9 1429 get_update_locks_for_mdev(matrix_mdev);
258287c9 1430
3211da0c
TK		 1431 ret = kstrtoul(buf, 0, &apqi);
1432 if (ret)
0cc00c8d 1433 goto done;
3211da0c 1434
0cc00c8d
TK		 1435 if (apqi > matrix_mdev->matrix.aqm_max) {
1436 ret = -ENODEV;
1437 goto done;
1438 }
3211da0c 1439
1918f2b2
TK		 1440 if (!test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {
1441 ret = count;
1442 goto done;
1443 }
1444
3211da0c 1445 clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm);
70aeefe5 1446 vfio_ap_mdev_hot_unplug_domain(matrix_mdev, apqi);
0cc00c8d 1447 ret = count;
3211da0c 1448
0cc00c8d 1449done:
8ee13ad9 1450 release_update_locks_for_mdev(matrix_mdev);
0cc00c8d 1451 return ret;
3211da0c 1452}
46623ab3 1453static DEVICE_ATTR_WO(unassign_domain);
3211da0c 1454
3b1eab7f 1455/**
0c1abe7c
RD		 1456 * assign_control_domain_store - parses the domain ID from @buf and sets
1457 * the corresponding bit in the mediated matrix device's ADM
1458 *
3b1eab7f
TK		 1459 * @dev: the matrix device
1460 * @attr: the mediated matrix device's assign_control_domain attribute
1461 * @buf: a buffer containing the domain ID to be assigned
1462 * @count: the number of bytes in @buf
1463 *
0c1abe7c 1464 * Return: the number of bytes processed if the domain ID is valid; otherwise,
3b1eab7f
TK		 1465 * returns one of the following errors:
1466 * -EINVAL if the ID is not a number
1467 * -ENODEV if the ID exceeds the maximum value configured for the system
1468 */
1469static ssize_t assign_control_domain_store(struct device *dev,
1470 struct device_attribute *attr,
1471 const char *buf, size_t count)
1472{
1473 int ret;
1474 unsigned long id;
eb0feefd 1475 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
3b1eab7f 1476
8ee13ad9 1477 get_update_locks_for_mdev(matrix_mdev);
258287c9 1478
3b1eab7f
TK		 1479 ret = kstrtoul(buf, 0, &id);
1480 if (ret)
0cc00c8d 1481 goto done;
3b1eab7f 1482
0cc00c8d
TK		 1483 if (id > matrix_mdev->matrix.adm_max) {
1484 ret = -ENODEV;
1485 goto done;
1486 }
3b1eab7f 1487
1918f2b2
TK		 1488 if (test_bit_inv(id, matrix_mdev->matrix.adm)) {
1489 ret = count;
1490 goto done;
1491 }
1492
3b1eab7f
TK		 1493 /* Set the bit in the ADM (bitmask) corresponding to the AP control
1494 * domain number (id). The bits in the mask, from most significant to
1495 * least significant, correspond to IDs 0 up to the one less than the
1496 * number of control domains that can be assigned.
1497 */
3b1eab7f 1498 set_bit_inv(id, matrix_mdev->matrix.adm);
51dc562a
TK		 1499 if (vfio_ap_mdev_filter_cdoms(matrix_mdev))
1500 vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1501
0cc00c8d
TK		 1502 ret = count;
1503done:
8ee13ad9 1504 release_update_locks_for_mdev(matrix_mdev);
0cc00c8d 1505 return ret;
3b1eab7f 1506}
46623ab3 1507static DEVICE_ATTR_WO(assign_control_domain);
3b1eab7f
TK		 1508
1509/**
0c1abe7c
RD		 1510 * unassign_control_domain_store - parses the domain ID from @buf and
1511 * clears the corresponding bit in the mediated matrix device's ADM
3b1eab7f
TK		 1512 *
1513 * @dev: the matrix device
1514 * @attr: the mediated matrix device's unassign_control_domain attribute
1515 * @buf: a buffer containing the domain ID to be unassigned
1516 * @count: the number of bytes in @buf
1517 *
0c1abe7c 1518 * Return: the number of bytes processed if the domain ID is valid; otherwise,
3b1eab7f
TK		 1519 * returns one of the following errors:
1520 * -EINVAL if the ID is not a number
1521 * -ENODEV if the ID exceeds the maximum value configured for the system
1522 */
1523static ssize_t unassign_control_domain_store(struct device *dev,
1524 struct device_attribute *attr,
1525 const char *buf, size_t count)
1526{
1527 int ret;
1528 unsigned long domid;
eb0feefd 1529 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
3b1eab7f 1530
8ee13ad9 1531 get_update_locks_for_mdev(matrix_mdev);
258287c9 1532
3b1eab7f
TK		 1533 ret = kstrtoul(buf, 0, &domid);
1534 if (ret)
0cc00c8d 1535 goto done;
51dc562a
TK		 1536
1537 if (domid > matrix_mdev->matrix.adm_max) {
0cc00c8d
TK		 1538 ret = -ENODEV;
1539 goto done;
1540 }
3b1eab7f 1541
1918f2b2
TK		 1542 if (!test_bit_inv(domid, matrix_mdev->matrix.adm)) {
1543 ret = count;
1544 goto done;
1545 }
1546
3b1eab7f 1547 clear_bit_inv(domid, matrix_mdev->matrix.adm);
48cae940 1548
51dc562a 1549 if (test_bit_inv(domid, matrix_mdev->shadow_apcb.adm)) {
48cae940 1550 clear_bit_inv(domid, matrix_mdev->shadow_apcb.adm);
51dc562a
TK		 1551 vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1552 }
48cae940 1553
0cc00c8d
TK		 1554 ret = count;
1555done:
8ee13ad9 1556 release_update_locks_for_mdev(matrix_mdev);
0cc00c8d 1557 return ret;
3b1eab7f 1558}
46623ab3 1559static DEVICE_ATTR_WO(unassign_control_domain);
3b1eab7f
TK		 1560
1561static ssize_t control_domains_show(struct device *dev,
1562 struct device_attribute *dev_attr,
1563 char *buf)
1564{
1565 unsigned long id;
eb0feefd 1566 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
3b1eab7f 1567 unsigned long max_domid = matrix_mdev->matrix.adm_max;
5c77c5d1 1568 int nchars = 0;
3b1eab7f 1569
d0786556 1570 mutex_lock(&matrix_dev->mdevs_lock);
5c77c5d1
MD		 1571 for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1)
1572 nchars += sysfs_emit_at(buf, nchars, "%04lx\n", id);
d0786556 1573 mutex_unlock(&matrix_dev->mdevs_lock);
3b1eab7f
TK		 1574
1575 return nchars;
1576}
46623ab3 1577static DEVICE_ATTR_RO(control_domains);
3b1eab7f 1578
f7f795c5 1579static ssize_t vfio_ap_mdev_matrix_show(struct ap_matrix *matrix, char *buf)
81b2b4b7 1580{
81b2b4b7
TK		 1581 unsigned long apid;
1582 unsigned long apqi;
1583 unsigned long apid1;
1584 unsigned long apqi1;
f7f795c5
TK		 1585 unsigned long napm_bits = matrix->apm_max + 1;
1586 unsigned long naqm_bits = matrix->aqm_max + 1;
81b2b4b7 1587 int nchars = 0;
81b2b4b7 1588
f7f795c5
TK		 1589 apid1 = find_first_bit_inv(matrix->apm, napm_bits);
1590 apqi1 = find_first_bit_inv(matrix->aqm, naqm_bits);
81b2b4b7
TK		 1591
1592 if ((apid1 < napm_bits) && (apqi1 < naqm_bits)) {
f7f795c5 1593 for_each_set_bit_inv(apid, matrix->apm, napm_bits) {
5c77c5d1
MD		 1594 for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits)
1595 nchars += sysfs_emit_at(buf, nchars, "%02lx.%04lx\n", apid, apqi);
81b2b4b7
TK		 1596 }
1597 } else if (apid1 < napm_bits) {
5c77c5d1
MD		 1598 for_each_set_bit_inv(apid, matrix->apm, napm_bits)
1599 nchars += sysfs_emit_at(buf, nchars, "%02lx.\n", apid);
81b2b4b7 1600 } else if (apqi1 < naqm_bits) {
5c77c5d1
MD		 1601 for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits)
1602 nchars += sysfs_emit_at(buf, nchars, ".%04lx\n", apqi);
81b2b4b7
TK		 1603 }
1604
f7f795c5
TK		 1605 return nchars;
1606}
1607
1608static ssize_t matrix_show(struct device *dev, struct device_attribute *attr,
1609 char *buf)
1610{
1611 ssize_t nchars;
1612 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1613
1614 mutex_lock(&matrix_dev->mdevs_lock);
1615 nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->matrix, buf);
d0786556 1616 mutex_unlock(&matrix_dev->mdevs_lock);
81b2b4b7
TK		 1617
1618 return nchars;
1619}
46623ab3 1620static DEVICE_ATTR_RO(matrix);
81b2b4b7 1621
f7f795c5
TK		 1622static ssize_t guest_matrix_show(struct device *dev,
1623 struct device_attribute *attr, char *buf)
1624{
1625 ssize_t nchars;
1626 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1627
1628 mutex_lock(&matrix_dev->mdevs_lock);
1629 nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->shadow_apcb, buf);
1630 mutex_unlock(&matrix_dev->mdevs_lock);
1631
1632 return nchars;
1633}
1634static DEVICE_ATTR_RO(guest_matrix);
1635
e12aa0b5
JH		 1636static ssize_t write_ap_bitmap(unsigned long *bitmap, char *buf, int offset, char sep)
1637{
1638 return sysfs_emit_at(buf, offset, "0x%016lx%016lx%016lx%016lx%c",
1639 bitmap[0], bitmap[1], bitmap[2], bitmap[3], sep);
1640}
1641
1642static ssize_t ap_config_show(struct device *dev, struct device_attribute *attr,
1643 char *buf)
1644{
1645 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1646 int idx = 0;
1647
1648 idx += write_ap_bitmap(matrix_mdev->matrix.apm, buf, idx, ',');
1649 idx += write_ap_bitmap(matrix_mdev->matrix.aqm, buf, idx, ',');
1650 idx += write_ap_bitmap(matrix_mdev->matrix.adm, buf, idx, '\n');
1651
1652 return idx;
1653}
1654
8fb456bc
JH		 1655/* Number of characters needed for a complete hex mask representing the bits in .. */
1656#define AP_DEVICES_STRLEN (AP_DEVICES / 4 + 3)
1657#define AP_DOMAINS_STRLEN (AP_DOMAINS / 4 + 3)
1658#define AP_CONFIG_STRLEN (AP_DEVICES_STRLEN + 2 * AP_DOMAINS_STRLEN)
1659
1660static int parse_bitmap(char **strbufptr, unsigned long *bitmap, int nbits)
1661{
1662 char *curmask;
1663
1664 curmask = strsep(strbufptr, ",\n");
1665 if (!curmask)
1666 return -EINVAL;
1667
1668 bitmap_clear(bitmap, 0, nbits);
1669 return ap_hex2bitmap(curmask, bitmap, nbits);
1670}
1671
1672static int ap_matrix_overflow_check(struct ap_matrix_mdev *matrix_mdev)
1673{
1674 unsigned long bit;
1675
1676 for_each_set_bit_inv(bit, matrix_mdev->matrix.apm, AP_DEVICES) {
1677 if (bit > matrix_mdev->matrix.apm_max)
1678 return -ENODEV;
1679 }
1680
1681 for_each_set_bit_inv(bit, matrix_mdev->matrix.aqm, AP_DOMAINS) {
1682 if (bit > matrix_mdev->matrix.aqm_max)
1683 return -ENODEV;
1684 }
1685
1686 for_each_set_bit_inv(bit, matrix_mdev->matrix.adm, AP_DOMAINS) {
1687 if (bit > matrix_mdev->matrix.adm_max)
1688 return -ENODEV;
1689 }
1690
1691 return 0;
1692}
1693
1694static void ap_matrix_copy(struct ap_matrix *dst, struct ap_matrix *src)
1695{
1696 /* This check works around false positive gcc -Wstringop-overread */
1697 if (!src)
1698 return;
1699
1700 bitmap_copy(dst->apm, src->apm, AP_DEVICES);
1701 bitmap_copy(dst->aqm, src->aqm, AP_DOMAINS);
1702 bitmap_copy(dst->adm, src->adm, AP_DOMAINS);
1703}
1704
e12aa0b5
JH		 1705static ssize_t ap_config_store(struct device *dev, struct device_attribute *attr,
1706 const char *buf, size_t count)
1707{
8fb456bc
JH		 1708 struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1709 struct ap_matrix m_new, m_old, m_added, m_removed;
1710 DECLARE_BITMAP(apm_filtered, AP_DEVICES);
1711 unsigned long newbit;
1712 char *newbuf, *rest;
1713 int rc = count;
1714 bool do_update;
1715
1716 newbuf = kstrndup(buf, AP_CONFIG_STRLEN, GFP_KERNEL);
1717 if (!newbuf)
1718 return -ENOMEM;
1719 rest = newbuf;
1720
1721 mutex_lock(&ap_perms_mutex);
1722 get_update_locks_for_mdev(matrix_mdev);
1723
1724 /* Save old state */
1725 ap_matrix_copy(&m_old, &matrix_mdev->matrix);
1726 if (parse_bitmap(&rest, m_new.apm, AP_DEVICES) ||
1727 parse_bitmap(&rest, m_new.aqm, AP_DOMAINS) ||
1728 parse_bitmap(&rest, m_new.adm, AP_DOMAINS)) {
1729 rc = -EINVAL;
1730 goto out;
1731 }
1732
1733 bitmap_andnot(m_removed.apm, m_old.apm, m_new.apm, AP_DEVICES);
1734 bitmap_andnot(m_removed.aqm, m_old.aqm, m_new.aqm, AP_DOMAINS);
1735 bitmap_andnot(m_added.apm, m_new.apm, m_old.apm, AP_DEVICES);
1736 bitmap_andnot(m_added.aqm, m_new.aqm, m_old.aqm, AP_DOMAINS);
1737
1738 /* Need new bitmaps in matrix_mdev for validation */
1739 ap_matrix_copy(&matrix_mdev->matrix, &m_new);
1740
1741 /* Ensure new state is valid, else undo new state */
1742 rc = vfio_ap_mdev_validate_masks(matrix_mdev);
1743 if (rc) {
1744 ap_matrix_copy(&matrix_mdev->matrix, &m_old);
1745 goto out;
1746 }
1747 rc = ap_matrix_overflow_check(matrix_mdev);
1748 if (rc) {
1749 ap_matrix_copy(&matrix_mdev->matrix, &m_old);
1750 goto out;
1751 }
1752 rc = count;
1753
1754 /* Need old bitmaps in matrix_mdev for unplug/unlink */
1755 ap_matrix_copy(&matrix_mdev->matrix, &m_old);
1756
1757 /* Unlink removed adapters/domains */
1758 vfio_ap_mdev_hot_unplug_adapters(matrix_mdev, m_removed.apm);
1759 vfio_ap_mdev_hot_unplug_domains(matrix_mdev, m_removed.aqm);
1760
1761 /* Need new bitmaps in matrix_mdev for linking new adapters/domains */
1762 ap_matrix_copy(&matrix_mdev->matrix, &m_new);
1763
1764 /* Link newly added adapters */
1765 for_each_set_bit_inv(newbit, m_added.apm, AP_DEVICES)
1766 vfio_ap_mdev_link_adapter(matrix_mdev, newbit);
1767
1768 for_each_set_bit_inv(newbit, m_added.aqm, AP_DOMAINS)
1769 vfio_ap_mdev_link_domain(matrix_mdev, newbit);
1770
1771 /* filter resources not bound to vfio-ap */
1772 do_update = vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered);
1773 do_update |= vfio_ap_mdev_filter_cdoms(matrix_mdev);
1774
1775 /* Apply changes to shadow apbc if things changed */
1776 if (do_update) {
1777 vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1778 reset_queues_for_apids(matrix_mdev, apm_filtered);
1779 }
1780out:
1781 release_update_locks_for_mdev(matrix_mdev);
1782 mutex_unlock(&ap_perms_mutex);
1783 kfree(newbuf);
1784 return rc;
e12aa0b5
JH		 1785}
1786static DEVICE_ATTR_RW(ap_config);
1787
96d152bd
TK		 1788static struct attribute *vfio_ap_mdev_attrs[] = {
1789 &dev_attr_assign_adapter.attr,
1790 &dev_attr_unassign_adapter.attr,
3211da0c
TK		 1791 &dev_attr_assign_domain.attr,
1792 &dev_attr_unassign_domain.attr,
3b1eab7f
TK		 1793 &dev_attr_assign_control_domain.attr,
1794 &dev_attr_unassign_control_domain.attr,
e12aa0b5 1795 &dev_attr_ap_config.attr,
3b1eab7f 1796 &dev_attr_control_domains.attr,
81b2b4b7 1797 &dev_attr_matrix.attr,
f7f795c5 1798 &dev_attr_guest_matrix.attr,
3211da0c 1799 NULL,
96d152bd
TK		 1800};
1801
1802static struct attribute_group vfio_ap_mdev_attr_group = {
1803 .attrs = vfio_ap_mdev_attrs
1804};
1805
1806static const struct attribute_group *vfio_ap_mdev_attr_groups[] = {
1807 &vfio_ap_mdev_attr_group,
1808 NULL
1809};
1810
258287c9 1811/**
0c1abe7c
RD		 1812 * vfio_ap_mdev_set_kvm - sets all data for @matrix_mdev that are needed
1813 * to manage AP resources for the guest whose state is represented by @kvm
258287c9
TK		 1814 *
1815 * @matrix_mdev: a mediated matrix device
1816 * @kvm: reference to KVM instance
1817 *
0c1abe7c 1818 * Return: 0 if no other mediated matrix device has a reference to @kvm;
258287c9
TK		 1819 * otherwise, returns an -EPERM.
1820 */
1821static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev,
1822 struct kvm *kvm)
1823{
1824 struct ap_matrix_mdev *m;
1825
0cc00c8d 1826 if (kvm->arch.crypto.crycbd) {
86956e70
TK		 1827 down_write(&kvm->arch.crypto.pqap_hook_rwsem);
1828 kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook;
1829 up_write(&kvm->arch.crypto.pqap_hook_rwsem);
1830
b84eb8e0 1831 get_update_locks_for_kvm(kvm);
86956e70 1832
0cc00c8d 1833 list_for_each_entry(m, &matrix_dev->mdev_list, node) {
86956e70 1834 if (m != matrix_mdev && m->kvm == kvm) {
b84eb8e0 1835 release_update_locks_for_kvm(kvm);
0cc00c8d 1836 return -EPERM;
86956e70 1837 }
0cc00c8d 1838 }
258287c9 1839
0cc00c8d 1840 kvm_get_kvm(kvm);
1e753732 1841 matrix_mdev->kvm = kvm;
51dc562a 1842 vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1e753732 1843
b84eb8e0 1844 release_update_locks_for_kvm(kvm);
0cc00c8d 1845 }
258287c9
TK		 1846
1847 return 0;
1848}
1849
2a54e347
MR		 1850static void unmap_iova(struct ap_matrix_mdev *matrix_mdev, u64 iova, u64 length)
1851{
1852 struct ap_queue_table *qtable = &matrix_mdev->qtable;
1853 struct vfio_ap_queue *q;
1854 int loop_cursor;
1855
1856 hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
1857 if (q->saved_iova >= iova && q->saved_iova < iova + length)
1858 vfio_ap_irq_disable(q);
1859 }
1860}
1861
ce4b4657
JG		 1862static void vfio_ap_mdev_dma_unmap(struct vfio_device *vdev, u64 iova,
1863 u64 length)
62e358ce 1864{
ce4b4657
JG		 1865 struct ap_matrix_mdev *matrix_mdev =
1866 container_of(vdev, struct ap_matrix_mdev, vdev);
62e358ce 1867
2a54e347
MR		 1868 mutex_lock(&matrix_dev->mdevs_lock);
1869
1870 unmap_iova(matrix_mdev, iova, length);
1871
1872 mutex_unlock(&matrix_dev->mdevs_lock);
62e358ce
PM		 1873}
1874
0cc00c8d 1875/**
0c1abe7c
RD		 1876 * vfio_ap_mdev_unset_kvm - performs clean-up of resources no longer needed
1877 * by @matrix_mdev.
0cc00c8d
TK		 1878 *
1879 * @matrix_mdev: a matrix mediated device
0cc00c8d 1880 */
71078220 1881static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev)
f21916ec 1882{
71078220
TK		 1883 struct kvm *kvm = matrix_mdev->kvm;
1884
86956e70
TK		 1885 if (kvm && kvm->arch.crypto.crycbd) {
1886 down_write(&kvm->arch.crypto.pqap_hook_rwsem);
1887 kvm->arch.crypto.pqap_hook = NULL;
1888 up_write(&kvm->arch.crypto.pqap_hook_rwsem);
0cc00c8d 1889
b84eb8e0 1890 get_update_locks_for_kvm(kvm);
86956e70
TK		 1891
1892 kvm_arch_crypto_clear_masks(kvm);
f848cba7 1893 vfio_ap_mdev_reset_queues(matrix_mdev);
86956e70 1894 kvm_put_kvm(kvm);
0cc00c8d 1895 matrix_mdev->kvm = NULL;
86956e70 1896
b84eb8e0 1897 release_update_locks_for_kvm(kvm);
0cc00c8d 1898 }
f21916ec
TK		 1899}
1900
6c12a638 1901static struct vfio_ap_queue *vfio_ap_find_queue(int apqn)
ec89b55e 1902{
034921cd 1903 struct ap_queue *queue;
6c12a638 1904 struct vfio_ap_queue *q = NULL;
ec89b55e 1905
034921cd
TK		 1906 queue = ap_get_qdev(apqn);
1907 if (!queue)
1908 return NULL;
1909
1910 if (queue->ap_dev.device.driver == &matrix_dev->vfio_ap_drv->driver)
1911 q = dev_get_drvdata(&queue->ap_dev.device);
1912
1913 put_device(&queue->ap_dev.device);
6c12a638
TK		 1914
1915 return q;
ec89b55e
PM		 1916}
1917
0daf9878
TK		 1918static int apq_status_check(int apqn, struct ap_queue_status *status)
1919{
1920 switch (status->response_code) {
1921 case AP_RESPONSE_NORMAL:
0daf9878 1922 case AP_RESPONSE_DECONFIGURED:
a681226c 1923 case AP_RESPONSE_CHECKSTOPPED:
0daf9878 1924 return 0;
680b7ddd 1925 case AP_RESPONSE_RESET_IN_PROGRESS:
411b0109 1926 case AP_RESPONSE_BUSY:
680b7ddd 1927 return -EBUSY;
7847a19b
TK		 1928 case AP_RESPONSE_ASSOC_SECRET_NOT_UNIQUE:
1929 case AP_RESPONSE_ASSOC_FAILED:
1930 /*
1931 * These asynchronous response codes indicate a PQAP(AAPQ)
1932 * instruction to associate a secret with the guest failed. All
1933 * subsequent AP instructions will end with the asynchronous
1934 * response code until the AP queue is reset; so, let's return
1935 * a value indicating a reset needs to be performed again.
1936 */
1937 return -EAGAIN;
0daf9878
TK		 1938 default:
1939 WARN(true,
1940 "failed to verify reset of queue %02x.%04x: TAPQ rc=%u\n",
1941 AP_QID_CARD(apqn), AP_QID_QUEUE(apqn),
1942 status->response_code);
1943 return -EIO;
1944 }
1945}
1946
dd174833
TK		 1947#define WAIT_MSG "Waited %dms for reset of queue %02x.%04x (%u, %u, %u)"
1948
9261f043 1949static void apq_reset_check(struct work_struct *reset_work)
62414d90 1950{
dd174833 1951 int ret = -EBUSY, elapsed = 0;
62414d90 1952 struct ap_queue_status status;
9261f043 1953 struct vfio_ap_queue *q;
62414d90 1954
9261f043 1955 q = container_of(reset_work, struct vfio_ap_queue, reset_work);
62aab082 1956 memcpy(&status, &q->reset_status, sizeof(status));
dd174833 1957 while (true) {
7cb7636a 1958 msleep(AP_RESET_INTERVAL);
dd174833 1959 elapsed += AP_RESET_INTERVAL;
62414d90 1960 status = ap_tapq(q->apqn, NULL);
0daf9878 1961 ret = apq_status_check(q->apqn, &status);
dd174833 1962 if (ret == -EIO)
9261f043 1963 return;
dd174833
TK		 1964 if (ret == -EBUSY) {
1965 pr_notice_ratelimited(WAIT_MSG, elapsed,
1966 AP_QID_CARD(q->apqn),
1967 AP_QID_QUEUE(q->apqn),
1968 status.response_code,
1969 status.queue_empty,
1970 status.irq_enabled);
1971 } else {
62aab082 1972 if (q->reset_status.response_code == AP_RESPONSE_RESET_IN_PROGRESS ||
e1f17f8e 1973 q->reset_status.response_code == AP_RESPONSE_BUSY ||
7847a19b
TK		 1974 q->reset_status.response_code == AP_RESPONSE_STATE_CHANGE_IN_PROGRESS ||
1975 ret == -EAGAIN) {
dd174833 1976 status = ap_zapq(q->apqn, 0);
62aab082 1977 memcpy(&q->reset_status, &status, sizeof(status));
dd174833
TK		 1978 continue;
1979 }
dd174833
TK		 1980 if (q->saved_isc != VFIO_AP_ISC_INVALID)
1981 vfio_ap_free_aqic_resources(q);
1982 break;
1983 }
62414d90 1984 }
62414d90
TK		 1985}
1986
9261f043 1987static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q)
46a7263d
TK		 1988{
1989 struct ap_queue_status status;
1990
6c12a638 1991 if (!q)
9261f043 1992 return;
4bdf3c39 1993 status = ap_zapq(q->apqn, 0);
62aab082 1994 memcpy(&q->reset_status, &status, sizeof(status));
6c12a638
TK		 1995 switch (status.response_code) {
1996 case AP_RESPONSE_NORMAL:
6c12a638 1997 case AP_RESPONSE_RESET_IN_PROGRESS:
411b0109 1998 case AP_RESPONSE_BUSY:
e1f17f8e 1999 case AP_RESPONSE_STATE_CHANGE_IN_PROGRESS:
9261f043
TK		 2000 /*
2001 * Let's verify whether the ZAPQ completed successfully on a work queue.
2002 */
2003 queue_work(system_long_wq, &q->reset_work);
dd174833 2004 break;
6c12a638 2005 case AP_RESPONSE_DECONFIGURED:
a681226c 2006 case AP_RESPONSE_CHECKSTOPPED:
7aa7b2a8 2007 vfio_ap_free_aqic_resources(q);
51d4d987 2008 break;
6c12a638 2009 default:
70aeefe5
TK		 2010 WARN(true,
2011 "PQAP/ZAPQ for %02x.%04x failed with invalid rc=%u\n",
2012 AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn),
6c12a638 2013 status.response_code);
6c12a638 2014 }
46a7263d
TK		 2015}
2016
f848cba7 2017static int vfio_ap_mdev_reset_queues(struct ap_matrix_mdev *matrix_mdev)
46a7263d 2018{
9261f043 2019 int ret = 0, loop_cursor;
6c12a638 2020 struct vfio_ap_queue *q;
46a7263d 2021
f848cba7 2022 hash_for_each(matrix_mdev->qtable.queues, loop_cursor, q, mdev_qnode)
9261f043
TK		 2023 vfio_ap_mdev_reset_queue(q);
2024
f848cba7
TK		 2025 hash_for_each(matrix_mdev->qtable.queues, loop_cursor, q, mdev_qnode) {
2026 flush_work(&q->reset_work);
2027
2028 if (q->reset_status.response_code)
2029 ret = -EIO;
2030 }
2031
2032 return ret;
2033}
2034
2035static int vfio_ap_mdev_reset_qlist(struct list_head *qlist)
2036{
2037 int ret = 0;
2038 struct vfio_ap_queue *q;
2039
2040 list_for_each_entry(q, qlist, reset_qnode)
2041 vfio_ap_mdev_reset_queue(q);
2042
2043 list_for_each_entry(q, qlist, reset_qnode) {
9261f043
TK		 2044 flush_work(&q->reset_work);
2045
2046 if (q->reset_status.response_code)
2047 ret = -EIO;
46a7263d
TK		 2048 }
2049
9261f043 2050 return ret;
46a7263d
TK		 2051}
2052
eb0feefd 2053static int vfio_ap_mdev_open_device(struct vfio_device *vdev)
258287c9 2054{
eb0feefd
JG		 2055 struct ap_matrix_mdev *matrix_mdev =
2056 container_of(vdev, struct ap_matrix_mdev, vdev);
258287c9 2057
421cfe65
MR		 2058 if (!vdev->kvm)
2059 return -EINVAL;
258287c9 2060
ce4b4657 2061 return vfio_ap_mdev_set_kvm(matrix_mdev, vdev->kvm);
258287c9
TK		 2062}
2063
eb0feefd 2064static void vfio_ap_mdev_close_device(struct vfio_device *vdev)
258287c9 2065{
eb0feefd
JG		 2066 struct ap_matrix_mdev *matrix_mdev =
2067 container_of(vdev, struct ap_matrix_mdev, vdev);
258287c9 2068
71078220 2069 vfio_ap_mdev_unset_kvm(matrix_mdev);
258287c9
TK		 2070}
2071
2e3d8d71
TK		 2072static void vfio_ap_mdev_request(struct vfio_device *vdev, unsigned int count)
2073{
2074 struct device *dev = vdev->dev;
2075 struct ap_matrix_mdev *matrix_mdev;
2076
2077 matrix_mdev = container_of(vdev, struct ap_matrix_mdev, vdev);
2078
cbf367d5
RR		 2079 get_update_locks_for_mdev(matrix_mdev);
2080
2081 if (matrix_mdev->kvm) {
2082 kvm_arch_crypto_clear_masks(matrix_mdev->kvm);
2083 signal_guest_ap_cfg_changed(matrix_mdev);
2084 }
2085
2e3d8d71
TK		 2086 if (matrix_mdev->req_trigger) {
2087 if (!(count % 10))
2088 dev_notice_ratelimited(dev,
2089 "Relaying device request to user (#%u)\n",
2090 count);
2091
3652117f 2092 eventfd_signal(matrix_mdev->req_trigger);
2e3d8d71
TK		 2093 } else if (count == 0) {
2094 dev_notice(dev,
2095 "No device request registered, blocked until released by user\n");
2096 }
cbf367d5
RR		 2097
2098 release_update_locks_for_mdev(matrix_mdev);
2e3d8d71
TK		 2099}
2100
e06670c5
TK		 2101static int vfio_ap_mdev_get_device_info(unsigned long arg)
2102{
2103 unsigned long minsz;
2104 struct vfio_device_info info;
2105
2106 minsz = offsetofend(struct vfio_device_info, num_irqs);
2107
2108 if (copy_from_user(&info, (void __user *)arg, minsz))
2109 return -EFAULT;
2110
2111 if (info.argsz < minsz)
2112 return -EINVAL;
2113
cd8a377e 2114 info.flags = VFIO_DEVICE_FLAGS_AP | VFIO_DEVICE_FLAGS_RESET;
e06670c5 2115 info.num_regions = 0;
6afc7700
TK		 2116 info.num_irqs = VFIO_AP_NUM_IRQS;
2117
2118 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
2119}
2120
2121static ssize_t vfio_ap_get_irq_info(unsigned long arg)
2122{
2123 unsigned long minsz;
2124 struct vfio_irq_info info;
2125
2126 minsz = offsetofend(struct vfio_irq_info, count);
2127
2128 if (copy_from_user(&info, (void __user *)arg, minsz))
2129 return -EFAULT;
2130
2131 if (info.argsz < minsz || info.index >= VFIO_AP_NUM_IRQS)
2132 return -EINVAL;
2133
2134 switch (info.index) {
2135 case VFIO_AP_REQ_IRQ_INDEX:
2136 info.count = 1;
2137 info.flags = VFIO_IRQ_INFO_EVENTFD;
2138 break;
07d89045
RR		 2139 case VFIO_AP_CFG_CHG_IRQ_INDEX:
2140 info.count = 1;
2141 info.flags = VFIO_IRQ_INFO_EVENTFD;
2142 break;
6afc7700
TK		 2143 default:
2144 return -EINVAL;
2145 }
e06670c5 2146
942df4be 2147 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
e06670c5
TK		 2148}
2149
bf48961f
TK		 2150static int vfio_ap_irq_set_init(struct vfio_irq_set *irq_set, unsigned long arg)
2151{
2152 int ret;
2153 size_t data_size;
2154 unsigned long minsz;
2155
2156 minsz = offsetofend(struct vfio_irq_set, count);
2157
2158 if (copy_from_user(irq_set, (void __user *)arg, minsz))
2159 return -EFAULT;
2160
2161 ret = vfio_set_irqs_validate_and_prepare(irq_set, 1, VFIO_AP_NUM_IRQS,
2162 &data_size);
2163 if (ret)
2164 return ret;
2165
2166 if (!(irq_set->flags & VFIO_IRQ_SET_ACTION_TRIGGER))
2167 return -EINVAL;
2168
2169 return 0;
2170}
2171
2172static int vfio_ap_set_request_irq(struct ap_matrix_mdev *matrix_mdev,
2173 unsigned long arg)
2174{
2175 s32 fd;
2176 void __user *data;
2177 unsigned long minsz;
2178 struct eventfd_ctx *req_trigger;
2179
2180 minsz = offsetofend(struct vfio_irq_set, count);
2181 data = (void __user *)(arg + minsz);
2182
2183 if (get_user(fd, (s32 __user *)data))
2184 return -EFAULT;
2185
2186 if (fd == -1) {
2187 if (matrix_mdev->req_trigger)
2188 eventfd_ctx_put(matrix_mdev->req_trigger);
2189 matrix_mdev->req_trigger = NULL;
2190 } else if (fd >= 0) {
2191 req_trigger = eventfd_ctx_fdget(fd);
2192 if (IS_ERR(req_trigger))
2193 return PTR_ERR(req_trigger);
2194
2195 if (matrix_mdev->req_trigger)
2196 eventfd_ctx_put(matrix_mdev->req_trigger);
2197
2198 matrix_mdev->req_trigger = req_trigger;
2199 } else {
2200 return -EINVAL;
2201 }
2202
2203 return 0;
2204}
2205
07d89045
RR		 2206static int vfio_ap_set_cfg_change_irq(struct ap_matrix_mdev *matrix_mdev, unsigned long arg)
2207{
2208 s32 fd;
2209 void __user *data;
2210 unsigned long minsz;
2211 struct eventfd_ctx *cfg_chg_trigger;
2212
2213 minsz = offsetofend(struct vfio_irq_set, count);
2214 data = (void __user *)(arg + minsz);
2215
2216 if (get_user(fd, (s32 __user *)data))
2217 return -EFAULT;
2218
2219 if (fd == -1) {
2220 if (matrix_mdev->cfg_chg_trigger)
2221 eventfd_ctx_put(matrix_mdev->cfg_chg_trigger);
2222 matrix_mdev->cfg_chg_trigger = NULL;
2223 } else if (fd >= 0) {
2224 cfg_chg_trigger = eventfd_ctx_fdget(fd);
2225 if (IS_ERR(cfg_chg_trigger))
2226 return PTR_ERR(cfg_chg_trigger);
2227
2228 if (matrix_mdev->cfg_chg_trigger)
2229 eventfd_ctx_put(matrix_mdev->cfg_chg_trigger);
2230
2231 matrix_mdev->cfg_chg_trigger = cfg_chg_trigger;
2232 } else {
2233 return -EINVAL;
2234 }
2235
2236 return 0;
2237}
2238
bf48961f
TK		 2239static int vfio_ap_set_irqs(struct ap_matrix_mdev *matrix_mdev,
2240 unsigned long arg)
2241{
2242 int ret;
2243 struct vfio_irq_set irq_set;
2244
2245 ret = vfio_ap_irq_set_init(&irq_set, arg);
2246 if (ret)
2247 return ret;
2248
2249 switch (irq_set.flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
2250 case VFIO_IRQ_SET_DATA_EVENTFD:
2251 switch (irq_set.index) {
2252 case VFIO_AP_REQ_IRQ_INDEX:
2253 return vfio_ap_set_request_irq(matrix_mdev, arg);
07d89045
RR		 2254 case VFIO_AP_CFG_CHG_IRQ_INDEX:
2255 return vfio_ap_set_cfg_change_irq(matrix_mdev, arg);
bf48961f
TK		 2256 default:
2257 return -EINVAL;
2258 }
2259 default:
2260 return -EINVAL;
2261 }
2262}
2263
eb0feefd 2264static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev,
e06670c5
TK		 2265 unsigned int cmd, unsigned long arg)
2266{
eb0feefd
JG		 2267 struct ap_matrix_mdev *matrix_mdev =
2268 container_of(vdev, struct ap_matrix_mdev, vdev);
e06670c5
TK		 2269 int ret;
2270
d0786556 2271 mutex_lock(&matrix_dev->mdevs_lock);
e06670c5
TK		 2272 switch (cmd) {
2273 case VFIO_DEVICE_GET_INFO:
2274 ret = vfio_ap_mdev_get_device_info(arg);
2275 break;
cd8a377e 2276 case VFIO_DEVICE_RESET:
f848cba7 2277 ret = vfio_ap_mdev_reset_queues(matrix_mdev);
cd8a377e 2278 break;
6afc7700 2279 case VFIO_DEVICE_GET_IRQ_INFO:
bd0ab337
TB		 2280 ret = vfio_ap_get_irq_info(arg);
2281 break;
bf48961f
TK		 2282 case VFIO_DEVICE_SET_IRQS:
2283 ret = vfio_ap_set_irqs(matrix_mdev, arg);
2284 break;
e06670c5
TK		 2285 default:
2286 ret = -EOPNOTSUPP;
2287 break;
2288 }
d0786556 2289 mutex_unlock(&matrix_dev->mdevs_lock);
e06670c5
TK		 2290
2291 return ret;
2292}
2293
260f3ea1
TK		 2294static struct ap_matrix_mdev *vfio_ap_mdev_for_queue(struct vfio_ap_queue *q)
2295{
2296 struct ap_matrix_mdev *matrix_mdev;
2297 unsigned long apid = AP_QID_CARD(q->apqn);
2298 unsigned long apqi = AP_QID_QUEUE(q->apqn);
2299
2300 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
2301 if (test_bit_inv(apid, matrix_mdev->matrix.apm) &&
2302 test_bit_inv(apqi, matrix_mdev->matrix.aqm))
2303 return matrix_mdev;
2304 }
2305
2306 return NULL;
2307}
2308
2309static ssize_t status_show(struct device *dev,
2310 struct device_attribute *attr,
2311 char *buf)
2312{
2313 ssize_t nchars = 0;
2314 struct vfio_ap_queue *q;
a0d8f4ee 2315 unsigned long apid, apqi;
260f3ea1
TK		 2316 struct ap_matrix_mdev *matrix_mdev;
2317 struct ap_device *apdev = to_ap_dev(dev);
2318
d0786556 2319 mutex_lock(&matrix_dev->mdevs_lock);
260f3ea1
TK		 2320 q = dev_get_drvdata(&apdev->device);
2321 matrix_mdev = vfio_ap_mdev_for_queue(q);
2322
a0d8f4ee
TK		 2323 /* If the queue is assigned to the matrix mediated device, then
2324 * determine whether it is passed through to a guest; otherwise,
2325 * indicate that it is unassigned.
2326 */
260f3ea1 2327 if (matrix_mdev) {
a0d8f4ee
TK		 2328 apid = AP_QID_CARD(q->apqn);
2329 apqi = AP_QID_QUEUE(q->apqn);
2330 /*
2331 * If the queue is passed through to the guest, then indicate
2332 * that it is in use; otherwise, indicate that it is
2333 * merely assigned to a matrix mediated device.
2334 */
2335 if (matrix_mdev->kvm &&
2336 test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
2337 test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm))
5c77c5d1 2338 nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_IN_USE);
260f3ea1 2339 else
5c77c5d1 2340 nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_ASSIGNED);
260f3ea1 2341 } else {
5c77c5d1 2342 nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_UNASSIGNED);
260f3ea1
TK		 2343 }
2344
d0786556 2345 mutex_unlock(&matrix_dev->mdevs_lock);
260f3ea1
TK		 2346
2347 return nchars;
2348}
2349
2350static DEVICE_ATTR_RO(status);
2351
2352static struct attribute *vfio_queue_attrs[] = {
2353 &dev_attr_status.attr,
2354 NULL,
2355};
2356
2357static const struct attribute_group vfio_queue_attr_group = {
2358 .attrs = vfio_queue_attrs,
2359};
2360
eb0feefd 2361static const struct vfio_device_ops vfio_ap_matrix_dev_ops = {
7cb5a82e 2362 .init = vfio_ap_mdev_init_dev,
eb0feefd
JG		 2363 .open_device = vfio_ap_mdev_open_device,
2364 .close_device = vfio_ap_mdev_close_device,
2365 .ioctl = vfio_ap_mdev_ioctl,
ce4b4657 2366 .dma_unmap = vfio_ap_mdev_dma_unmap,
4741f2e9
JG		 2367 .bind_iommufd = vfio_iommufd_emulated_bind,
2368 .unbind_iommufd = vfio_iommufd_emulated_unbind,
2369 .attach_ioas = vfio_iommufd_emulated_attach_ioas,
8cfa7186 2370 .detach_ioas = vfio_iommufd_emulated_detach_ioas,
2e3d8d71 2371 .request = vfio_ap_mdev_request
eb0feefd
JG		 2372};
2373
2374static struct mdev_driver vfio_ap_matrix_driver = {
290aac5d 2375 .device_api = VFIO_DEVICE_API_AP_STRING,
9c799c22 2376 .max_instances = MAX_ZDEV_ENTRIES_EXT,
eb0feefd
JG		 2377 .driver = {
2378 .name = "vfio_ap_mdev",
2379 .owner = THIS_MODULE,
2380 .mod_name = KBUILD_MODNAME,
2381 .dev_groups = vfio_ap_mdev_attr_groups,
2382 },
2383 .probe = vfio_ap_mdev_probe,
2384 .remove = vfio_ap_mdev_remove,
65f06713
TK		 2385};
2386
2387int vfio_ap_mdev_register(void)
2388{
eb0feefd
JG		 2389 int ret;
2390
eb0feefd
JG		 2391 ret = mdev_register_driver(&vfio_ap_matrix_driver);
2392 if (ret)
2393 return ret;
2394
da44c340 2395 matrix_dev->mdev_type.sysfs_name = VFIO_AP_MDEV_TYPE_HWVIRT;
0bc79069 2396 matrix_dev->mdev_type.pretty_name = VFIO_AP_MDEV_NAME_HWVIRT;
92d03904 2397 matrix_dev->mdev_types = &matrix_dev->mdev_type;
89345d51 2398 ret = mdev_register_parent(&matrix_dev->parent, &matrix_dev->device,
da44c340 2399 &vfio_ap_matrix_driver,
92d03904 2400 &matrix_dev->mdev_types, 1);
eb0feefd
JG		 2401 if (ret)
2402 goto err_driver;
2403 return 0;
2404
2405err_driver:
2406 mdev_unregister_driver(&vfio_ap_matrix_driver);
2407 return ret;
65f06713
TK		 2408}
2409
2410void vfio_ap_mdev_unregister(void)
2411{
89345d51 2412 mdev_unregister_parent(&matrix_dev->parent);
eb0feefd 2413 mdev_unregister_driver(&vfio_ap_matrix_driver);
65f06713 2414}
260f3ea1
TK		 2415
2416int vfio_ap_mdev_probe_queue(struct ap_device *apdev)
2417{
2418 int ret;
2419 struct vfio_ap_queue *q;
f848cba7 2420 DECLARE_BITMAP(apm_filtered, AP_DEVICES);
2c1ee898 2421 struct ap_matrix_mdev *matrix_mdev;
260f3ea1
TK		 2422
2423 ret = sysfs_create_group(&apdev->device.kobj, &vfio_queue_attr_group);
2424 if (ret)
2425 return ret;
2426
2427 q = kzalloc(sizeof(*q), GFP_KERNEL);
08866d34
CJ		 2428 if (!q) {
2429 ret = -ENOMEM;
2430 goto err_remove_group;
2431 }
260f3ea1 2432
260f3ea1
TK		 2433 q->apqn = to_ap_queue(&apdev->device)->qid;
2434 q->saved_isc = VFIO_AP_ISC_INVALID;
62aab082 2435 memset(&q->reset_status, 0, sizeof(q->reset_status));
9261f043 2436 INIT_WORK(&q->reset_work, apq_reset_check);
2c1ee898
TK		 2437 matrix_mdev = get_update_locks_by_apqn(q->apqn);
2438
2439 if (matrix_mdev) {
2440 vfio_ap_mdev_link_queue(matrix_mdev, q);
09d31ff7 2441
774d1019
TK		 2442 /*
2443 * If we're in the process of handling the adding of adapters or
2444 * domains to the host's AP configuration, then let the
2445 * vfio_ap device driver's on_scan_complete callback filter the
2446 * matrix and update the guest's AP configuration after all of
2447 * the new queue devices are probed.
2448 */
2449 if (!bitmap_empty(matrix_mdev->apm_add, AP_DEVICES) ||
2450 !bitmap_empty(matrix_mdev->aqm_add, AP_DOMAINS))
2451 goto done;
2452
f848cba7 2453 if (vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered)) {
09d31ff7 2454 vfio_ap_mdev_update_guest_apcb(matrix_mdev);
f848cba7
TK		 2455 reset_queues_for_apids(matrix_mdev, apm_filtered);
2456 }
48cae940 2457 }
774d1019
TK		 2458
2459done:
260f3ea1 2460 dev_set_drvdata(&apdev->device, q);
2c1ee898 2461 release_update_locks_for_mdev(matrix_mdev);
260f3ea1 2462
f848cba7 2463 return ret;
08866d34
CJ		 2464
2465err_remove_group:
2466 sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group);
2467 return ret;
260f3ea1
TK		 2468}
2469
2470void vfio_ap_mdev_remove_queue(struct ap_device *apdev)
2471{
09d31ff7 2472 unsigned long apid, apqi;
260f3ea1 2473 struct vfio_ap_queue *q;
2c1ee898 2474 struct ap_matrix_mdev *matrix_mdev;
260f3ea1 2475
260f3ea1
TK		 2476 sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group);
2477 q = dev_get_drvdata(&apdev->device);
2c1ee898
TK		 2478 get_update_locks_for_queue(q);
2479 matrix_mdev = q->matrix_mdev;
b9bd10c4
TK		 2480 apid = AP_QID_CARD(q->apqn);
2481 apqi = AP_QID_QUEUE(q->apqn);
11cb2419 2482
2c1ee898 2483 if (matrix_mdev) {
f009cfa4 2484 /* If the queue is assigned to the guest's AP configuration */
09d31ff7
TK		 2485 if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
2486 test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) {
f009cfa4
TK		 2487 /*
2488 * Since the queues are defined via a matrix of adapters
2489 * and domains, it is not possible to hot unplug a
2490 * single queue; so, let's unplug the adapter.
2491 */
09d31ff7
TK		 2492 clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm);
2493 vfio_ap_mdev_update_guest_apcb(matrix_mdev);
f009cfa4
TK		 2494 reset_queues_for_apid(matrix_mdev, apid);
2495 goto done;
09d31ff7 2496 }
48cae940
TK		 2497 }
2498
b9bd10c4
TK		 2499 /*
2500 * If the queue is not in the host's AP configuration, then resetting
2501 * it will fail with response code 01, (APQN not valid); so, let's make
2502 * sure it is in the host's config.
2503 */
2504 if (test_bit_inv(apid, (unsigned long *)matrix_dev->info.apm) &&
2505 test_bit_inv(apqi, (unsigned long *)matrix_dev->info.aqm)) {
2506 vfio_ap_mdev_reset_queue(q);
2507 flush_work(&q->reset_work);
2508 }
f009cfa4
TK		 2509
2510done:
2511 if (matrix_mdev)
2512 vfio_ap_unlink_queue_fr_mdev(q);
2513
260f3ea1
TK		 2514 dev_set_drvdata(&apdev->device, NULL);
2515 kfree(q);
2c1ee898 2516 release_update_locks_for_mdev(matrix_mdev);
260f3ea1 2517}
3f85d1df
TK		 2518
2519/**
2520 * vfio_ap_mdev_resource_in_use: check whether any of a set of APQNs is
2521 * assigned to a mediated device under the control
2522 * of the vfio_ap device driver.
2523 *
2524 * @apm: a bitmap specifying a set of APIDs comprising the APQNs to check.
2525 * @aqm: a bitmap specifying a set of APQIs comprising the APQNs to check.
2526 *
2527 * Return:
2528 * * -EADDRINUSE if one or more of the APQNs specified via @apm/@aqm are
2529 * assigned to a mediated device under the control of the vfio_ap
2530 * device driver.
2531 * * Otherwise, return 0.
2532 */
2533int vfio_ap_mdev_resource_in_use(unsigned long *apm, unsigned long *aqm)
2534{
2535 int ret;
2536
2537 mutex_lock(&matrix_dev->guests_lock);
2538 mutex_lock(&matrix_dev->mdevs_lock);
d33d729a 2539 ret = vfio_ap_mdev_verify_no_sharing(NULL, apm, aqm);
3f85d1df
TK		 2540 mutex_unlock(&matrix_dev->mdevs_lock);
2541 mutex_unlock(&matrix_dev->guests_lock);
2542
2543 return ret;
2544}
eeb386ae
TK		 2545
2546/**
2547 * vfio_ap_mdev_hot_unplug_cfg - hot unplug the adapters, domains and control
2548 * domains that have been removed from the host's
2549 * AP configuration from a guest.
2550 *
2551 * @matrix_mdev: an ap_matrix_mdev object attached to a KVM guest.
2552 * @aprem: the adapters that have been removed from the host's AP configuration
2553 * @aqrem: the domains that have been removed from the host's AP configuration
2554 * @cdrem: the control domains that have been removed from the host's AP
2555 * configuration.
2556 */
2557static void vfio_ap_mdev_hot_unplug_cfg(struct ap_matrix_mdev *matrix_mdev,
2558 unsigned long *aprem,
2559 unsigned long *aqrem,
2560 unsigned long *cdrem)
2561{
2562 int do_hotplug = 0;
2563
2564 if (!bitmap_empty(aprem, AP_DEVICES)) {
2565 do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.apm,
2566 matrix_mdev->shadow_apcb.apm,
2567 aprem, AP_DEVICES);
2568 }
2569
2570 if (!bitmap_empty(aqrem, AP_DOMAINS)) {
2571 do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.aqm,
2572 matrix_mdev->shadow_apcb.aqm,
2573 aqrem, AP_DEVICES);
2574 }
2575
2576 if (!bitmap_empty(cdrem, AP_DOMAINS))
2577 do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.adm,
2578 matrix_mdev->shadow_apcb.adm,
2579 cdrem, AP_DOMAINS);
2580
2581 if (do_hotplug)
2582 vfio_ap_mdev_update_guest_apcb(matrix_mdev);
2583}
2584
2585/**
2586 * vfio_ap_mdev_cfg_remove - determines which guests are using the adapters,
2587 * domains and control domains that have been removed
2588 * from the host AP configuration and unplugs them
2589 * from those guests.
2590 *
2591 * @ap_remove: bitmap specifying which adapters have been removed from the host
2592 * config.
2593 * @aq_remove: bitmap specifying which domains have been removed from the host
2594 * config.
2595 * @cd_remove: bitmap specifying which control domains have been removed from
2596 * the host config.
2597 */
2598static void vfio_ap_mdev_cfg_remove(unsigned long *ap_remove,
2599 unsigned long *aq_remove,
2600 unsigned long *cd_remove)
2601{
2602 struct ap_matrix_mdev *matrix_mdev;
2603 DECLARE_BITMAP(aprem, AP_DEVICES);
2604 DECLARE_BITMAP(aqrem, AP_DOMAINS);
2605 DECLARE_BITMAP(cdrem, AP_DOMAINS);
2606 int do_remove = 0;
2607
2608 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
2609 mutex_lock(&matrix_mdev->kvm->lock);
2610 mutex_lock(&matrix_dev->mdevs_lock);
2611
2612 do_remove |= bitmap_and(aprem, ap_remove,
2613 matrix_mdev->matrix.apm,
2614 AP_DEVICES);
2615 do_remove |= bitmap_and(aqrem, aq_remove,
2616 matrix_mdev->matrix.aqm,
2617 AP_DOMAINS);
2618 do_remove |= bitmap_andnot(cdrem, cd_remove,
2619 matrix_mdev->matrix.adm,
2620 AP_DOMAINS);
2621
2622 if (do_remove)
2623 vfio_ap_mdev_hot_unplug_cfg(matrix_mdev, aprem, aqrem,
2624 cdrem);
2625
2626 mutex_unlock(&matrix_dev->mdevs_lock);
2627 mutex_unlock(&matrix_mdev->kvm->lock);
2628 }
2629}
2630
2631/**
2632 * vfio_ap_mdev_on_cfg_remove - responds to the removal of adapters, domains and
2633 * control domains from the host AP configuration
2634 * by unplugging them from the guests that are
2635 * using them.
2636 * @cur_config_info: the current host AP configuration information
2637 * @prev_config_info: the previous host AP configuration information
2638 */
2639static void vfio_ap_mdev_on_cfg_remove(struct ap_config_info *cur_config_info,
2640 struct ap_config_info *prev_config_info)
2641{
2642 int do_remove;
2643 DECLARE_BITMAP(aprem, AP_DEVICES);
2644 DECLARE_BITMAP(aqrem, AP_DOMAINS);
2645 DECLARE_BITMAP(cdrem, AP_DOMAINS);
2646
2647 do_remove = bitmap_andnot(aprem,
2648 (unsigned long *)prev_config_info->apm,
2649 (unsigned long *)cur_config_info->apm,
2650 AP_DEVICES);
2651 do_remove |= bitmap_andnot(aqrem,
2652 (unsigned long *)prev_config_info->aqm,
2653 (unsigned long *)cur_config_info->aqm,
2654 AP_DEVICES);
2655 do_remove |= bitmap_andnot(cdrem,
2656 (unsigned long *)prev_config_info->adm,
2657 (unsigned long *)cur_config_info->adm,
2658 AP_DEVICES);
2659
2660 if (do_remove)
2661 vfio_ap_mdev_cfg_remove(aprem, aqrem, cdrem);
2662}
2663
2664/**
2665 * vfio_ap_filter_apid_by_qtype: filter APIDs from an AP mask for adapters that
2666 * are older than AP type 10 (CEX4).
2667 * @apm: a bitmap of the APIDs to examine
2668 * @aqm: a bitmap of the APQIs of the queues to query for the AP type.
2669 */
2670static void vfio_ap_filter_apid_by_qtype(unsigned long *apm, unsigned long *aqm)
2671{
2672 bool apid_cleared;
2673 struct ap_queue_status status;
211c06d8 2674 unsigned long apid, apqi;
d4c53ae8 2675 struct ap_tapq_hwinfo info;
eeb386ae
TK		 2676
2677 for_each_set_bit_inv(apid, apm, AP_DEVICES) {
2678 apid_cleared = false;
2679
2680 for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) {
2681 status = ap_test_queue(AP_MKQID(apid, apqi), 1, &info);
2682 switch (status.response_code) {
2683 /*
2684 * According to the architecture in each case
2685 * below, the queue's info should be filled.
2686 */
2687 case AP_RESPONSE_NORMAL:
2688 case AP_RESPONSE_RESET_IN_PROGRESS:
2689 case AP_RESPONSE_DECONFIGURED:
2690 case AP_RESPONSE_CHECKSTOPPED:
2691 case AP_RESPONSE_BUSY:
eeb386ae
TK		 2692 /*
2693 * The vfio_ap device driver only
2694 * supports CEX4 and newer adapters, so
2695 * remove the APID if the adapter is
2696 * older than a CEX4.
2697 */
211c06d8 2698 if (info.at < AP_DEVICE_TYPE_CEX4) {
eeb386ae
TK		 2699 clear_bit_inv(apid, apm);
2700 apid_cleared = true;
2701 }
2702
2703 break;
2704
2705 default:
2706 /*
2707 * If we don't know the adapter type,
2708 * clear its APID since it can't be
2709 * determined whether the vfio_ap
2710 * device driver supports it.
2711 */
2712 clear_bit_inv(apid, apm);
2713 apid_cleared = true;
2714 break;
2715 }
2716
2717 /*
2718 * If we've already cleared the APID from the apm, there
2719 * is no need to continue examining the remainin AP
2720 * queues to determine the type of the adapter.
2721 */
2722 if (apid_cleared)
2723 continue;
2724 }
2725 }
2726}
2727
2728/**
2729 * vfio_ap_mdev_cfg_add - store bitmaps specifying the adapters, domains and
2730 * control domains that have been added to the host's
2731 * AP configuration for each matrix mdev to which they
2732 * are assigned.
2733 *
2734 * @apm_add: a bitmap specifying the adapters that have been added to the AP
2735 * configuration.
2736 * @aqm_add: a bitmap specifying the domains that have been added to the AP
2737 * configuration.
2738 * @adm_add: a bitmap specifying the control domains that have been added to the
2739 * AP configuration.
2740 */
2741static void vfio_ap_mdev_cfg_add(unsigned long *apm_add, unsigned long *aqm_add,
2742 unsigned long *adm_add)
2743{
2744 struct ap_matrix_mdev *matrix_mdev;
2745
2746 if (list_empty(&matrix_dev->mdev_list))
2747 return;
2748
2749 vfio_ap_filter_apid_by_qtype(apm_add, aqm_add);
2750
2751 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
2752 bitmap_and(matrix_mdev->apm_add,
2753 matrix_mdev->matrix.apm, apm_add, AP_DEVICES);
2754 bitmap_and(matrix_mdev->aqm_add,
2755 matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS);
2756 bitmap_and(matrix_mdev->adm_add,
2757 matrix_mdev->matrix.adm, adm_add, AP_DEVICES);
2758 }
2759}
2760
2761/**
2762 * vfio_ap_mdev_on_cfg_add - responds to the addition of adapters, domains and
2763 * control domains to the host AP configuration
2764 * by updating the bitmaps that specify what adapters,
2765 * domains and control domains have been added so they
2766 * can be hot plugged into the guest when the AP bus
2767 * scan completes (see vfio_ap_on_scan_complete
2768 * function).
2769 * @cur_config_info: the current AP configuration information
2770 * @prev_config_info: the previous AP configuration information
2771 */
2772static void vfio_ap_mdev_on_cfg_add(struct ap_config_info *cur_config_info,
2773 struct ap_config_info *prev_config_info)
2774{
2775 bool do_add;
2776 DECLARE_BITMAP(apm_add, AP_DEVICES);
2777 DECLARE_BITMAP(aqm_add, AP_DOMAINS);
2778 DECLARE_BITMAP(adm_add, AP_DOMAINS);
2779
2780 do_add = bitmap_andnot(apm_add,
2781 (unsigned long *)cur_config_info->apm,
2782 (unsigned long *)prev_config_info->apm,
2783 AP_DEVICES);
2784 do_add |= bitmap_andnot(aqm_add,
2785 (unsigned long *)cur_config_info->aqm,
2786 (unsigned long *)prev_config_info->aqm,
2787 AP_DOMAINS);
2788 do_add |= bitmap_andnot(adm_add,
2789 (unsigned long *)cur_config_info->adm,
2790 (unsigned long *)prev_config_info->adm,
2791 AP_DOMAINS);
2792
2793 if (do_add)
2794 vfio_ap_mdev_cfg_add(apm_add, aqm_add, adm_add);
2795}
2796
2797/**
2798 * vfio_ap_on_cfg_changed - handles notification of changes to the host AP
2799 * configuration.
2800 *
2801 * @cur_cfg_info: the current host AP configuration
2802 * @prev_cfg_info: the previous host AP configuration
2803 */
2804void vfio_ap_on_cfg_changed(struct ap_config_info *cur_cfg_info,
2805 struct ap_config_info *prev_cfg_info)
2806{
2807 if (!cur_cfg_info || !prev_cfg_info)
2808 return;
2809
2810 mutex_lock(&matrix_dev->guests_lock);
2811
2812 vfio_ap_mdev_on_cfg_remove(cur_cfg_info, prev_cfg_info);
2813 vfio_ap_mdev_on_cfg_add(cur_cfg_info, prev_cfg_info);
2814 memcpy(&matrix_dev->info, cur_cfg_info, sizeof(*cur_cfg_info));
2815
2816 mutex_unlock(&matrix_dev->guests_lock);
2817}
2818
2819static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev)
2820{
f848cba7 2821 DECLARE_BITMAP(apm_filtered, AP_DEVICES);
850fb7fa 2822 bool filter_domains, filter_adapters, filter_cdoms, do_hotplug = false;
eeb386ae
TK		 2823
2824 mutex_lock(&matrix_mdev->kvm->lock);
2825 mutex_lock(&matrix_dev->mdevs_lock);
2826
850fb7fa
TK		 2827 filter_adapters = bitmap_intersects(matrix_mdev->matrix.apm,
2828 matrix_mdev->apm_add, AP_DEVICES);
2829 filter_domains = bitmap_intersects(matrix_mdev->matrix.aqm,
2830 matrix_mdev->aqm_add, AP_DOMAINS);
2831 filter_cdoms = bitmap_intersects(matrix_mdev->matrix.adm,
2832 matrix_mdev->adm_add, AP_DOMAINS);
2833
2834 if (filter_adapters || filter_domains)
f848cba7 2835 do_hotplug = vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered);
eeb386ae 2836
850fb7fa 2837 if (filter_cdoms)
eeb386ae
TK		 2838 do_hotplug |= vfio_ap_mdev_filter_cdoms(matrix_mdev);
2839
2840 if (do_hotplug)
2841 vfio_ap_mdev_update_guest_apcb(matrix_mdev);
2842
f848cba7
TK		 2843 reset_queues_for_apids(matrix_mdev, apm_filtered);
2844
eeb386ae
TK		 2845 mutex_unlock(&matrix_dev->mdevs_lock);
2846 mutex_unlock(&matrix_mdev->kvm->lock);
2847}
2848
2849void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info,
2850 struct ap_config_info *old_config_info)
2851{
2852 struct ap_matrix_mdev *matrix_mdev;
2853
2854 mutex_lock(&matrix_dev->guests_lock);
2855
2856 list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
2857 if (bitmap_empty(matrix_mdev->apm_add, AP_DEVICES) &&
2858 bitmap_empty(matrix_mdev->aqm_add, AP_DOMAINS) &&
2859 bitmap_empty(matrix_mdev->adm_add, AP_DOMAINS))
2860 continue;
2861
2862 vfio_ap_mdev_hot_plug_cfg(matrix_mdev);
2863 bitmap_clear(matrix_mdev->apm_add, 0, AP_DEVICES);
2864 bitmap_clear(matrix_mdev->aqm_add, 0, AP_DOMAINS);
2865 bitmap_clear(matrix_mdev->adm_add, 0, AP_DOMAINS);
2866 }
2867
2868 mutex_unlock(&matrix_dev->guests_lock);
2869}
Linux 6.x block layer and io_uring tree(s)