projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: PPC: Book3S HV: XIVE: Add a mapping for the source ESB pages
[linux-2.6-block.git] / arch / powerpc / kvm / book3s_xive_native.c
CommitLineData
90c73795
CLG		 1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2017-2019, IBM Corporation.
4 */
5
6#define pr_fmt(fmt) "xive-kvm: " fmt
7
8#include <linux/kernel.h>
9#include <linux/kvm_host.h>
10#include <linux/err.h>
11#include <linux/gfp.h>
12#include <linux/spinlock.h>
13#include <linux/delay.h>
14#include <asm/uaccess.h>
15#include <asm/kvm_book3s.h>
16#include <asm/kvm_ppc.h>
17#include <asm/hvcall.h>
18#include <asm/xive.h>
19#include <asm/xive-regs.h>
20#include <asm/debug.h>
21#include <asm/debugfs.h>
22#include <asm/opal.h>
23
24#include <linux/debugfs.h>
25#include <linux/seq_file.h>
26
27#include "book3s_xive.h"
28
4131f83c
CLG		 29static u8 xive_vm_esb_load(struct xive_irq_data *xd, u32 offset)
30{
31 u64 val;
32
33 if (xd->flags & XIVE_IRQ_FLAG_SHIFT_BUG)
34 offset |= offset << 4;
35
36 val = in_be64(xd->eoi_mmio + offset);
37 return (u8)val;
38}
39
eacc56bb
CLG		 40static void kvmppc_xive_native_cleanup_queue(struct kvm_vcpu *vcpu, int prio)
41{
42 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
43 struct xive_q *q = &xc->queues[prio];
44
45 xive_native_disable_queue(xc->vp_id, q, prio);
46 if (q->qpage) {
47 put_page(virt_to_page(q->qpage));
48 q->qpage = NULL;
49 }
50}
51
52void kvmppc_xive_native_cleanup_vcpu(struct kvm_vcpu *vcpu)
53{
54 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
55 int i;
56
57 if (!kvmppc_xive_enabled(vcpu))
58 return;
59
60 if (!xc)
61 return;
62
63 pr_devel("native_cleanup_vcpu(cpu=%d)\n", xc->server_num);
64
65 /* Ensure no interrupt is still routed to that VP */
66 xc->valid = false;
67 kvmppc_xive_disable_vcpu_interrupts(vcpu);
68
69 /* Disable the VP */
70 xive_native_disable_vp(xc->vp_id);
71
72 /* Free the queues & associated interrupts */
73 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
74 /* Free the escalation irq */
75 if (xc->esc_virq[i]) {
76 free_irq(xc->esc_virq[i], vcpu);
77 irq_dispose_mapping(xc->esc_virq[i]);
78 kfree(xc->esc_virq_names[i]);
79 xc->esc_virq[i] = 0;
80 }
81
82 /* Free the queue */
83 kvmppc_xive_native_cleanup_queue(vcpu, i);
84 }
85
86 /* Free the VP */
87 kfree(xc);
88
89 /* Cleanup the vcpu */
90 vcpu->arch.irq_type = KVMPPC_IRQ_DEFAULT;
91 vcpu->arch.xive_vcpu = NULL;
92}
93
94int kvmppc_xive_native_connect_vcpu(struct kvm_device *dev,
95 struct kvm_vcpu *vcpu, u32 server_num)
96{
97 struct kvmppc_xive *xive = dev->private;
98 struct kvmppc_xive_vcpu *xc = NULL;
99 int rc;
100
101 pr_devel("native_connect_vcpu(server=%d)\n", server_num);
102
103 if (dev->ops != &kvm_xive_native_ops) {
104 pr_devel("Wrong ops !\n");
105 return -EPERM;
106 }
107 if (xive->kvm != vcpu->kvm)
108 return -EPERM;
109 if (vcpu->arch.irq_type != KVMPPC_IRQ_DEFAULT)
110 return -EBUSY;
111 if (server_num >= KVM_MAX_VCPUS) {
112 pr_devel("Out of bounds !\n");
113 return -EINVAL;
114 }
115
116 mutex_lock(&vcpu->kvm->lock);
117
118 if (kvmppc_xive_find_server(vcpu->kvm, server_num)) {
119 pr_devel("Duplicate !\n");
120 rc = -EEXIST;
121 goto bail;
122 }
123
124 xc = kzalloc(sizeof(*xc), GFP_KERNEL);
125 if (!xc) {
126 rc = -ENOMEM;
127 goto bail;
128 }
129
130 vcpu->arch.xive_vcpu = xc;
131 xc->xive = xive;
132 xc->vcpu = vcpu;
133 xc->server_num = server_num;
134
135 xc->vp_id = kvmppc_xive_vp(xive, server_num);
136 xc->valid = true;
137 vcpu->arch.irq_type = KVMPPC_IRQ_XIVE;
138
139 rc = xive_native_get_vp_info(xc->vp_id, &xc->vp_cam, &xc->vp_chip_id);
140 if (rc) {
141 pr_err("Failed to get VP info from OPAL: %d\n", rc);
142 goto bail;
143 }
144
145 /*
146 * Enable the VP first as the single escalation mode will
147 * affect escalation interrupts numbering
148 */
149 rc = xive_native_enable_vp(xc->vp_id, xive->single_escalation);
150 if (rc) {
151 pr_err("Failed to enable VP in OPAL: %d\n", rc);
152 goto bail;
153 }
154
155 /* Configure VCPU fields for use by assembly push/pull */
156 vcpu->arch.xive_saved_state.w01 = cpu_to_be64(0xff000000);
157 vcpu->arch.xive_cam_word = cpu_to_be32(xc->vp_cam | TM_QW1W2_VO);
158
159 /* TODO: reset all queues to a clean state ? */
160bail:
161 mutex_unlock(&vcpu->kvm->lock);
162 if (rc)
163 kvmppc_xive_native_cleanup_vcpu(vcpu);
164
165 return rc;
166}
167
6520ca64
CLG		 168static vm_fault_t xive_native_esb_fault(struct vm_fault *vmf)
169{
170 struct vm_area_struct *vma = vmf->vma;
171 struct kvm_device *dev = vma->vm_file->private_data;
172 struct kvmppc_xive *xive = dev->private;
173 struct kvmppc_xive_src_block *sb;
174 struct kvmppc_xive_irq_state *state;
175 struct xive_irq_data *xd;
176 u32 hw_num;
177 u16 src;
178 u64 page;
179 unsigned long irq;
180 u64 page_offset;
181
182 /*
183 * Linux/KVM uses a two pages ESB setting, one for trigger and
184 * one for EOI
185 */
186 page_offset = vmf->pgoff - vma->vm_pgoff;
187 irq = page_offset / 2;
188
189 sb = kvmppc_xive_find_source(xive, irq, &src);
190 if (!sb) {
191 pr_devel("%s: source %lx not found !\n", __func__, irq);
192 return VM_FAULT_SIGBUS;
193 }
194
195 state = &sb->irq_state[src];
196 kvmppc_xive_select_irq(state, &hw_num, &xd);
197
198 arch_spin_lock(&sb->lock);
199
200 /*
201 * first/even page is for trigger
202 * second/odd page is for EOI and management.
203 */
204 page = page_offset % 2 ? xd->eoi_page : xd->trig_page;
205 arch_spin_unlock(&sb->lock);
206
207 if (WARN_ON(!page)) {
208 pr_err("%s: acessing invalid ESB page for source %lx !\n",
209 __func__, irq);
210 return VM_FAULT_SIGBUS;
211 }
212
213 vmf_insert_pfn(vma, vmf->address, page >> PAGE_SHIFT);
214 return VM_FAULT_NOPAGE;
215}
216
217static const struct vm_operations_struct xive_native_esb_vmops = {
218 .fault = xive_native_esb_fault,
219};
220
39e9af3d
CLG		 221static vm_fault_t xive_native_tima_fault(struct vm_fault *vmf)
222{
223 struct vm_area_struct *vma = vmf->vma;
224
225 switch (vmf->pgoff - vma->vm_pgoff) {
226 case 0: /* HW - forbid access */
227 case 1: /* HV - forbid access */
228 return VM_FAULT_SIGBUS;
229 case 2: /* OS */
230 vmf_insert_pfn(vma, vmf->address, xive_tima_os >> PAGE_SHIFT);
231 return VM_FAULT_NOPAGE;
232 case 3: /* USER - TODO */
233 default:
234 return VM_FAULT_SIGBUS;
235 }
236}
237
238static const struct vm_operations_struct xive_native_tima_vmops = {
239 .fault = xive_native_tima_fault,
240};
241
242static int kvmppc_xive_native_mmap(struct kvm_device *dev,
243 struct vm_area_struct *vma)
244{
245 /* We only allow mappings at fixed offset for now */
246 if (vma->vm_pgoff == KVM_XIVE_TIMA_PAGE_OFFSET) {
247 if (vma_pages(vma) > 4)
248 return -EINVAL;
249 vma->vm_ops = &xive_native_tima_vmops;
6520ca64
CLG		 250 } else if (vma->vm_pgoff == KVM_XIVE_ESB_PAGE_OFFSET) {
251 if (vma_pages(vma) > KVMPPC_XIVE_NR_IRQS * 2)
252 return -EINVAL;
253 vma->vm_ops = &xive_native_esb_vmops;
39e9af3d
CLG		 254 } else {
255 return -EINVAL;
256 }
257
258 vma->vm_flags |= VM_IO | VM_PFNMAP;
259 vma->vm_page_prot = pgprot_noncached_wc(vma->vm_page_prot);
260 return 0;
261}
262
4131f83c
CLG		 263static int kvmppc_xive_native_set_source(struct kvmppc_xive *xive, long irq,
264 u64 addr)
265{
266 struct kvmppc_xive_src_block *sb;
267 struct kvmppc_xive_irq_state *state;
268 u64 __user *ubufp = (u64 __user *) addr;
269 u64 val;
270 u16 idx;
271 int rc;
272
273 pr_devel("%s irq=0x%lx\n", __func__, irq);
274
275 if (irq < KVMPPC_XIVE_FIRST_IRQ || irq >= KVMPPC_XIVE_NR_IRQS)
276 return -E2BIG;
277
278 sb = kvmppc_xive_find_source(xive, irq, &idx);
279 if (!sb) {
280 pr_debug("No source, creating source block...\n");
281 sb = kvmppc_xive_create_src_block(xive, irq);
282 if (!sb) {
283 pr_err("Failed to create block...\n");
284 return -ENOMEM;
285 }
286 }
287 state = &sb->irq_state[idx];
288
289 if (get_user(val, ubufp)) {
290 pr_err("fault getting user info !\n");
291 return -EFAULT;
292 }
293
294 arch_spin_lock(&sb->lock);
295
296 /*
297 * If the source doesn't already have an IPI, allocate
298 * one and get the corresponding data
299 */
300 if (!state->ipi_number) {
301 state->ipi_number = xive_native_alloc_irq();
302 if (state->ipi_number == 0) {
303 pr_err("Failed to allocate IRQ !\n");
304 rc = -ENXIO;
305 goto unlock;
306 }
307 xive_native_populate_irq_data(state->ipi_number,
308 &state->ipi_data);
309 pr_debug("%s allocated hw_irq=0x%x for irq=0x%lx\n", __func__,
310 state->ipi_number, irq);
311 }
312
313 /* Restore LSI state */
314 if (val & KVM_XIVE_LEVEL_SENSITIVE) {
315 state->lsi = true;
316 if (val & KVM_XIVE_LEVEL_ASSERTED)
317 state->asserted = true;
318 pr_devel(" LSI ! Asserted=%d\n", state->asserted);
319 }
320
321 /* Mask IRQ to start with */
322 state->act_server = 0;
323 state->act_priority = MASKED;
324 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
325 xive_native_configure_irq(state->ipi_number, 0, MASKED, 0);
326
327 /* Increment the number of valid sources and mark this one valid */
328 if (!state->valid)
329 xive->src_count++;
330 state->valid = true;
331
332 rc = 0;
333
334unlock:
335 arch_spin_unlock(&sb->lock);
336
337 return rc;
338}
339
e8676ce5
CLG		 340static int kvmppc_xive_native_update_source_config(struct kvmppc_xive *xive,
341 struct kvmppc_xive_src_block *sb,
342 struct kvmppc_xive_irq_state *state,
343 u32 server, u8 priority, bool masked,
344 u32 eisn)
345{
346 struct kvm *kvm = xive->kvm;
347 u32 hw_num;
348 int rc = 0;
349
350 arch_spin_lock(&sb->lock);
351
352 if (state->act_server == server && state->act_priority == priority &&
353 state->eisn == eisn)
354 goto unlock;
355
356 pr_devel("new_act_prio=%d new_act_server=%d mask=%d act_server=%d act_prio=%d\n",
357 priority, server, masked, state->act_server,
358 state->act_priority);
359
360 kvmppc_xive_select_irq(state, &hw_num, NULL);
361
362 if (priority != MASKED && !masked) {
363 rc = kvmppc_xive_select_target(kvm, &server, priority);
364 if (rc)
365 goto unlock;
366
367 state->act_priority = priority;
368 state->act_server = server;
369 state->eisn = eisn;
370
371 rc = xive_native_configure_irq(hw_num,
372 kvmppc_xive_vp(xive, server),
373 priority, eisn);
374 } else {
375 state->act_priority = MASKED;
376 state->act_server = 0;
377 state->eisn = 0;
378
379 rc = xive_native_configure_irq(hw_num, 0, MASKED, 0);
380 }
381
382unlock:
383 arch_spin_unlock(&sb->lock);
384 return rc;
385}
386
387static int kvmppc_xive_native_set_source_config(struct kvmppc_xive *xive,
388 long irq, u64 addr)
389{
390 struct kvmppc_xive_src_block *sb;
391 struct kvmppc_xive_irq_state *state;
392 u64 __user *ubufp = (u64 __user *) addr;
393 u16 src;
394 u64 kvm_cfg;
395 u32 server;
396 u8 priority;
397 bool masked;
398 u32 eisn;
399
400 sb = kvmppc_xive_find_source(xive, irq, &src);
401 if (!sb)
402 return -ENOENT;
403
404 state = &sb->irq_state[src];
405
406 if (!state->valid)
407 return -EINVAL;
408
409 if (get_user(kvm_cfg, ubufp))
410 return -EFAULT;
411
412 pr_devel("%s irq=0x%lx cfg=%016llx\n", __func__, irq, kvm_cfg);
413
414 priority = (kvm_cfg & KVM_XIVE_SOURCE_PRIORITY_MASK) >>
415 KVM_XIVE_SOURCE_PRIORITY_SHIFT;
416 server = (kvm_cfg & KVM_XIVE_SOURCE_SERVER_MASK) >>
417 KVM_XIVE_SOURCE_SERVER_SHIFT;
418 masked = (kvm_cfg & KVM_XIVE_SOURCE_MASKED_MASK) >>
419 KVM_XIVE_SOURCE_MASKED_SHIFT;
420 eisn = (kvm_cfg & KVM_XIVE_SOURCE_EISN_MASK) >>
421 KVM_XIVE_SOURCE_EISN_SHIFT;
422
423 if (priority != xive_prio_from_guest(priority)) {
424 pr_err("invalid priority for queue %d for VCPU %d\n",
425 priority, server);
426 return -EINVAL;
427 }
428
429 return kvmppc_xive_native_update_source_config(xive, sb, state, server,
430 priority, masked, eisn);
431}
432
7b46b616
CLG		 433static int kvmppc_xive_native_sync_source(struct kvmppc_xive *xive,
434 long irq, u64 addr)
435{
436 struct kvmppc_xive_src_block *sb;
437 struct kvmppc_xive_irq_state *state;
438 struct xive_irq_data *xd;
439 u32 hw_num;
440 u16 src;
441 int rc = 0;
442
443 pr_devel("%s irq=0x%lx", __func__, irq);
444
445 sb = kvmppc_xive_find_source(xive, irq, &src);
446 if (!sb)
447 return -ENOENT;
448
449 state = &sb->irq_state[src];
450
451 rc = -EINVAL;
452
453 arch_spin_lock(&sb->lock);
454
455 if (state->valid) {
456 kvmppc_xive_select_irq(state, &hw_num, &xd);
457 xive_native_sync_source(hw_num);
458 rc = 0;
459 }
460
461 arch_spin_unlock(&sb->lock);
462 return rc;
463}
464
13ce3297
CLG		 465static int xive_native_validate_queue_size(u32 qshift)
466{
467 /*
468 * We only support 64K pages for the moment. This is also
469 * advertised in the DT property "ibm,xive-eq-sizes"
470 */
471 switch (qshift) {
472 case 0: /* EQ reset */
473 case 16:
474 return 0;
475 case 12:
476 case 21:
477 case 24:
478 default:
479 return -EINVAL;
480 }
481}
482
483static int kvmppc_xive_native_set_queue_config(struct kvmppc_xive *xive,
484 long eq_idx, u64 addr)
485{
486 struct kvm *kvm = xive->kvm;
487 struct kvm_vcpu *vcpu;
488 struct kvmppc_xive_vcpu *xc;
489 void __user *ubufp = (void __user *) addr;
490 u32 server;
491 u8 priority;
492 struct kvm_ppc_xive_eq kvm_eq;
493 int rc;
494 __be32 *qaddr = 0;
495 struct page *page;
496 struct xive_q *q;
497 gfn_t gfn;
498 unsigned long page_size;
499
500 /*
501 * Demangle priority/server tuple from the EQ identifier
502 */
503 priority = (eq_idx & KVM_XIVE_EQ_PRIORITY_MASK) >>
504 KVM_XIVE_EQ_PRIORITY_SHIFT;
505 server = (eq_idx & KVM_XIVE_EQ_SERVER_MASK) >>
506 KVM_XIVE_EQ_SERVER_SHIFT;
507
508 if (copy_from_user(&kvm_eq, ubufp, sizeof(kvm_eq)))
509 return -EFAULT;
510
511 vcpu = kvmppc_xive_find_server(kvm, server);
512 if (!vcpu) {
513 pr_err("Can't find server %d\n", server);
514 return -ENOENT;
515 }
516 xc = vcpu->arch.xive_vcpu;
517
518 if (priority != xive_prio_from_guest(priority)) {
519 pr_err("Trying to restore invalid queue %d for VCPU %d\n",
520 priority, server);
521 return -EINVAL;
522 }
523 q = &xc->queues[priority];
524
525 pr_devel("%s VCPU %d priority %d fl:%x shift:%d addr:%llx g:%d idx:%d\n",
526 __func__, server, priority, kvm_eq.flags,
527 kvm_eq.qshift, kvm_eq.qaddr, kvm_eq.qtoggle, kvm_eq.qindex);
528
529 /*
530 * sPAPR specifies a "Unconditional Notify (n) flag" for the
531 * H_INT_SET_QUEUE_CONFIG hcall which forces notification
532 * without using the coalescing mechanisms provided by the
533 * XIVE END ESBs. This is required on KVM as notification
534 * using the END ESBs is not supported.
535 */
536 if (kvm_eq.flags != KVM_XIVE_EQ_ALWAYS_NOTIFY) {
537 pr_err("invalid flags %d\n", kvm_eq.flags);
538 return -EINVAL;
539 }
540
541 rc = xive_native_validate_queue_size(kvm_eq.qshift);
542 if (rc) {
543 pr_err("invalid queue size %d\n", kvm_eq.qshift);
544 return rc;
545 }
546
547 /* reset queue and disable queueing */
548 if (!kvm_eq.qshift) {
549 q->guest_qaddr = 0;
550 q->guest_qshift = 0;
551
552 rc = xive_native_configure_queue(xc->vp_id, q, priority,
553 NULL, 0, true);
554 if (rc) {
555 pr_err("Failed to reset queue %d for VCPU %d: %d\n",
556 priority, xc->server_num, rc);
557 return rc;
558 }
559
560 if (q->qpage) {
561 put_page(virt_to_page(q->qpage));
562 q->qpage = NULL;
563 }
564
565 return 0;
566 }
567
568 if (kvm_eq.qaddr & ((1ull << kvm_eq.qshift) - 1)) {
569 pr_err("queue page is not aligned %llx/%llx\n", kvm_eq.qaddr,
570 1ull << kvm_eq.qshift);
571 return -EINVAL;
572 }
573
574 gfn = gpa_to_gfn(kvm_eq.qaddr);
575 page = gfn_to_page(kvm, gfn);
576 if (is_error_page(page)) {
577 pr_err("Couldn't get queue page %llx!\n", kvm_eq.qaddr);
578 return -EINVAL;
579 }
580
581 page_size = kvm_host_page_size(kvm, gfn);
582 if (1ull << kvm_eq.qshift > page_size) {
583 pr_warn("Incompatible host page size %lx!\n", page_size);
584 return -EINVAL;
585 }
586
587 qaddr = page_to_virt(page) + (kvm_eq.qaddr & ~PAGE_MASK);
588
589 /*
590 * Backup the queue page guest address to the mark EQ page
591 * dirty for migration.
592 */
593 q->guest_qaddr = kvm_eq.qaddr;
594 q->guest_qshift = kvm_eq.qshift;
595
596 /*
597 * Unconditional Notification is forced by default at the
598 * OPAL level because the use of END ESBs is not supported by
599 * Linux.
600 */
601 rc = xive_native_configure_queue(xc->vp_id, q, priority,
602 (__be32 *) qaddr, kvm_eq.qshift, true);
603 if (rc) {
604 pr_err("Failed to configure queue %d for VCPU %d: %d\n",
605 priority, xc->server_num, rc);
606 put_page(page);
607 return rc;
608 }
609
610 /*
611 * Only restore the queue state when needed. When doing the
612 * H_INT_SET_SOURCE_CONFIG hcall, it should not.
613 */
614 if (kvm_eq.qtoggle != 1 || kvm_eq.qindex != 0) {
615 rc = xive_native_set_queue_state(xc->vp_id, priority,
616 kvm_eq.qtoggle,
617 kvm_eq.qindex);
618 if (rc)
619 goto error;
620 }
621
622 rc = kvmppc_xive_attach_escalation(vcpu, priority,
623 xive->single_escalation);
624error:
625 if (rc)
626 kvmppc_xive_native_cleanup_queue(vcpu, priority);
627 return rc;
628}
629
630static int kvmppc_xive_native_get_queue_config(struct kvmppc_xive *xive,
631 long eq_idx, u64 addr)
632{
633 struct kvm *kvm = xive->kvm;
634 struct kvm_vcpu *vcpu;
635 struct kvmppc_xive_vcpu *xc;
636 struct xive_q *q;
637 void __user *ubufp = (u64 __user *) addr;
638 u32 server;
639 u8 priority;
640 struct kvm_ppc_xive_eq kvm_eq;
641 u64 qaddr;
642 u64 qshift;
643 u64 qeoi_page;
644 u32 escalate_irq;
645 u64 qflags;
646 int rc;
647
648 /*
649 * Demangle priority/server tuple from the EQ identifier
650 */
651 priority = (eq_idx & KVM_XIVE_EQ_PRIORITY_MASK) >>
652 KVM_XIVE_EQ_PRIORITY_SHIFT;
653 server = (eq_idx & KVM_XIVE_EQ_SERVER_MASK) >>
654 KVM_XIVE_EQ_SERVER_SHIFT;
655
656 vcpu = kvmppc_xive_find_server(kvm, server);
657 if (!vcpu) {
658 pr_err("Can't find server %d\n", server);
659 return -ENOENT;
660 }
661 xc = vcpu->arch.xive_vcpu;
662
663 if (priority != xive_prio_from_guest(priority)) {
664 pr_err("invalid priority for queue %d for VCPU %d\n",
665 priority, server);
666 return -EINVAL;
667 }
668 q = &xc->queues[priority];
669
670 memset(&kvm_eq, 0, sizeof(kvm_eq));
671
672 if (!q->qpage)
673 return 0;
674
675 rc = xive_native_get_queue_info(xc->vp_id, priority, &qaddr, &qshift,
676 &qeoi_page, &escalate_irq, &qflags);
677 if (rc)
678 return rc;
679
680 kvm_eq.flags = 0;
681 if (qflags & OPAL_XIVE_EQ_ALWAYS_NOTIFY)
682 kvm_eq.flags |= KVM_XIVE_EQ_ALWAYS_NOTIFY;
683
684 kvm_eq.qshift = q->guest_qshift;
685 kvm_eq.qaddr = q->guest_qaddr;
686
687 rc = xive_native_get_queue_state(xc->vp_id, priority, &kvm_eq.qtoggle,
688 &kvm_eq.qindex);
689 if (rc)
690 return rc;
691
692 pr_devel("%s VCPU %d priority %d fl:%x shift:%d addr:%llx g:%d idx:%d\n",
693 __func__, server, priority, kvm_eq.flags,
694 kvm_eq.qshift, kvm_eq.qaddr, kvm_eq.qtoggle, kvm_eq.qindex);
695
696 if (copy_to_user(ubufp, &kvm_eq, sizeof(kvm_eq)))
697 return -EFAULT;
698
699 return 0;
700}
701
5ca80647
CLG		 702static void kvmppc_xive_reset_sources(struct kvmppc_xive_src_block *sb)
703{
704 int i;
705
706 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
707 struct kvmppc_xive_irq_state *state = &sb->irq_state[i];
708
709 if (!state->valid)
710 continue;
711
712 if (state->act_priority == MASKED)
713 continue;
714
715 state->eisn = 0;
716 state->act_server = 0;
717 state->act_priority = MASKED;
718 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
719 xive_native_configure_irq(state->ipi_number, 0, MASKED, 0);
720 if (state->pt_number) {
721 xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_01);
722 xive_native_configure_irq(state->pt_number,
723 0, MASKED, 0);
724 }
725 }
726}
727
728static int kvmppc_xive_reset(struct kvmppc_xive *xive)
729{
730 struct kvm *kvm = xive->kvm;
731 struct kvm_vcpu *vcpu;
732 unsigned int i;
733
734 pr_devel("%s\n", __func__);
735
736 mutex_lock(&kvm->lock);
737
738 kvm_for_each_vcpu(i, vcpu, kvm) {
739 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
740 unsigned int prio;
741
742 if (!xc)
743 continue;
744
745 kvmppc_xive_disable_vcpu_interrupts(vcpu);
746
747 for (prio = 0; prio < KVMPPC_XIVE_Q_COUNT; prio++) {
748
749 /* Single escalation, no queue 7 */
750 if (prio == 7 && xive->single_escalation)
751 break;
752
753 if (xc->esc_virq[prio]) {
754 free_irq(xc->esc_virq[prio], vcpu);
755 irq_dispose_mapping(xc->esc_virq[prio]);
756 kfree(xc->esc_virq_names[prio]);
757 xc->esc_virq[prio] = 0;
758 }
759
760 kvmppc_xive_native_cleanup_queue(vcpu, prio);
761 }
762 }
763
764 for (i = 0; i <= xive->max_sbid; i++) {
765 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
766
767 if (sb) {
768 arch_spin_lock(&sb->lock);
769 kvmppc_xive_reset_sources(sb);
770 arch_spin_unlock(&sb->lock);
771 }
772 }
773
774 mutex_unlock(&kvm->lock);
775
776 return 0;
777}
778
e6714bd1
CLG		 779static void kvmppc_xive_native_sync_sources(struct kvmppc_xive_src_block *sb)
780{
781 int j;
782
783 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) {
784 struct kvmppc_xive_irq_state *state = &sb->irq_state[j];
785 struct xive_irq_data *xd;
786 u32 hw_num;
787
788 if (!state->valid)
789 continue;
790
791 /*
792 * The struct kvmppc_xive_irq_state reflects the state
793 * of the EAS configuration and not the state of the
794 * source. The source is masked setting the PQ bits to
795 * '-Q', which is what is being done before calling
796 * the KVM_DEV_XIVE_EQ_SYNC control.
797 *
798 * If a source EAS is configured, OPAL syncs the XIVE
799 * IC of the source and the XIVE IC of the previous
800 * target if any.
801 *
802 * So it should be fine ignoring MASKED sources as
803 * they have been synced already.
804 */
805 if (state->act_priority == MASKED)
806 continue;
807
808 kvmppc_xive_select_irq(state, &hw_num, &xd);
809 xive_native_sync_source(hw_num);
810 xive_native_sync_queue(hw_num);
811 }
812}
813
814static int kvmppc_xive_native_vcpu_eq_sync(struct kvm_vcpu *vcpu)
815{
816 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
817 unsigned int prio;
818
819 if (!xc)
820 return -ENOENT;
821
822 for (prio = 0; prio < KVMPPC_XIVE_Q_COUNT; prio++) {
823 struct xive_q *q = &xc->queues[prio];
824
825 if (!q->qpage)
826 continue;
827
828 /* Mark EQ page dirty for migration */
829 mark_page_dirty(vcpu->kvm, gpa_to_gfn(q->guest_qaddr));
830 }
831 return 0;
832}
833
834static int kvmppc_xive_native_eq_sync(struct kvmppc_xive *xive)
835{
836 struct kvm *kvm = xive->kvm;
837 struct kvm_vcpu *vcpu;
838 unsigned int i;
839
840 pr_devel("%s\n", __func__);
841
842 mutex_lock(&kvm->lock);
843 for (i = 0; i <= xive->max_sbid; i++) {
844 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
845
846 if (sb) {
847 arch_spin_lock(&sb->lock);
848 kvmppc_xive_native_sync_sources(sb);
849 arch_spin_unlock(&sb->lock);
850 }
851 }
852
853 kvm_for_each_vcpu(i, vcpu, kvm) {
854 kvmppc_xive_native_vcpu_eq_sync(vcpu);
855 }
856 mutex_unlock(&kvm->lock);
857
858 return 0;
859}
860
90c73795
CLG		 861static int kvmppc_xive_native_set_attr(struct kvm_device *dev,
862 struct kvm_device_attr *attr)
863{
4131f83c
CLG		 864 struct kvmppc_xive *xive = dev->private;
865
90c73795
CLG		 866 switch (attr->group) {
867 case KVM_DEV_XIVE_GRP_CTRL:
5ca80647
CLG		 868 switch (attr->attr) {
869 case KVM_DEV_XIVE_RESET:
870 return kvmppc_xive_reset(xive);
e6714bd1
CLG		 871 case KVM_DEV_XIVE_EQ_SYNC:
872 return kvmppc_xive_native_eq_sync(xive);
5ca80647 873 }
90c73795 874 break;
4131f83c
CLG		 875 case KVM_DEV_XIVE_GRP_SOURCE:
876 return kvmppc_xive_native_set_source(xive, attr->attr,
877 attr->addr);
e8676ce5
CLG		 878 case KVM_DEV_XIVE_GRP_SOURCE_CONFIG:
879 return kvmppc_xive_native_set_source_config(xive, attr->attr,
880 attr->addr);
13ce3297
CLG		 881 case KVM_DEV_XIVE_GRP_EQ_CONFIG:
882 return kvmppc_xive_native_set_queue_config(xive, attr->attr,
883 attr->addr);
7b46b616
CLG		 884 case KVM_DEV_XIVE_GRP_SOURCE_SYNC:
885 return kvmppc_xive_native_sync_source(xive, attr->attr,
886 attr->addr);
90c73795
CLG		 887 }
888 return -ENXIO;
889}
890
891static int kvmppc_xive_native_get_attr(struct kvm_device *dev,
892 struct kvm_device_attr *attr)
893{
13ce3297
CLG		 894 struct kvmppc_xive *xive = dev->private;
895
896 switch (attr->group) {
897 case KVM_DEV_XIVE_GRP_EQ_CONFIG:
898 return kvmppc_xive_native_get_queue_config(xive, attr->attr,
899 attr->addr);
900 }
90c73795
CLG		 901 return -ENXIO;
902}
903
904static int kvmppc_xive_native_has_attr(struct kvm_device *dev,
905 struct kvm_device_attr *attr)
906{
907 switch (attr->group) {
908 case KVM_DEV_XIVE_GRP_CTRL:
5ca80647
CLG		 909 switch (attr->attr) {
910 case KVM_DEV_XIVE_RESET:
e6714bd1 911 case KVM_DEV_XIVE_EQ_SYNC:
5ca80647
CLG		 912 return 0;
913 }
90c73795 914 break;
4131f83c 915 case KVM_DEV_XIVE_GRP_SOURCE:
e8676ce5 916 case KVM_DEV_XIVE_GRP_SOURCE_CONFIG:
7b46b616 917 case KVM_DEV_XIVE_GRP_SOURCE_SYNC:
4131f83c
CLG		 918 if (attr->attr >= KVMPPC_XIVE_FIRST_IRQ &&
919 attr->attr < KVMPPC_XIVE_NR_IRQS)
920 return 0;
921 break;
13ce3297
CLG		 922 case KVM_DEV_XIVE_GRP_EQ_CONFIG:
923 return 0;
90c73795
CLG		 924 }
925 return -ENXIO;
926}
927
928static void kvmppc_xive_native_free(struct kvm_device *dev)
929{
930 struct kvmppc_xive *xive = dev->private;
931 struct kvm *kvm = xive->kvm;
4131f83c 932 int i;
90c73795
CLG		 933
934 debugfs_remove(xive->dentry);
935
936 pr_devel("Destroying xive native device\n");
937
938 if (kvm)
939 kvm->arch.xive = NULL;
940
4131f83c
CLG		 941 for (i = 0; i <= xive->max_sbid; i++) {
942 if (xive->src_blocks[i])
943 kvmppc_xive_free_sources(xive->src_blocks[i]);
944 kfree(xive->src_blocks[i]);
945 xive->src_blocks[i] = NULL;
946 }
947
90c73795
CLG		 948 if (xive->vp_base != XIVE_INVALID_VP)
949 xive_native_free_vp_block(xive->vp_base);
950
951 kfree(xive);
952 kfree(dev);
953}
954
955static int kvmppc_xive_native_create(struct kvm_device *dev, u32 type)
956{
957 struct kvmppc_xive *xive;
958 struct kvm *kvm = dev->kvm;
959 int ret = 0;
960
961 pr_devel("Creating xive native device\n");
962
963 if (kvm->arch.xive)
964 return -EEXIST;
965
966 xive = kzalloc(sizeof(*xive), GFP_KERNEL);
967 if (!xive)
968 return -ENOMEM;
969
970 dev->private = xive;
971 xive->dev = dev;
972 xive->kvm = kvm;
973 kvm->arch.xive = xive;
974
975 /*
976 * Allocate a bunch of VPs. KVM_MAX_VCPUS is a large value for
977 * a default. Getting the max number of CPUs the VM was
978 * configured with would improve our usage of the XIVE VP space.
979 */
980 xive->vp_base = xive_native_alloc_vp_block(KVM_MAX_VCPUS);
981 pr_devel("VP_Base=%x\n", xive->vp_base);
982
983 if (xive->vp_base == XIVE_INVALID_VP)
984 ret = -ENXIO;
985
986 xive->single_escalation = xive_native_has_single_escalation();
987
988 if (ret)
989 kfree(xive);
990
991 return ret;
992}
993
e4945b9d
CLG		 994/*
995 * Interrupt Pending Buffer (IPB) offset
996 */
997#define TM_IPB_SHIFT 40
998#define TM_IPB_MASK (((u64) 0xFF) << TM_IPB_SHIFT)
999
1000int kvmppc_xive_native_get_vp(struct kvm_vcpu *vcpu, union kvmppc_one_reg *val)
1001{
1002 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1003 u64 opal_state;
1004 int rc;
1005
1006 if (!kvmppc_xive_enabled(vcpu))
1007 return -EPERM;
1008
1009 if (!xc)
1010 return -ENOENT;
1011
1012 /* Thread context registers. We only care about IPB and CPPR */
1013 val->xive_timaval[0] = vcpu->arch.xive_saved_state.w01;
1014
1015 /* Get the VP state from OPAL */
1016 rc = xive_native_get_vp_state(xc->vp_id, &opal_state);
1017 if (rc)
1018 return rc;
1019
1020 /*
1021 * Capture the backup of IPB register in the NVT structure and
1022 * merge it in our KVM VP state.
1023 */
1024 val->xive_timaval[0] |= cpu_to_be64(opal_state & TM_IPB_MASK);
1025
1026 pr_devel("%s NSR=%02x CPPR=%02x IBP=%02x PIPR=%02x w01=%016llx w2=%08x opal=%016llx\n",
1027 __func__,
1028 vcpu->arch.xive_saved_state.nsr,
1029 vcpu->arch.xive_saved_state.cppr,
1030 vcpu->arch.xive_saved_state.ipb,
1031 vcpu->arch.xive_saved_state.pipr,
1032 vcpu->arch.xive_saved_state.w01,
1033 (u32) vcpu->arch.xive_cam_word, opal_state);
1034
1035 return 0;
1036}
1037
1038int kvmppc_xive_native_set_vp(struct kvm_vcpu *vcpu, union kvmppc_one_reg *val)
1039{
1040 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1041 struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
1042
1043 pr_devel("%s w01=%016llx vp=%016llx\n", __func__,
1044 val->xive_timaval[0], val->xive_timaval[1]);
1045
1046 if (!kvmppc_xive_enabled(vcpu))
1047 return -EPERM;
1048
1049 if (!xc || !xive)
1050 return -ENOENT;
1051
1052 /* We can't update the state of a "pushed" VCPU */
1053 if (WARN_ON(vcpu->arch.xive_pushed))
1054 return -EBUSY;
1055
1056 /*
1057 * Restore the thread context registers. IPB and CPPR should
1058 * be the only ones that matter.
1059 */
1060 vcpu->arch.xive_saved_state.w01 = val->xive_timaval[0];
1061
1062 /*
1063 * There is no need to restore the XIVE internal state (IPB
1064 * stored in the NVT) as the IPB register was merged in KVM VP
1065 * state when captured.
1066 */
1067 return 0;
1068}
1069
90c73795
CLG		 1070static int xive_native_debug_show(struct seq_file *m, void *private)
1071{
1072 struct kvmppc_xive *xive = m->private;
1073 struct kvm *kvm = xive->kvm;
eacc56bb
CLG		 1074 struct kvm_vcpu *vcpu;
1075 unsigned int i;
90c73795
CLG		 1076
1077 if (!kvm)
1078 return 0;
1079
eacc56bb
CLG		 1080 seq_puts(m, "=========\nVCPU state\n=========\n");
1081
1082 kvm_for_each_vcpu(i, vcpu, kvm) {
1083 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1084
1085 if (!xc)
1086 continue;
1087
1088 seq_printf(m, "cpu server %#x NSR=%02x CPPR=%02x IBP=%02x PIPR=%02x w01=%016llx w2=%08x\n",
1089 xc->server_num,
1090 vcpu->arch.xive_saved_state.nsr,
1091 vcpu->arch.xive_saved_state.cppr,
1092 vcpu->arch.xive_saved_state.ipb,
1093 vcpu->arch.xive_saved_state.pipr,
1094 vcpu->arch.xive_saved_state.w01,
1095 (u32) vcpu->arch.xive_cam_word);
1096
1097 kvmppc_xive_debug_show_queues(m, vcpu);
1098 }
1099
90c73795
CLG		 1100 return 0;
1101}
1102
1103static int xive_native_debug_open(struct inode *inode, struct file *file)
1104{
1105 return single_open(file, xive_native_debug_show, inode->i_private);
1106}
1107
1108static const struct file_operations xive_native_debug_fops = {
1109 .open = xive_native_debug_open,
1110 .read = seq_read,
1111 .llseek = seq_lseek,
1112 .release = single_release,
1113};
1114
1115static void xive_native_debugfs_init(struct kvmppc_xive *xive)
1116{
1117 char *name;
1118
1119 name = kasprintf(GFP_KERNEL, "kvm-xive-%p", xive);
1120 if (!name) {
1121 pr_err("%s: no memory for name\n", __func__);
1122 return;
1123 }
1124
1125 xive->dentry = debugfs_create_file(name, 0444, powerpc_debugfs_root,
1126 xive, &xive_native_debug_fops);
1127
1128 pr_debug("%s: created %s\n", __func__, name);
1129 kfree(name);
1130}
1131
1132static void kvmppc_xive_native_init(struct kvm_device *dev)
1133{
1134 struct kvmppc_xive *xive = (struct kvmppc_xive *)dev->private;
1135
1136 /* Register some debug interfaces */
1137 xive_native_debugfs_init(xive);
1138}
1139
1140struct kvm_device_ops kvm_xive_native_ops = {
1141 .name = "kvm-xive-native",
1142 .create = kvmppc_xive_native_create,
1143 .init = kvmppc_xive_native_init,
1144 .destroy = kvmppc_xive_native_free,
1145 .set_attr = kvmppc_xive_native_set_attr,
1146 .get_attr = kvmppc_xive_native_get_attr,
1147 .has_attr = kvmppc_xive_native_has_attr,
39e9af3d 1148 .mmap = kvmppc_xive_native_mmap,
90c73795
CLG		 1149};
1150
1151void kvmppc_xive_native_init_module(void)
1152{
1153 ;
1154}
1155
1156void kvmppc_xive_native_exit_module(void)
1157{
1158 ;
1159}
Linux 6.x block layer and io_uring tree(s)