1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/kvm_host.h>
4 #include <asm/irq_remapping.h>
9 #include "posted_intr.h"
14 * We maintain a per-CPU linked-list of vCPU, so in wakeup_handler() we
15 * can find which vCPU should be waken up.
17 static DEFINE_PER_CPU(struct list_head, blocked_vcpu_on_cpu);
18 static DEFINE_PER_CPU(spinlock_t, blocked_vcpu_on_cpu_lock);
20 static inline struct pi_desc *vcpu_to_pi_desc(struct kvm_vcpu *vcpu)
22 return &(to_vmx(vcpu)->pi_desc);
25 void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
27 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
28 struct pi_desc old, new;
32 * In case of hot-plug or hot-unplug, we may have to undo
33 * vmx_vcpu_pi_put even if there is no assigned device. And we
34 * always keep PI.NDST up to date for simplicity: it makes the
35 * code easier, and CPU migration is not a fast path.
37 if (!pi_test_sn(pi_desc) && vcpu->cpu == cpu)
41 * If the 'nv' field is POSTED_INTR_WAKEUP_VECTOR, do not change
42 * PI.NDST: pi_post_block is the one expected to change PID.NDST and the
43 * wakeup handler expects the vCPU to be on the blocked_vcpu_list that
44 * matches PI.NDST. Otherwise, a vcpu may not be able to be woken up
47 if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR || vcpu->cpu == cpu) {
54 old.control = new.control = pi_desc->control;
56 dest = cpu_physical_id(cpu);
61 new.ndst = (dest << 8) & 0xFF00;
64 } while (cmpxchg64(&pi_desc->control, old.control,
65 new.control) != old.control);
70 * Clear SN before reading the bitmap. The VT-d firmware
71 * writes the bitmap and reads SN atomically (5.2.3 in the
72 * spec), so it doesn't really have a memory barrier that
73 * pairs with this, but we cannot do that and we need one.
75 smp_mb__after_atomic();
77 if (!pi_is_pir_empty(pi_desc))
81 static bool vmx_can_use_vtd_pi(struct kvm *kvm)
83 return irqchip_in_kernel(kvm) && enable_apicv &&
84 kvm_arch_has_assigned_device(kvm) &&
85 irq_remapping_cap(IRQ_POSTING_CAP);
88 void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu)
90 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
92 if (!vmx_can_use_vtd_pi(vcpu->kvm))
95 /* Set SN when the vCPU is preempted */
100 static void __pi_post_block(struct kvm_vcpu *vcpu)
102 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
103 struct pi_desc old, new;
107 old.control = new.control = pi_desc->control;
108 WARN(old.nv != POSTED_INTR_WAKEUP_VECTOR,
109 "Wakeup handler not enabled while the VCPU is blocked\n");
111 dest = cpu_physical_id(vcpu->cpu);
116 new.ndst = (dest << 8) & 0xFF00;
118 /* set 'NV' to 'notification vector' */
119 new.nv = POSTED_INTR_VECTOR;
120 } while (cmpxchg64(&pi_desc->control, old.control,
121 new.control) != old.control);
123 if (!WARN_ON_ONCE(vcpu->pre_pcpu == -1)) {
124 spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
125 list_del(&vcpu->blocked_vcpu_list);
126 spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
132 * This routine does the following things for vCPU which is going
133 * to be blocked if VT-d PI is enabled.
134 * - Store the vCPU to the wakeup list, so when interrupts happen
135 * we can find the right vCPU to wake up.
136 * - Change the Posted-interrupt descriptor as below:
137 * 'NDST' <-- vcpu->pre_pcpu
138 * 'NV' <-- POSTED_INTR_WAKEUP_VECTOR
139 * - If 'ON' is set during this process, which means at least one
140 * interrupt is posted for this vCPU, we cannot block it, in
141 * this case, return 1, otherwise, return 0.
144 int pi_pre_block(struct kvm_vcpu *vcpu)
147 struct pi_desc old, new;
148 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
150 if (!vmx_can_use_vtd_pi(vcpu->kvm) ||
151 vmx_interrupt_blocked(vcpu))
154 WARN_ON(irqs_disabled());
156 if (!WARN_ON_ONCE(vcpu->pre_pcpu != -1)) {
157 vcpu->pre_pcpu = vcpu->cpu;
158 spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
159 list_add_tail(&vcpu->blocked_vcpu_list,
160 &per_cpu(blocked_vcpu_on_cpu,
162 spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
166 old.control = new.control = pi_desc->control;
168 WARN((pi_desc->sn == 1),
169 "Warning: SN field of posted-interrupts "
170 "is set before blocking\n");
173 * Since vCPU can be preempted during this process,
174 * vcpu->cpu could be different with pre_pcpu, we
175 * need to set pre_pcpu as the destination of wakeup
176 * notification event, then we can find the right vCPU
177 * to wakeup in wakeup handler if interrupts happen
178 * when the vCPU is in blocked state.
180 dest = cpu_physical_id(vcpu->pre_pcpu);
185 new.ndst = (dest << 8) & 0xFF00;
187 /* set 'NV' to 'wakeup vector' */
188 new.nv = POSTED_INTR_WAKEUP_VECTOR;
189 } while (cmpxchg64(&pi_desc->control, old.control,
190 new.control) != old.control);
192 /* We should not block the vCPU if an interrupt is posted for it. */
193 if (pi_test_on(pi_desc) == 1)
194 __pi_post_block(vcpu);
197 return (vcpu->pre_pcpu == -1);
200 void pi_post_block(struct kvm_vcpu *vcpu)
202 if (vcpu->pre_pcpu == -1)
205 WARN_ON(irqs_disabled());
207 __pi_post_block(vcpu);
212 * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
214 void pi_wakeup_handler(void)
216 struct kvm_vcpu *vcpu;
217 int cpu = smp_processor_id();
219 spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
220 list_for_each_entry(vcpu, &per_cpu(blocked_vcpu_on_cpu, cpu),
222 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
224 if (pi_test_on(pi_desc) == 1)
227 spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
230 void __init pi_init_cpu(int cpu)
232 INIT_LIST_HEAD(&per_cpu(blocked_vcpu_on_cpu, cpu));
233 spin_lock_init(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
236 bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu)
238 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
240 return pi_test_on(pi_desc) ||
241 (pi_test_sn(pi_desc) && !pi_is_pir_empty(pi_desc));
246 * Bail out of the block loop if the VM has an assigned
247 * device, but the blocking vCPU didn't reconfigure the
248 * PI.NV to the wakeup vector, i.e. the assigned device
249 * came along after the initial check in pi_pre_block().
251 void vmx_pi_start_assignment(struct kvm *kvm)
253 if (!irq_remapping_cap(IRQ_POSTING_CAP))
256 kvm_make_all_cpus_request(kvm, KVM_REQ_UNBLOCK);
260 * pi_update_irte - set IRTE for Posted-Interrupts
263 * @host_irq: host irq of the interrupt
264 * @guest_irq: gsi of the interrupt
265 * @set: set or unset PI
266 * returns 0 on success, < 0 on failure
268 int pi_update_irte(struct kvm *kvm, unsigned int host_irq, uint32_t guest_irq,
271 struct kvm_kernel_irq_routing_entry *e;
272 struct kvm_irq_routing_table *irq_rt;
273 struct kvm_lapic_irq irq;
274 struct kvm_vcpu *vcpu;
275 struct vcpu_data vcpu_info;
278 if (!vmx_can_use_vtd_pi(kvm))
281 idx = srcu_read_lock(&kvm->irq_srcu);
282 irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
283 if (guest_irq >= irq_rt->nr_rt_entries ||
284 hlist_empty(&irq_rt->map[guest_irq])) {
285 pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n",
286 guest_irq, irq_rt->nr_rt_entries);
290 hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
291 if (e->type != KVM_IRQ_ROUTING_MSI)
294 * VT-d PI cannot support posting multicast/broadcast
295 * interrupts to a vCPU, we still use interrupt remapping
296 * for these kind of interrupts.
298 * For lowest-priority interrupts, we only support
299 * those with single CPU as the destination, e.g. user
300 * configures the interrupts via /proc/irq or uses
301 * irqbalance to make the interrupts single-CPU.
303 * We will support full lowest-priority interrupt later.
305 * In addition, we can only inject generic interrupts using
306 * the PI mechanism, refuse to route others through it.
309 kvm_set_msi_irq(kvm, e, &irq);
310 if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) ||
311 !kvm_irq_is_postable(&irq)) {
313 * Make sure the IRTE is in remapped mode if
314 * we don't handle it in posted mode.
316 ret = irq_set_vcpu_affinity(host_irq, NULL);
319 "failed to back to remapped mode, irq: %u\n",
327 vcpu_info.pi_desc_addr = __pa(&to_vmx(vcpu)->pi_desc);
328 vcpu_info.vector = irq.vector;
330 trace_kvm_pi_irte_update(host_irq, vcpu->vcpu_id, e->gsi,
331 vcpu_info.vector, vcpu_info.pi_desc_addr, set);
334 ret = irq_set_vcpu_affinity(host_irq, &vcpu_info);
336 ret = irq_set_vcpu_affinity(host_irq, NULL);
339 printk(KERN_INFO "%s: failed to update PI IRTE\n",
347 srcu_read_unlock(&kvm->irq_srcu, idx);