Commit | Line | Data |
---|---|---|
5af50993 BH |
1 | /* |
2 | * Copyright 2017 Benjamin Herrenschmidt, IBM Corporation | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License, version 2, as | |
6 | * published by the Free Software Foundation. | |
7 | */ | |
8 | ||
9 | /* File to be included by other .c files */ | |
10 | ||
11 | #define XGLUE(a,b) a##b | |
12 | #define GLUE(a,b) XGLUE(a,b) | |
13 | ||
9dc81d6b BH |
14 | /* Dummy interrupt used when taking interrupts out of a queue in H_CPPR */ |
15 | #define XICS_DUMMY 1 | |
16 | ||
5af50993 BH |
17 | static void GLUE(X_PFX,ack_pending)(struct kvmppc_xive_vcpu *xc) |
18 | { | |
19 | u8 cppr; | |
20 | u16 ack; | |
21 | ||
bb9b52bd BH |
22 | /* |
23 | * Ensure any previous store to CPPR is ordered vs. | |
24 | * the subsequent loads from PIPR or ACK. | |
25 | */ | |
26 | eieio(); | |
27 | ||
5af50993 BH |
28 | /* Perform the acknowledge OS to register cycle. */ |
29 | ack = be16_to_cpu(__x_readw(__x_tima + TM_SPC_ACK_OS_REG)); | |
30 | ||
31 | /* Synchronize subsequent queue accesses */ | |
32 | mb(); | |
33 | ||
34 | /* XXX Check grouping level */ | |
35 | ||
36 | /* Anything ? */ | |
37 | if (!((ack >> 8) & TM_QW1_NSR_EO)) | |
38 | return; | |
39 | ||
40 | /* Grab CPPR of the most favored pending interrupt */ | |
41 | cppr = ack & 0xff; | |
42 | if (cppr < 8) | |
43 | xc->pending |= 1 << cppr; | |
44 | ||
45 | #ifdef XIVE_RUNTIME_CHECKS | |
46 | /* Check consistency */ | |
47 | if (cppr >= xc->hw_cppr) | |
48 | pr_warn("KVM-XIVE: CPU %d odd ack CPPR, got %d at %d\n", | |
49 | smp_processor_id(), cppr, xc->hw_cppr); | |
50 | #endif | |
51 | ||
52 | /* | |
53 | * Update our image of the HW CPPR. We don't yet modify | |
54 | * xc->cppr, this will be done as we scan for interrupts | |
55 | * in the queues. | |
56 | */ | |
57 | xc->hw_cppr = cppr; | |
58 | } | |
59 | ||
60 | static u8 GLUE(X_PFX,esb_load)(struct xive_irq_data *xd, u32 offset) | |
61 | { | |
62 | u64 val; | |
63 | ||
64 | if (xd->flags & XIVE_IRQ_FLAG_SHIFT_BUG) | |
65 | offset |= offset << 4; | |
66 | ||
67 | val =__x_readq(__x_eoi_page(xd) + offset); | |
68 | #ifdef __LITTLE_ENDIAN__ | |
69 | val >>= 64-8; | |
70 | #endif | |
71 | return (u8)val; | |
72 | } | |
73 | ||
74 | ||
75 | static void GLUE(X_PFX,source_eoi)(u32 hw_irq, struct xive_irq_data *xd) | |
76 | { | |
77 | /* If the XIVE supports the new "store EOI facility, use it */ | |
78 | if (xd->flags & XIVE_IRQ_FLAG_STORE_EOI) | |
25642705 | 79 | __x_writeq(0, __x_eoi_page(xd) + XIVE_ESB_STORE_EOI); |
2bf1071a | 80 | else if (hw_irq && xd->flags & XIVE_IRQ_FLAG_EOI_FW) |
5af50993 | 81 | opal_int_eoi(hw_irq); |
2bf1071a NP |
82 | else if (xd->flags & XIVE_IRQ_FLAG_LSI) { |
83 | /* | |
84 | * For LSIs the HW EOI cycle is used rather than PQ bits, | |
85 | * as they are automatically re-triggred in HW when still | |
86 | * pending. | |
87 | */ | |
88 | __x_readq(__x_eoi_page(xd) + XIVE_ESB_LOAD_EOI); | |
5af50993 BH |
89 | } else { |
90 | uint64_t eoi_val; | |
91 | ||
92 | /* | |
93 | * Otherwise for EOI, we use the special MMIO that does | |
94 | * a clear of both P and Q and returns the old Q, | |
95 | * except for LSIs where we use the "EOI cycle" special | |
96 | * load. | |
97 | * | |
98 | * This allows us to then do a re-trigger if Q was set | |
99 | * rather than synthetizing an interrupt in software | |
5af50993 | 100 | */ |
2bf1071a NP |
101 | eoi_val = GLUE(X_PFX,esb_load)(xd, XIVE_ESB_SET_PQ_00); |
102 | ||
103 | /* Re-trigger if needed */ | |
104 | if ((eoi_val & 1) && __x_trig_page(xd)) | |
105 | __x_writeq(0, __x_trig_page(xd)); | |
5af50993 BH |
106 | } |
107 | } | |
108 | ||
109 | enum { | |
110 | scan_fetch, | |
111 | scan_poll, | |
112 | scan_eoi, | |
113 | }; | |
114 | ||
115 | static u32 GLUE(X_PFX,scan_interrupts)(struct kvmppc_xive_vcpu *xc, | |
116 | u8 pending, int scan_type) | |
117 | { | |
118 | u32 hirq = 0; | |
119 | u8 prio = 0xff; | |
120 | ||
121 | /* Find highest pending priority */ | |
122 | while ((xc->mfrr != 0xff || pending != 0) && hirq == 0) { | |
123 | struct xive_q *q; | |
124 | u32 idx, toggle; | |
125 | __be32 *qpage; | |
126 | ||
127 | /* | |
128 | * If pending is 0 this will return 0xff which is what | |
129 | * we want | |
130 | */ | |
131 | prio = ffs(pending) - 1; | |
132 | ||
133 | /* | |
134 | * If the most favoured prio we found pending is less | |
135 | * favored (or equal) than a pending IPI, we return | |
136 | * the IPI instead. | |
137 | * | |
138 | * Note: If pending was 0 and mfrr is 0xff, we will | |
139 | * not spurriously take an IPI because mfrr cannot | |
140 | * then be smaller than cppr. | |
141 | */ | |
142 | if (prio >= xc->mfrr && xc->mfrr < xc->cppr) { | |
143 | prio = xc->mfrr; | |
144 | hirq = XICS_IPI; | |
145 | break; | |
146 | } | |
147 | ||
148 | /* Don't scan past the guest cppr */ | |
149 | if (prio >= xc->cppr || prio > 7) | |
150 | break; | |
151 | ||
152 | /* Grab queue and pointers */ | |
153 | q = &xc->queues[prio]; | |
154 | idx = q->idx; | |
155 | toggle = q->toggle; | |
156 | ||
157 | /* | |
158 | * Snapshot the queue page. The test further down for EOI | |
159 | * must use the same "copy" that was used by __xive_read_eq | |
160 | * since qpage can be set concurrently and we don't want | |
161 | * to miss an EOI. | |
162 | */ | |
163 | qpage = READ_ONCE(q->qpage); | |
164 | ||
165 | skip_ipi: | |
166 | /* | |
167 | * Try to fetch from the queue. Will return 0 for a | |
168 | * non-queueing priority (ie, qpage = 0). | |
169 | */ | |
170 | hirq = __xive_read_eq(qpage, q->msk, &idx, &toggle); | |
171 | ||
172 | /* | |
173 | * If this was a signal for an MFFR change done by | |
174 | * H_IPI we skip it. Additionally, if we were fetching | |
175 | * we EOI it now, thus re-enabling reception of a new | |
176 | * such signal. | |
177 | * | |
178 | * We also need to do that if prio is 0 and we had no | |
179 | * page for the queue. In this case, we have non-queued | |
180 | * IPI that needs to be EOId. | |
181 | * | |
182 | * This is safe because if we have another pending MFRR | |
183 | * change that wasn't observed above, the Q bit will have | |
184 | * been set and another occurrence of the IPI will trigger. | |
185 | */ | |
186 | if (hirq == XICS_IPI || (prio == 0 && !qpage)) { | |
187 | if (scan_type == scan_fetch) | |
188 | GLUE(X_PFX,source_eoi)(xc->vp_ipi, | |
189 | &xc->vp_ipi_data); | |
190 | /* Loop back on same queue with updated idx/toggle */ | |
191 | #ifdef XIVE_RUNTIME_CHECKS | |
192 | WARN_ON(hirq && hirq != XICS_IPI); | |
193 | #endif | |
194 | if (hirq) | |
195 | goto skip_ipi; | |
196 | } | |
197 | ||
9dc81d6b BH |
198 | /* If it's the dummy interrupt, continue searching */ |
199 | if (hirq == XICS_DUMMY) | |
200 | goto skip_ipi; | |
201 | ||
5af50993 BH |
202 | /* If fetching, update queue pointers */ |
203 | if (scan_type == scan_fetch) { | |
204 | q->idx = idx; | |
205 | q->toggle = toggle; | |
206 | } | |
207 | ||
208 | /* Something found, stop searching */ | |
209 | if (hirq) | |
210 | break; | |
211 | ||
212 | /* Clear the pending bit on the now empty queue */ | |
213 | pending &= ~(1 << prio); | |
214 | ||
215 | /* | |
216 | * Check if the queue count needs adjusting due to | |
217 | * interrupts being moved away. | |
218 | */ | |
219 | if (atomic_read(&q->pending_count)) { | |
220 | int p = atomic_xchg(&q->pending_count, 0); | |
221 | if (p) { | |
222 | #ifdef XIVE_RUNTIME_CHECKS | |
223 | WARN_ON(p > atomic_read(&q->count)); | |
224 | #endif | |
225 | atomic_sub(p, &q->count); | |
226 | } | |
227 | } | |
228 | } | |
229 | ||
230 | /* If we are just taking a "peek", do nothing else */ | |
231 | if (scan_type == scan_poll) | |
232 | return hirq; | |
233 | ||
234 | /* Update the pending bits */ | |
235 | xc->pending = pending; | |
236 | ||
237 | /* | |
238 | * If this is an EOI that's it, no CPPR adjustment done here, | |
239 | * all we needed was cleanup the stale pending bits and check | |
240 | * if there's anything left. | |
241 | */ | |
242 | if (scan_type == scan_eoi) | |
243 | return hirq; | |
244 | ||
245 | /* | |
246 | * If we found an interrupt, adjust what the guest CPPR should | |
247 | * be as if we had just fetched that interrupt from HW. | |
bb9b52bd BH |
248 | * |
249 | * Note: This can only make xc->cppr smaller as the previous | |
250 | * loop will only exit with hirq != 0 if prio is lower than | |
251 | * the current xc->cppr. Thus we don't need to re-check xc->mfrr | |
252 | * for pending IPIs. | |
5af50993 BH |
253 | */ |
254 | if (hirq) | |
255 | xc->cppr = prio; | |
256 | /* | |
257 | * If it was an IPI the HW CPPR might have been lowered too much | |
258 | * as the HW interrupt we use for IPIs is routed to priority 0. | |
259 | * | |
260 | * We re-sync it here. | |
261 | */ | |
262 | if (xc->cppr != xc->hw_cppr) { | |
263 | xc->hw_cppr = xc->cppr; | |
264 | __x_writeb(xc->cppr, __x_tima + TM_QW1_OS + TM_CPPR); | |
265 | } | |
266 | ||
267 | return hirq; | |
268 | } | |
269 | ||
270 | X_STATIC unsigned long GLUE(X_PFX,h_xirr)(struct kvm_vcpu *vcpu) | |
271 | { | |
272 | struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; | |
273 | u8 old_cppr; | |
274 | u32 hirq; | |
275 | ||
276 | pr_devel("H_XIRR\n"); | |
277 | ||
278 | xc->GLUE(X_STAT_PFX,h_xirr)++; | |
279 | ||
280 | /* First collect pending bits from HW */ | |
281 | GLUE(X_PFX,ack_pending)(xc); | |
282 | ||
283 | /* | |
284 | * Cleanup the old-style bits if needed (they may have been | |
285 | * set by pull or an escalation interrupts). | |
286 | */ | |
287 | if (test_bit(BOOK3S_IRQPRIO_EXTERNAL, &vcpu->arch.pending_exceptions)) | |
288 | clear_bit(BOOK3S_IRQPRIO_EXTERNAL_LEVEL, | |
289 | &vcpu->arch.pending_exceptions); | |
290 | ||
291 | pr_devel(" new pending=0x%02x hw_cppr=%d cppr=%d\n", | |
292 | xc->pending, xc->hw_cppr, xc->cppr); | |
293 | ||
294 | /* Grab previous CPPR and reverse map it */ | |
295 | old_cppr = xive_prio_to_guest(xc->cppr); | |
296 | ||
297 | /* Scan for actual interrupts */ | |
298 | hirq = GLUE(X_PFX,scan_interrupts)(xc, xc->pending, scan_fetch); | |
299 | ||
300 | pr_devel(" got hirq=0x%x hw_cppr=%d cppr=%d\n", | |
301 | hirq, xc->hw_cppr, xc->cppr); | |
302 | ||
303 | #ifdef XIVE_RUNTIME_CHECKS | |
304 | /* That should never hit */ | |
305 | if (hirq & 0xff000000) | |
306 | pr_warn("XIVE: Weird guest interrupt number 0x%08x\n", hirq); | |
307 | #endif | |
308 | ||
309 | /* | |
310 | * XXX We could check if the interrupt is masked here and | |
311 | * filter it. If we chose to do so, we would need to do: | |
312 | * | |
313 | * if (masked) { | |
314 | * lock(); | |
315 | * if (masked) { | |
316 | * old_Q = true; | |
317 | * hirq = 0; | |
318 | * } | |
319 | * unlock(); | |
320 | * } | |
321 | */ | |
322 | ||
323 | /* Return interrupt and old CPPR in GPR4 */ | |
1143a706 | 324 | vcpu->arch.regs.gpr[4] = hirq | (old_cppr << 24); |
5af50993 BH |
325 | |
326 | return H_SUCCESS; | |
327 | } | |
328 | ||
329 | X_STATIC unsigned long GLUE(X_PFX,h_ipoll)(struct kvm_vcpu *vcpu, unsigned long server) | |
330 | { | |
331 | struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; | |
332 | u8 pending = xc->pending; | |
333 | u32 hirq; | |
5af50993 BH |
334 | |
335 | pr_devel("H_IPOLL(server=%ld)\n", server); | |
336 | ||
337 | xc->GLUE(X_STAT_PFX,h_ipoll)++; | |
338 | ||
339 | /* Grab the target VCPU if not the current one */ | |
340 | if (xc->server_num != server) { | |
341 | vcpu = kvmppc_xive_find_server(vcpu->kvm, server); | |
342 | if (!vcpu) | |
343 | return H_PARAMETER; | |
344 | xc = vcpu->arch.xive_vcpu; | |
345 | ||
346 | /* Scan all priorities */ | |
347 | pending = 0xff; | |
348 | } else { | |
349 | /* Grab pending interrupt if any */ | |
d222af07 BH |
350 | __be64 qw1 = __x_readq(__x_tima + TM_QW1_OS); |
351 | u8 pipr = be64_to_cpu(qw1) & 0xff; | |
5af50993 BH |
352 | if (pipr < 8) |
353 | pending |= 1 << pipr; | |
354 | } | |
355 | ||
356 | hirq = GLUE(X_PFX,scan_interrupts)(xc, pending, scan_poll); | |
357 | ||
358 | /* Return interrupt and old CPPR in GPR4 */ | |
1143a706 | 359 | vcpu->arch.regs.gpr[4] = hirq | (xc->cppr << 24); |
5af50993 BH |
360 | |
361 | return H_SUCCESS; | |
362 | } | |
363 | ||
364 | static void GLUE(X_PFX,push_pending_to_hw)(struct kvmppc_xive_vcpu *xc) | |
365 | { | |
366 | u8 pending, prio; | |
367 | ||
368 | pending = xc->pending; | |
369 | if (xc->mfrr != 0xff) { | |
370 | if (xc->mfrr < 8) | |
371 | pending |= 1 << xc->mfrr; | |
372 | else | |
373 | pending |= 0x80; | |
374 | } | |
375 | if (!pending) | |
376 | return; | |
377 | prio = ffs(pending) - 1; | |
378 | ||
379 | __x_writeb(prio, __x_tima + TM_SPC_SET_OS_PENDING); | |
380 | } | |
381 | ||
9dc81d6b BH |
382 | static void GLUE(X_PFX,scan_for_rerouted_irqs)(struct kvmppc_xive *xive, |
383 | struct kvmppc_xive_vcpu *xc) | |
384 | { | |
385 | unsigned int prio; | |
386 | ||
387 | /* For each priority that is now masked */ | |
388 | for (prio = xc->cppr; prio < KVMPPC_XIVE_Q_COUNT; prio++) { | |
389 | struct xive_q *q = &xc->queues[prio]; | |
390 | struct kvmppc_xive_irq_state *state; | |
391 | struct kvmppc_xive_src_block *sb; | |
392 | u32 idx, toggle, entry, irq, hw_num; | |
393 | struct xive_irq_data *xd; | |
394 | __be32 *qpage; | |
395 | u16 src; | |
396 | ||
397 | idx = q->idx; | |
398 | toggle = q->toggle; | |
399 | qpage = READ_ONCE(q->qpage); | |
400 | if (!qpage) | |
401 | continue; | |
402 | ||
403 | /* For each interrupt in the queue */ | |
404 | for (;;) { | |
405 | entry = be32_to_cpup(qpage + idx); | |
406 | ||
407 | /* No more ? */ | |
408 | if ((entry >> 31) == toggle) | |
409 | break; | |
410 | irq = entry & 0x7fffffff; | |
411 | ||
412 | /* Skip dummies and IPIs */ | |
413 | if (irq == XICS_DUMMY || irq == XICS_IPI) | |
414 | goto next; | |
415 | sb = kvmppc_xive_find_source(xive, irq, &src); | |
416 | if (!sb) | |
417 | goto next; | |
418 | state = &sb->irq_state[src]; | |
419 | ||
420 | /* Has it been rerouted ? */ | |
421 | if (xc->server_num == state->act_server) | |
422 | goto next; | |
423 | ||
424 | /* | |
425 | * Allright, it *has* been re-routed, kill it from | |
426 | * the queue. | |
427 | */ | |
428 | qpage[idx] = cpu_to_be32((entry & 0x80000000) | XICS_DUMMY); | |
429 | ||
430 | /* Find the HW interrupt */ | |
431 | kvmppc_xive_select_irq(state, &hw_num, &xd); | |
432 | ||
433 | /* If it's not an LSI, set PQ to 11 the EOI will force a resend */ | |
434 | if (!(xd->flags & XIVE_IRQ_FLAG_LSI)) | |
435 | GLUE(X_PFX,esb_load)(xd, XIVE_ESB_SET_PQ_11); | |
436 | ||
437 | /* EOI the source */ | |
438 | GLUE(X_PFX,source_eoi)(hw_num, xd); | |
439 | ||
440 | next: | |
441 | idx = (idx + 1) & q->msk; | |
442 | if (idx == 0) | |
443 | toggle ^= 1; | |
444 | } | |
445 | } | |
446 | } | |
447 | ||
5af50993 BH |
448 | X_STATIC int GLUE(X_PFX,h_cppr)(struct kvm_vcpu *vcpu, unsigned long cppr) |
449 | { | |
450 | struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; | |
9dc81d6b | 451 | struct kvmppc_xive *xive = vcpu->kvm->arch.xive; |
5af50993 BH |
452 | u8 old_cppr; |
453 | ||
454 | pr_devel("H_CPPR(cppr=%ld)\n", cppr); | |
455 | ||
456 | xc->GLUE(X_STAT_PFX,h_cppr)++; | |
457 | ||
458 | /* Map CPPR */ | |
459 | cppr = xive_prio_from_guest(cppr); | |
460 | ||
461 | /* Remember old and update SW state */ | |
462 | old_cppr = xc->cppr; | |
463 | xc->cppr = cppr; | |
464 | ||
bb9b52bd BH |
465 | /* |
466 | * Order the above update of xc->cppr with the subsequent | |
467 | * read of xc->mfrr inside push_pending_to_hw() | |
468 | */ | |
469 | smp_mb(); | |
470 | ||
9dc81d6b BH |
471 | if (cppr > old_cppr) { |
472 | /* | |
473 | * We are masking less, we need to look for pending things | |
474 | * to deliver and set VP pending bits accordingly to trigger | |
475 | * a new interrupt otherwise we might miss MFRR changes for | |
476 | * which we have optimized out sending an IPI signal. | |
477 | */ | |
5af50993 | 478 | GLUE(X_PFX,push_pending_to_hw)(xc); |
9dc81d6b BH |
479 | } else { |
480 | /* | |
481 | * We are masking more, we need to check the queue for any | |
482 | * interrupt that has been routed to another CPU, take | |
483 | * it out (replace it with the dummy) and retrigger it. | |
484 | * | |
485 | * This is necessary since those interrupts may otherwise | |
486 | * never be processed, at least not until this CPU restores | |
487 | * its CPPR. | |
488 | * | |
489 | * This is in theory racy vs. HW adding new interrupts to | |
490 | * the queue. In practice this works because the interesting | |
491 | * cases are when the guest has done a set_xive() to move the | |
492 | * interrupt away, which flushes the xive, followed by the | |
493 | * target CPU doing a H_CPPR. So any new interrupt coming into | |
494 | * the queue must still be routed to us and isn't a source | |
495 | * of concern. | |
496 | */ | |
497 | GLUE(X_PFX,scan_for_rerouted_irqs)(xive, xc); | |
498 | } | |
5af50993 BH |
499 | |
500 | /* Apply new CPPR */ | |
501 | xc->hw_cppr = cppr; | |
502 | __x_writeb(cppr, __x_tima + TM_QW1_OS + TM_CPPR); | |
503 | ||
504 | return H_SUCCESS; | |
505 | } | |
506 | ||
507 | X_STATIC int GLUE(X_PFX,h_eoi)(struct kvm_vcpu *vcpu, unsigned long xirr) | |
508 | { | |
509 | struct kvmppc_xive *xive = vcpu->kvm->arch.xive; | |
510 | struct kvmppc_xive_src_block *sb; | |
511 | struct kvmppc_xive_irq_state *state; | |
512 | struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; | |
513 | struct xive_irq_data *xd; | |
514 | u8 new_cppr = xirr >> 24; | |
515 | u32 irq = xirr & 0x00ffffff, hw_num; | |
516 | u16 src; | |
517 | int rc = 0; | |
518 | ||
519 | pr_devel("H_EOI(xirr=%08lx)\n", xirr); | |
520 | ||
521 | xc->GLUE(X_STAT_PFX,h_eoi)++; | |
522 | ||
523 | xc->cppr = xive_prio_from_guest(new_cppr); | |
524 | ||
525 | /* | |
526 | * IPIs are synthetized from MFRR and thus don't need | |
527 | * any special EOI handling. The underlying interrupt | |
528 | * used to signal MFRR changes is EOId when fetched from | |
529 | * the queue. | |
530 | */ | |
bb9b52bd BH |
531 | if (irq == XICS_IPI || irq == 0) { |
532 | /* | |
533 | * This barrier orders the setting of xc->cppr vs. | |
534 | * subsquent test of xc->mfrr done inside | |
535 | * scan_interrupts and push_pending_to_hw | |
536 | */ | |
537 | smp_mb(); | |
5af50993 | 538 | goto bail; |
bb9b52bd | 539 | } |
5af50993 BH |
540 | |
541 | /* Find interrupt source */ | |
542 | sb = kvmppc_xive_find_source(xive, irq, &src); | |
543 | if (!sb) { | |
544 | pr_devel(" source not found !\n"); | |
545 | rc = H_PARAMETER; | |
bb9b52bd BH |
546 | /* Same as above */ |
547 | smp_mb(); | |
5af50993 BH |
548 | goto bail; |
549 | } | |
550 | state = &sb->irq_state[src]; | |
551 | kvmppc_xive_select_irq(state, &hw_num, &xd); | |
552 | ||
553 | state->in_eoi = true; | |
bb9b52bd BH |
554 | |
555 | /* | |
556 | * This barrier orders both setting of in_eoi above vs, | |
557 | * subsequent test of guest_priority, and the setting | |
558 | * of xc->cppr vs. subsquent test of xc->mfrr done inside | |
559 | * scan_interrupts and push_pending_to_hw | |
560 | */ | |
561 | smp_mb(); | |
5af50993 BH |
562 | |
563 | again: | |
564 | if (state->guest_priority == MASKED) { | |
565 | arch_spin_lock(&sb->lock); | |
566 | if (state->guest_priority != MASKED) { | |
567 | arch_spin_unlock(&sb->lock); | |
568 | goto again; | |
569 | } | |
570 | pr_devel(" EOI on saved P...\n"); | |
571 | ||
572 | /* Clear old_p, that will cause unmask to perform an EOI */ | |
573 | state->old_p = false; | |
574 | ||
575 | arch_spin_unlock(&sb->lock); | |
576 | } else { | |
577 | pr_devel(" EOI on source...\n"); | |
578 | ||
579 | /* Perform EOI on the source */ | |
580 | GLUE(X_PFX,source_eoi)(hw_num, xd); | |
581 | ||
582 | /* If it's an emulated LSI, check level and resend */ | |
583 | if (state->lsi && state->asserted) | |
584 | __x_writeq(0, __x_trig_page(xd)); | |
585 | ||
586 | } | |
587 | ||
bb9b52bd BH |
588 | /* |
589 | * This barrier orders the above guest_priority check | |
590 | * and spin_lock/unlock with clearing in_eoi below. | |
591 | * | |
592 | * It also has to be a full mb() as it must ensure | |
593 | * the MMIOs done in source_eoi() are completed before | |
594 | * state->in_eoi is visible. | |
595 | */ | |
5af50993 BH |
596 | mb(); |
597 | state->in_eoi = false; | |
598 | bail: | |
599 | ||
600 | /* Re-evaluate pending IRQs and update HW */ | |
601 | GLUE(X_PFX,scan_interrupts)(xc, xc->pending, scan_eoi); | |
602 | GLUE(X_PFX,push_pending_to_hw)(xc); | |
603 | pr_devel(" after scan pending=%02x\n", xc->pending); | |
604 | ||
605 | /* Apply new CPPR */ | |
606 | xc->hw_cppr = xc->cppr; | |
607 | __x_writeb(xc->cppr, __x_tima + TM_QW1_OS + TM_CPPR); | |
608 | ||
609 | return rc; | |
610 | } | |
611 | ||
612 | X_STATIC int GLUE(X_PFX,h_ipi)(struct kvm_vcpu *vcpu, unsigned long server, | |
613 | unsigned long mfrr) | |
614 | { | |
615 | struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; | |
616 | ||
617 | pr_devel("H_IPI(server=%08lx,mfrr=%ld)\n", server, mfrr); | |
618 | ||
619 | xc->GLUE(X_STAT_PFX,h_ipi)++; | |
620 | ||
621 | /* Find target */ | |
622 | vcpu = kvmppc_xive_find_server(vcpu->kvm, server); | |
623 | if (!vcpu) | |
624 | return H_PARAMETER; | |
625 | xc = vcpu->arch.xive_vcpu; | |
626 | ||
627 | /* Locklessly write over MFRR */ | |
628 | xc->mfrr = mfrr; | |
629 | ||
bb9b52bd BH |
630 | /* |
631 | * The load of xc->cppr below and the subsequent MMIO store | |
632 | * to the IPI must happen after the above mfrr update is | |
633 | * globally visible so that: | |
634 | * | |
635 | * - Synchronize with another CPU doing an H_EOI or a H_CPPR | |
636 | * updating xc->cppr then reading xc->mfrr. | |
637 | * | |
638 | * - The target of the IPI sees the xc->mfrr update | |
639 | */ | |
640 | mb(); | |
641 | ||
5af50993 BH |
642 | /* Shoot the IPI if most favored than target cppr */ |
643 | if (mfrr < xc->cppr) | |
644 | __x_writeq(0, __x_trig_page(&xc->vp_ipi_data)); | |
645 | ||
646 | return H_SUCCESS; | |
647 | } |