Commit | Line | Data |
---|---|---|
b4072df4 PM |
1 | /* |
2 | * This program is free software; you can redistribute it and/or modify | |
3 | * it under the terms of the GNU General Public License, version 2, as | |
4 | * published by the Free Software Foundation. | |
5 | * | |
6 | * Copyright 2012 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> | |
7 | */ | |
8 | ||
9 | #include <linux/types.h> | |
10 | #include <linux/string.h> | |
11 | #include <linux/kvm.h> | |
12 | #include <linux/kvm_host.h> | |
13 | #include <linux/kernel.h> | |
14 | #include <asm/opal.h> | |
36df96f8 | 15 | #include <asm/mce.h> |
fd7bacbc MS |
16 | #include <asm/machdep.h> |
17 | #include <asm/cputhreads.h> | |
18 | #include <asm/hmi.h> | |
e34af784 | 19 | #include <asm/kvm_ppc.h> |
b4072df4 PM |
20 | |
21 | /* SRR1 bits for machine check on POWER7 */ | |
22 | #define SRR1_MC_LDSTERR (1ul << (63-42)) | |
23 | #define SRR1_MC_IFETCH_SH (63-45) | |
24 | #define SRR1_MC_IFETCH_MASK 0x7 | |
25 | #define SRR1_MC_IFETCH_SLBPAR 2 /* SLB parity error */ | |
26 | #define SRR1_MC_IFETCH_SLBMULTI 3 /* SLB multi-hit */ | |
27 | #define SRR1_MC_IFETCH_SLBPARMULTI 4 /* SLB parity + multi-hit */ | |
28 | #define SRR1_MC_IFETCH_TLBMULTI 5 /* I-TLB multi-hit */ | |
29 | ||
30 | /* DSISR bits for machine check on POWER7 */ | |
31 | #define DSISR_MC_DERAT_MULTI 0x800 /* D-ERAT multi-hit */ | |
32 | #define DSISR_MC_TLB_MULTI 0x400 /* D-TLB multi-hit */ | |
33 | #define DSISR_MC_SLB_PARITY 0x100 /* SLB parity error */ | |
34 | #define DSISR_MC_SLB_MULTI 0x080 /* SLB multi-hit */ | |
35 | #define DSISR_MC_SLB_PARMULTI 0x040 /* SLB parity + multi-hit */ | |
36 | ||
37 | /* POWER7 SLB flush and reload */ | |
38 | static void reload_slb(struct kvm_vcpu *vcpu) | |
39 | { | |
40 | struct slb_shadow *slb; | |
41 | unsigned long i, n; | |
42 | ||
43 | /* First clear out SLB */ | |
44 | asm volatile("slbmte %0,%0; slbia" : : "r" (0)); | |
45 | ||
46 | /* Do they have an SLB shadow buffer registered? */ | |
47 | slb = vcpu->arch.slb_shadow.pinned_addr; | |
48 | if (!slb) | |
49 | return; | |
50 | ||
51 | /* Sanity check */ | |
02407552 | 52 | n = min_t(u32, be32_to_cpu(slb->persistent), SLB_MIN_SIZE); |
b4072df4 PM |
53 | if ((void *) &slb->save_area[n] > vcpu->arch.slb_shadow.pinned_end) |
54 | return; | |
55 | ||
56 | /* Load up the SLB from that */ | |
57 | for (i = 0; i < n; ++i) { | |
02407552 AG |
58 | unsigned long rb = be64_to_cpu(slb->save_area[i].esid); |
59 | unsigned long rs = be64_to_cpu(slb->save_area[i].vsid); | |
b4072df4 PM |
60 | |
61 | rb = (rb & ~0xFFFul) | i; /* insert entry number */ | |
62 | asm volatile("slbmte %0,%1" : : "r" (rs), "r" (rb)); | |
63 | } | |
64 | } | |
65 | ||
b4072df4 PM |
66 | /* |
67 | * On POWER7, see if we can handle a machine check that occurred inside | |
68 | * the guest in real mode, without switching to the host partition. | |
69 | * | |
70 | * Returns: 0 => exit guest, 1 => deliver machine check to guest | |
71 | */ | |
72 | static long kvmppc_realmode_mc_power7(struct kvm_vcpu *vcpu) | |
73 | { | |
74 | unsigned long srr1 = vcpu->arch.shregs.msr; | |
36df96f8 | 75 | struct machine_check_event mce_evt; |
b4072df4 PM |
76 | long handled = 1; |
77 | ||
78 | if (srr1 & SRR1_MC_LDSTERR) { | |
79 | /* error on load/store */ | |
80 | unsigned long dsisr = vcpu->arch.shregs.dsisr; | |
81 | ||
82 | if (dsisr & (DSISR_MC_SLB_PARMULTI | DSISR_MC_SLB_MULTI | | |
83 | DSISR_MC_SLB_PARITY | DSISR_MC_DERAT_MULTI)) { | |
84 | /* flush and reload SLB; flushes D-ERAT too */ | |
85 | reload_slb(vcpu); | |
86 | dsisr &= ~(DSISR_MC_SLB_PARMULTI | DSISR_MC_SLB_MULTI | | |
87 | DSISR_MC_SLB_PARITY | DSISR_MC_DERAT_MULTI); | |
88 | } | |
89 | if (dsisr & DSISR_MC_TLB_MULTI) { | |
04407050 | 90 | if (cur_cpu_spec && cur_cpu_spec->flush_tlb) |
45706bb5 | 91 | cur_cpu_spec->flush_tlb(TLB_INVAL_SCOPE_LPID); |
b4072df4 PM |
92 | dsisr &= ~DSISR_MC_TLB_MULTI; |
93 | } | |
94 | /* Any other errors we don't understand? */ | |
95 | if (dsisr & 0xffffffffUL) | |
96 | handled = 0; | |
97 | } | |
98 | ||
99 | switch ((srr1 >> SRR1_MC_IFETCH_SH) & SRR1_MC_IFETCH_MASK) { | |
100 | case 0: | |
101 | break; | |
102 | case SRR1_MC_IFETCH_SLBPAR: | |
103 | case SRR1_MC_IFETCH_SLBMULTI: | |
104 | case SRR1_MC_IFETCH_SLBPARMULTI: | |
105 | reload_slb(vcpu); | |
106 | break; | |
107 | case SRR1_MC_IFETCH_TLBMULTI: | |
04407050 | 108 | if (cur_cpu_spec && cur_cpu_spec->flush_tlb) |
45706bb5 | 109 | cur_cpu_spec->flush_tlb(TLB_INVAL_SCOPE_LPID); |
b4072df4 PM |
110 | break; |
111 | default: | |
112 | handled = 0; | |
113 | } | |
114 | ||
115 | /* | |
36df96f8 MS |
116 | * See if we have already handled the condition in the linux host. |
117 | * We assume that if the condition is recovered then linux host | |
b4072df4 PM |
118 | * will have generated an error log event that we will pick |
119 | * up and log later. | |
74845bc2 MS |
120 | * Don't release mce event now. We will queue up the event so that |
121 | * we can log the MCE event info on host console. | |
b4072df4 | 122 | */ |
36df96f8 MS |
123 | if (!get_mce_event(&mce_evt, MCE_EVENT_DONTRELEASE)) |
124 | goto out; | |
125 | ||
126 | if (mce_evt.version == MCE_V1 && | |
127 | (mce_evt.severity == MCE_SEV_NO_ERROR || | |
128 | mce_evt.disposition == MCE_DISPOSITION_RECOVERED)) | |
b4072df4 PM |
129 | handled = 1; |
130 | ||
36df96f8 MS |
131 | out: |
132 | /* | |
e20bbd3d AP |
133 | * For guest that supports FWNMI capability, hook the MCE event into |
134 | * vcpu structure. We are going to exit the guest with KVM_EXIT_NMI | |
135 | * exit reason. On our way to exit we will pull this event from vcpu | |
136 | * structure and print it from thread 0 of the core/subcore. | |
137 | * | |
138 | * For guest that does not support FWNMI capability (old QEMU): | |
74845bc2 MS |
139 | * We are now going enter guest either through machine check |
140 | * interrupt (for unhandled errors) or will continue from | |
141 | * current HSRR0 (for handled errors) in guest. Hence | |
142 | * queue up the event so that we can log it from host console later. | |
36df96f8 | 143 | */ |
e20bbd3d AP |
144 | if (vcpu->kvm->arch.fwnmi_enabled) { |
145 | /* | |
146 | * Hook up the mce event on to vcpu structure. | |
147 | * First clear the old event. | |
148 | */ | |
149 | memset(&vcpu->arch.mce_evt, 0, sizeof(vcpu->arch.mce_evt)); | |
150 | if (get_mce_event(&mce_evt, MCE_EVENT_RELEASE)) { | |
151 | vcpu->arch.mce_evt = mce_evt; | |
152 | } | |
153 | } else | |
154 | machine_check_queue_event(); | |
b4072df4 PM |
155 | |
156 | return handled; | |
157 | } | |
158 | ||
159 | long kvmppc_realmode_machine_check(struct kvm_vcpu *vcpu) | |
160 | { | |
c17b98cf | 161 | return kvmppc_realmode_mc_power7(vcpu); |
b4072df4 | 162 | } |
fd7bacbc MS |
163 | |
164 | /* Check if dynamic split is in force and return subcore size accordingly. */ | |
165 | static inline int kvmppc_cur_subcore_size(void) | |
166 | { | |
167 | if (local_paca->kvm_hstate.kvm_split_mode) | |
168 | return local_paca->kvm_hstate.kvm_split_mode->subcore_size; | |
169 | ||
170 | return threads_per_subcore; | |
171 | } | |
172 | ||
173 | void kvmppc_subcore_enter_guest(void) | |
174 | { | |
175 | int thread_id, subcore_id; | |
176 | ||
177 | thread_id = cpu_thread_in_core(local_paca->paca_index); | |
178 | subcore_id = thread_id / kvmppc_cur_subcore_size(); | |
179 | ||
180 | local_paca->sibling_subcore_state->in_guest[subcore_id] = 1; | |
181 | } | |
182 | ||
183 | void kvmppc_subcore_exit_guest(void) | |
184 | { | |
185 | int thread_id, subcore_id; | |
186 | ||
187 | thread_id = cpu_thread_in_core(local_paca->paca_index); | |
188 | subcore_id = thread_id / kvmppc_cur_subcore_size(); | |
189 | ||
190 | local_paca->sibling_subcore_state->in_guest[subcore_id] = 0; | |
191 | } | |
192 | ||
193 | static bool kvmppc_tb_resync_required(void) | |
194 | { | |
195 | if (test_and_set_bit(CORE_TB_RESYNC_REQ_BIT, | |
196 | &local_paca->sibling_subcore_state->flags)) | |
197 | return false; | |
198 | ||
199 | return true; | |
200 | } | |
201 | ||
202 | static void kvmppc_tb_resync_done(void) | |
203 | { | |
204 | clear_bit(CORE_TB_RESYNC_REQ_BIT, | |
205 | &local_paca->sibling_subcore_state->flags); | |
206 | } | |
207 | ||
208 | /* | |
209 | * kvmppc_realmode_hmi_handler() is called only by primary thread during | |
210 | * guest exit path. | |
211 | * | |
212 | * There are multiple reasons why HMI could occur, one of them is | |
213 | * Timebase (TB) error. If this HMI is due to TB error, then TB would | |
214 | * have been in stopped state. The opal hmi handler Will fix it and | |
215 | * restore the TB value with host timebase value. For HMI caused due | |
216 | * to non-TB errors, opal hmi handler will not touch/restore TB register | |
217 | * and hence there won't be any change in TB value. | |
218 | * | |
219 | * Since we are not sure about the cause of this HMI, we can't be sure | |
220 | * about the content of TB register whether it holds guest or host timebase | |
221 | * value. Hence the idea is to resync the TB on every HMI, so that we | |
222 | * know about the exact state of the TB value. Resync TB call will | |
223 | * restore TB to host timebase. | |
224 | * | |
225 | * Things to consider: | |
226 | * - On TB error, HMI interrupt is reported on all the threads of the core | |
227 | * that has encountered TB error irrespective of split-core mode. | |
228 | * - The very first thread on the core that get chance to fix TB error | |
229 | * would rsync the TB with local chipTOD value. | |
230 | * - The resync TB is a core level action i.e. it will sync all the TBs | |
231 | * in that core independent of split-core mode. This means if we trigger | |
232 | * TB sync from a thread from one subcore, it would affect TB values of | |
233 | * sibling subcores of the same core. | |
234 | * | |
235 | * All threads need to co-ordinate before making opal hmi handler. | |
236 | * All threads will use sibling_subcore_state->in_guest[] (shared by all | |
237 | * threads in the core) in paca which holds information about whether | |
238 | * sibling subcores are in Guest mode or host mode. The in_guest[] array | |
239 | * is of size MAX_SUBCORE_PER_CORE=4, indexed using subcore id to set/unset | |
240 | * subcore status. Only primary threads from each subcore is responsible | |
241 | * to set/unset its designated array element while entering/exiting the | |
242 | * guset. | |
243 | * | |
244 | * After invoking opal hmi handler call, one of the thread (of entire core) | |
245 | * will need to resync the TB. Bit 63 from subcore state bitmap flags | |
246 | * (sibling_subcore_state->flags) will be used to co-ordinate between | |
247 | * primary threads to decide who takes up the responsibility. | |
248 | * | |
249 | * This is what we do: | |
250 | * - Primary thread from each subcore tries to set resync required bit[63] | |
251 | * of paca->sibling_subcore_state->flags. | |
252 | * - The first primary thread that is able to set the flag takes the | |
253 | * responsibility of TB resync. (Let us call it as thread leader) | |
254 | * - All other threads which are in host will call | |
255 | * wait_for_subcore_guest_exit() and wait for in_guest[0-3] from | |
256 | * paca->sibling_subcore_state to get cleared. | |
257 | * - All the primary thread will clear its subcore status from subcore | |
258 | * state in_guest[] array respectively. | |
259 | * - Once all primary threads clear in_guest[0-3], all of them will invoke | |
260 | * opal hmi handler. | |
261 | * - Now all threads will wait for TB resync to complete by invoking | |
262 | * wait_for_tb_resync() except the thread leader. | |
263 | * - Thread leader will do a TB resync by invoking opal_resync_timebase() | |
264 | * call and the it will clear the resync required bit. | |
265 | * - All other threads will now come out of resync wait loop and proceed | |
266 | * with individual execution. | |
267 | * - On return of this function, primary thread will signal all | |
268 | * secondary threads to proceed. | |
269 | * - All secondary threads will eventually call opal hmi handler on | |
270 | * their exit path. | |
d075745d PM |
271 | * |
272 | * Returns 1 if the timebase offset should be applied, 0 if not. | |
fd7bacbc MS |
273 | */ |
274 | ||
275 | long kvmppc_realmode_hmi_handler(void) | |
276 | { | |
fd7bacbc MS |
277 | bool resync_req; |
278 | ||
fd7bacbc MS |
279 | __this_cpu_inc(irq_stat.hmi_exceptions); |
280 | ||
d075745d PM |
281 | if (hmi_handle_debugtrig(NULL) >= 0) |
282 | return 1; | |
283 | ||
fd7bacbc MS |
284 | /* |
285 | * By now primary thread has already completed guest->host | |
286 | * partition switch but haven't signaled secondaries yet. | |
287 | * All the secondary threads on this subcore is waiting | |
288 | * for primary thread to signal them to go ahead. | |
289 | * | |
290 | * For threads from subcore which isn't in guest, they all will | |
291 | * wait until all other subcores on this core exit the guest. | |
292 | * | |
293 | * Now set the resync required bit. If you are the first to | |
294 | * set this bit then kvmppc_tb_resync_required() function will | |
295 | * return true. For rest all other subcores | |
296 | * kvmppc_tb_resync_required() will return false. | |
297 | * | |
298 | * If resync_req == true, then this thread is responsible to | |
299 | * initiate TB resync after hmi handler has completed. | |
300 | * All other threads on this core will wait until this thread | |
301 | * clears the resync required bit flag. | |
302 | */ | |
303 | resync_req = kvmppc_tb_resync_required(); | |
304 | ||
305 | /* Reset the subcore status to indicate it has exited guest */ | |
306 | kvmppc_subcore_exit_guest(); | |
307 | ||
308 | /* | |
309 | * Wait for other subcores on this core to exit the guest. | |
310 | * All the primary threads and threads from subcore that are | |
311 | * not in guest will wait here until all subcores are out | |
312 | * of guest context. | |
313 | */ | |
314 | wait_for_subcore_guest_exit(); | |
315 | ||
316 | /* | |
317 | * At this point we are sure that primary threads from each | |
318 | * subcore on this core have completed guest->host partition | |
319 | * switch. Now it is safe to call HMI handler. | |
320 | */ | |
321 | if (ppc_md.hmi_exception_early) | |
322 | ppc_md.hmi_exception_early(NULL); | |
323 | ||
324 | /* | |
325 | * Check if this thread is responsible to resync TB. | |
326 | * All other threads will wait until this thread completes the | |
327 | * TB resync. | |
328 | */ | |
329 | if (resync_req) { | |
330 | opal_resync_timebase(); | |
331 | /* Reset TB resync req bit */ | |
332 | kvmppc_tb_resync_done(); | |
333 | } else { | |
334 | wait_for_tb_resync(); | |
335 | } | |
336 | return 0; | |
337 | } |