Commit | Line | Data |
---|---|---|
d2912cb1 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
b4072df4 | 2 | /* |
b4072df4 PM |
3 | * |
4 | * Copyright 2012 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> | |
5 | */ | |
6 | ||
7 | #include <linux/types.h> | |
8 | #include <linux/string.h> | |
9 | #include <linux/kvm.h> | |
10 | #include <linux/kvm_host.h> | |
11 | #include <linux/kernel.h> | |
12 | #include <asm/opal.h> | |
36df96f8 | 13 | #include <asm/mce.h> |
fd7bacbc MS |
14 | #include <asm/machdep.h> |
15 | #include <asm/cputhreads.h> | |
16 | #include <asm/hmi.h> | |
e34af784 | 17 | #include <asm/kvm_ppc.h> |
b4072df4 PM |
18 | |
19 | /* SRR1 bits for machine check on POWER7 */ | |
20 | #define SRR1_MC_LDSTERR (1ul << (63-42)) | |
21 | #define SRR1_MC_IFETCH_SH (63-45) | |
22 | #define SRR1_MC_IFETCH_MASK 0x7 | |
23 | #define SRR1_MC_IFETCH_SLBPAR 2 /* SLB parity error */ | |
24 | #define SRR1_MC_IFETCH_SLBMULTI 3 /* SLB multi-hit */ | |
25 | #define SRR1_MC_IFETCH_SLBPARMULTI 4 /* SLB parity + multi-hit */ | |
26 | #define SRR1_MC_IFETCH_TLBMULTI 5 /* I-TLB multi-hit */ | |
27 | ||
28 | /* DSISR bits for machine check on POWER7 */ | |
29 | #define DSISR_MC_DERAT_MULTI 0x800 /* D-ERAT multi-hit */ | |
30 | #define DSISR_MC_TLB_MULTI 0x400 /* D-TLB multi-hit */ | |
31 | #define DSISR_MC_SLB_PARITY 0x100 /* SLB parity error */ | |
32 | #define DSISR_MC_SLB_MULTI 0x080 /* SLB multi-hit */ | |
33 | #define DSISR_MC_SLB_PARMULTI 0x040 /* SLB parity + multi-hit */ | |
34 | ||
35 | /* POWER7 SLB flush and reload */ | |
36 | static void reload_slb(struct kvm_vcpu *vcpu) | |
37 | { | |
38 | struct slb_shadow *slb; | |
39 | unsigned long i, n; | |
40 | ||
41 | /* First clear out SLB */ | |
42 | asm volatile("slbmte %0,%0; slbia" : : "r" (0)); | |
43 | ||
44 | /* Do they have an SLB shadow buffer registered? */ | |
45 | slb = vcpu->arch.slb_shadow.pinned_addr; | |
46 | if (!slb) | |
47 | return; | |
48 | ||
49 | /* Sanity check */ | |
02407552 | 50 | n = min_t(u32, be32_to_cpu(slb->persistent), SLB_MIN_SIZE); |
b4072df4 PM |
51 | if ((void *) &slb->save_area[n] > vcpu->arch.slb_shadow.pinned_end) |
52 | return; | |
53 | ||
54 | /* Load up the SLB from that */ | |
55 | for (i = 0; i < n; ++i) { | |
02407552 AG |
56 | unsigned long rb = be64_to_cpu(slb->save_area[i].esid); |
57 | unsigned long rs = be64_to_cpu(slb->save_area[i].vsid); | |
b4072df4 PM |
58 | |
59 | rb = (rb & ~0xFFFul) | i; /* insert entry number */ | |
60 | asm volatile("slbmte %0,%1" : : "r" (rs), "r" (rb)); | |
61 | } | |
62 | } | |
63 | ||
b4072df4 PM |
64 | /* |
65 | * On POWER7, see if we can handle a machine check that occurred inside | |
66 | * the guest in real mode, without switching to the host partition. | |
b4072df4 | 67 | */ |
884dfb72 | 68 | static void kvmppc_realmode_mc_power7(struct kvm_vcpu *vcpu) |
b4072df4 PM |
69 | { |
70 | unsigned long srr1 = vcpu->arch.shregs.msr; | |
36df96f8 | 71 | struct machine_check_event mce_evt; |
b4072df4 PM |
72 | long handled = 1; |
73 | ||
74 | if (srr1 & SRR1_MC_LDSTERR) { | |
75 | /* error on load/store */ | |
76 | unsigned long dsisr = vcpu->arch.shregs.dsisr; | |
77 | ||
78 | if (dsisr & (DSISR_MC_SLB_PARMULTI | DSISR_MC_SLB_MULTI | | |
79 | DSISR_MC_SLB_PARITY | DSISR_MC_DERAT_MULTI)) { | |
80 | /* flush and reload SLB; flushes D-ERAT too */ | |
81 | reload_slb(vcpu); | |
82 | dsisr &= ~(DSISR_MC_SLB_PARMULTI | DSISR_MC_SLB_MULTI | | |
83 | DSISR_MC_SLB_PARITY | DSISR_MC_DERAT_MULTI); | |
84 | } | |
85 | if (dsisr & DSISR_MC_TLB_MULTI) { | |
d4748276 | 86 | tlbiel_all_lpid(vcpu->kvm->arch.radix); |
b4072df4 PM |
87 | dsisr &= ~DSISR_MC_TLB_MULTI; |
88 | } | |
89 | /* Any other errors we don't understand? */ | |
90 | if (dsisr & 0xffffffffUL) | |
91 | handled = 0; | |
92 | } | |
93 | ||
94 | switch ((srr1 >> SRR1_MC_IFETCH_SH) & SRR1_MC_IFETCH_MASK) { | |
95 | case 0: | |
96 | break; | |
97 | case SRR1_MC_IFETCH_SLBPAR: | |
98 | case SRR1_MC_IFETCH_SLBMULTI: | |
99 | case SRR1_MC_IFETCH_SLBPARMULTI: | |
100 | reload_slb(vcpu); | |
101 | break; | |
102 | case SRR1_MC_IFETCH_TLBMULTI: | |
d4748276 | 103 | tlbiel_all_lpid(vcpu->kvm->arch.radix); |
b4072df4 PM |
104 | break; |
105 | default: | |
106 | handled = 0; | |
107 | } | |
108 | ||
109 | /* | |
884dfb72 PM |
110 | * Now get the event and stash it in the vcpu struct so it can |
111 | * be handled by the primary thread in virtual mode. We can't | |
112 | * call machine_check_queue_event() here if we are running on | |
113 | * an offline secondary thread. | |
b4072df4 | 114 | */ |
884dfb72 PM |
115 | if (get_mce_event(&mce_evt, MCE_EVENT_RELEASE)) { |
116 | if (handled && mce_evt.version == MCE_V1) | |
117 | mce_evt.disposition = MCE_DISPOSITION_RECOVERED; | |
118 | } else { | |
119 | memset(&mce_evt, 0, sizeof(mce_evt)); | |
120 | } | |
b4072df4 | 121 | |
884dfb72 | 122 | vcpu->arch.mce_evt = mce_evt; |
b4072df4 PM |
123 | } |
124 | ||
884dfb72 | 125 | void kvmppc_realmode_machine_check(struct kvm_vcpu *vcpu) |
b4072df4 | 126 | { |
884dfb72 | 127 | kvmppc_realmode_mc_power7(vcpu); |
b4072df4 | 128 | } |
fd7bacbc MS |
129 | |
130 | /* Check if dynamic split is in force and return subcore size accordingly. */ | |
131 | static inline int kvmppc_cur_subcore_size(void) | |
132 | { | |
133 | if (local_paca->kvm_hstate.kvm_split_mode) | |
134 | return local_paca->kvm_hstate.kvm_split_mode->subcore_size; | |
135 | ||
136 | return threads_per_subcore; | |
137 | } | |
138 | ||
139 | void kvmppc_subcore_enter_guest(void) | |
140 | { | |
141 | int thread_id, subcore_id; | |
142 | ||
143 | thread_id = cpu_thread_in_core(local_paca->paca_index); | |
144 | subcore_id = thread_id / kvmppc_cur_subcore_size(); | |
145 | ||
146 | local_paca->sibling_subcore_state->in_guest[subcore_id] = 1; | |
147 | } | |
95a6432c | 148 | EXPORT_SYMBOL_GPL(kvmppc_subcore_enter_guest); |
fd7bacbc MS |
149 | |
150 | void kvmppc_subcore_exit_guest(void) | |
151 | { | |
152 | int thread_id, subcore_id; | |
153 | ||
154 | thread_id = cpu_thread_in_core(local_paca->paca_index); | |
155 | subcore_id = thread_id / kvmppc_cur_subcore_size(); | |
156 | ||
157 | local_paca->sibling_subcore_state->in_guest[subcore_id] = 0; | |
158 | } | |
95a6432c | 159 | EXPORT_SYMBOL_GPL(kvmppc_subcore_exit_guest); |
fd7bacbc MS |
160 | |
161 | static bool kvmppc_tb_resync_required(void) | |
162 | { | |
163 | if (test_and_set_bit(CORE_TB_RESYNC_REQ_BIT, | |
164 | &local_paca->sibling_subcore_state->flags)) | |
165 | return false; | |
166 | ||
167 | return true; | |
168 | } | |
169 | ||
170 | static void kvmppc_tb_resync_done(void) | |
171 | { | |
172 | clear_bit(CORE_TB_RESYNC_REQ_BIT, | |
173 | &local_paca->sibling_subcore_state->flags); | |
174 | } | |
175 | ||
176 | /* | |
177 | * kvmppc_realmode_hmi_handler() is called only by primary thread during | |
178 | * guest exit path. | |
179 | * | |
180 | * There are multiple reasons why HMI could occur, one of them is | |
181 | * Timebase (TB) error. If this HMI is due to TB error, then TB would | |
182 | * have been in stopped state. The opal hmi handler Will fix it and | |
183 | * restore the TB value with host timebase value. For HMI caused due | |
184 | * to non-TB errors, opal hmi handler will not touch/restore TB register | |
185 | * and hence there won't be any change in TB value. | |
186 | * | |
187 | * Since we are not sure about the cause of this HMI, we can't be sure | |
188 | * about the content of TB register whether it holds guest or host timebase | |
189 | * value. Hence the idea is to resync the TB on every HMI, so that we | |
190 | * know about the exact state of the TB value. Resync TB call will | |
191 | * restore TB to host timebase. | |
192 | * | |
193 | * Things to consider: | |
194 | * - On TB error, HMI interrupt is reported on all the threads of the core | |
195 | * that has encountered TB error irrespective of split-core mode. | |
196 | * - The very first thread on the core that get chance to fix TB error | |
197 | * would rsync the TB with local chipTOD value. | |
198 | * - The resync TB is a core level action i.e. it will sync all the TBs | |
199 | * in that core independent of split-core mode. This means if we trigger | |
200 | * TB sync from a thread from one subcore, it would affect TB values of | |
201 | * sibling subcores of the same core. | |
202 | * | |
203 | * All threads need to co-ordinate before making opal hmi handler. | |
204 | * All threads will use sibling_subcore_state->in_guest[] (shared by all | |
205 | * threads in the core) in paca which holds information about whether | |
206 | * sibling subcores are in Guest mode or host mode. The in_guest[] array | |
207 | * is of size MAX_SUBCORE_PER_CORE=4, indexed using subcore id to set/unset | |
208 | * subcore status. Only primary threads from each subcore is responsible | |
209 | * to set/unset its designated array element while entering/exiting the | |
210 | * guset. | |
211 | * | |
212 | * After invoking opal hmi handler call, one of the thread (of entire core) | |
213 | * will need to resync the TB. Bit 63 from subcore state bitmap flags | |
214 | * (sibling_subcore_state->flags) will be used to co-ordinate between | |
215 | * primary threads to decide who takes up the responsibility. | |
216 | * | |
217 | * This is what we do: | |
218 | * - Primary thread from each subcore tries to set resync required bit[63] | |
219 | * of paca->sibling_subcore_state->flags. | |
220 | * - The first primary thread that is able to set the flag takes the | |
221 | * responsibility of TB resync. (Let us call it as thread leader) | |
222 | * - All other threads which are in host will call | |
223 | * wait_for_subcore_guest_exit() and wait for in_guest[0-3] from | |
224 | * paca->sibling_subcore_state to get cleared. | |
225 | * - All the primary thread will clear its subcore status from subcore | |
226 | * state in_guest[] array respectively. | |
227 | * - Once all primary threads clear in_guest[0-3], all of them will invoke | |
228 | * opal hmi handler. | |
229 | * - Now all threads will wait for TB resync to complete by invoking | |
230 | * wait_for_tb_resync() except the thread leader. | |
231 | * - Thread leader will do a TB resync by invoking opal_resync_timebase() | |
232 | * call and the it will clear the resync required bit. | |
233 | * - All other threads will now come out of resync wait loop and proceed | |
234 | * with individual execution. | |
235 | * - On return of this function, primary thread will signal all | |
236 | * secondary threads to proceed. | |
237 | * - All secondary threads will eventually call opal hmi handler on | |
238 | * their exit path. | |
d075745d PM |
239 | * |
240 | * Returns 1 if the timebase offset should be applied, 0 if not. | |
fd7bacbc MS |
241 | */ |
242 | ||
243 | long kvmppc_realmode_hmi_handler(void) | |
244 | { | |
fd7bacbc MS |
245 | bool resync_req; |
246 | ||
fd7bacbc MS |
247 | __this_cpu_inc(irq_stat.hmi_exceptions); |
248 | ||
d075745d PM |
249 | if (hmi_handle_debugtrig(NULL) >= 0) |
250 | return 1; | |
251 | ||
fd7bacbc MS |
252 | /* |
253 | * By now primary thread has already completed guest->host | |
254 | * partition switch but haven't signaled secondaries yet. | |
255 | * All the secondary threads on this subcore is waiting | |
256 | * for primary thread to signal them to go ahead. | |
257 | * | |
258 | * For threads from subcore which isn't in guest, they all will | |
259 | * wait until all other subcores on this core exit the guest. | |
260 | * | |
261 | * Now set the resync required bit. If you are the first to | |
262 | * set this bit then kvmppc_tb_resync_required() function will | |
263 | * return true. For rest all other subcores | |
264 | * kvmppc_tb_resync_required() will return false. | |
265 | * | |
266 | * If resync_req == true, then this thread is responsible to | |
267 | * initiate TB resync after hmi handler has completed. | |
268 | * All other threads on this core will wait until this thread | |
269 | * clears the resync required bit flag. | |
270 | */ | |
271 | resync_req = kvmppc_tb_resync_required(); | |
272 | ||
273 | /* Reset the subcore status to indicate it has exited guest */ | |
274 | kvmppc_subcore_exit_guest(); | |
275 | ||
276 | /* | |
277 | * Wait for other subcores on this core to exit the guest. | |
278 | * All the primary threads and threads from subcore that are | |
279 | * not in guest will wait here until all subcores are out | |
280 | * of guest context. | |
281 | */ | |
282 | wait_for_subcore_guest_exit(); | |
283 | ||
284 | /* | |
285 | * At this point we are sure that primary threads from each | |
286 | * subcore on this core have completed guest->host partition | |
287 | * switch. Now it is safe to call HMI handler. | |
288 | */ | |
289 | if (ppc_md.hmi_exception_early) | |
290 | ppc_md.hmi_exception_early(NULL); | |
291 | ||
292 | /* | |
293 | * Check if this thread is responsible to resync TB. | |
294 | * All other threads will wait until this thread completes the | |
295 | * TB resync. | |
296 | */ | |
297 | if (resync_req) { | |
298 | opal_resync_timebase(); | |
299 | /* Reset TB resync req bit */ | |
300 | kvmppc_tb_resync_done(); | |
301 | } else { | |
302 | wait_for_tb_resync(); | |
303 | } | |
df709a29 PM |
304 | |
305 | /* | |
306 | * Reset tb_offset_applied so the guest exit code won't try | |
307 | * to subtract the previous timebase offset from the timebase. | |
308 | */ | |
309 | if (local_paca->kvm_hstate.kvm_vcore) | |
310 | local_paca->kvm_hstate.kvm_vcore->tb_offset_applied = 0; | |
311 | ||
fd7bacbc MS |
312 | return 0; |
313 | } |