Commit | Line | Data |
---|---|---|
53631b54 CM |
1 | /* |
2 | * FP/SIMD context switching and fault handling | |
3 | * | |
4 | * Copyright (C) 2012 ARM Ltd. | |
5 | * Author: Catalin Marinas <catalin.marinas@arm.com> | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License version 2 as | |
9 | * published by the Free Software Foundation. | |
10 | * | |
11 | * This program is distributed in the hope that it will be useful, | |
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | * GNU General Public License for more details. | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License | |
17 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
18 | */ | |
19 | ||
cb84d11e | 20 | #include <linux/bottom_half.h> |
32365e64 | 21 | #include <linux/cpu.h> |
fb1ab1ab | 22 | #include <linux/cpu_pm.h> |
53631b54 | 23 | #include <linux/kernel.h> |
94ef7ecb | 24 | #include <linux/linkage.h> |
53631b54 | 25 | #include <linux/init.h> |
cb84d11e | 26 | #include <linux/percpu.h> |
4328825d | 27 | #include <linux/preempt.h> |
3f07c014 | 28 | #include <linux/sched/signal.h> |
53631b54 CM |
29 | #include <linux/signal.h> |
30 | ||
31 | #include <asm/fpsimd.h> | |
32 | #include <asm/cputype.h> | |
4328825d | 33 | #include <asm/simd.h> |
53631b54 CM |
34 | |
35 | #define FPEXC_IOF (1 << 0) | |
36 | #define FPEXC_DZF (1 << 1) | |
37 | #define FPEXC_OFF (1 << 2) | |
38 | #define FPEXC_UFF (1 << 3) | |
39 | #define FPEXC_IXF (1 << 4) | |
40 | #define FPEXC_IDF (1 << 7) | |
41 | ||
005f78cd AB |
42 | /* |
43 | * In order to reduce the number of times the FPSIMD state is needlessly saved | |
44 | * and restored, we need to keep track of two things: | |
45 | * (a) for each task, we need to remember which CPU was the last one to have | |
46 | * the task's FPSIMD state loaded into its FPSIMD registers; | |
47 | * (b) for each CPU, we need to remember which task's userland FPSIMD state has | |
48 | * been loaded into its FPSIMD registers most recently, or whether it has | |
49 | * been used to perform kernel mode NEON in the meantime. | |
50 | * | |
51 | * For (a), we add a 'cpu' field to struct fpsimd_state, which gets updated to | |
ef769e32 | 52 | * the id of the current CPU every time the state is loaded onto a CPU. For (b), |
005f78cd AB |
53 | * we add the per-cpu variable 'fpsimd_last_state' (below), which contains the |
54 | * address of the userland FPSIMD state of the task that was loaded onto the CPU | |
55 | * the most recently, or NULL if kernel mode NEON has been performed after that. | |
56 | * | |
57 | * With this in place, we no longer have to restore the next FPSIMD state right | |
58 | * when switching between tasks. Instead, we can defer this check to userland | |
59 | * resume, at which time we verify whether the CPU's fpsimd_last_state and the | |
60 | * task's fpsimd_state.cpu are still mutually in sync. If this is the case, we | |
61 | * can omit the FPSIMD restore. | |
62 | * | |
63 | * As an optimization, we use the thread_info flag TIF_FOREIGN_FPSTATE to | |
64 | * indicate whether or not the userland FPSIMD state of the current task is | |
65 | * present in the registers. The flag is set unless the FPSIMD registers of this | |
66 | * CPU currently contain the most recent userland FPSIMD state of the current | |
67 | * task. | |
68 | * | |
cb84d11e DM |
69 | * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may |
70 | * save the task's FPSIMD context back to task_struct from softirq context. | |
71 | * To prevent this from racing with the manipulation of the task's FPSIMD state | |
72 | * from task context and thereby corrupting the state, it is necessary to | |
73 | * protect any manipulation of a task's fpsimd_state or TIF_FOREIGN_FPSTATE | |
74 | * flag with local_bh_disable() unless softirqs are already masked. | |
75 | * | |
005f78cd AB |
76 | * For a certain task, the sequence may look something like this: |
77 | * - the task gets scheduled in; if both the task's fpsimd_state.cpu field | |
78 | * contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu | |
79 | * variable points to the task's fpsimd_state, the TIF_FOREIGN_FPSTATE flag is | |
80 | * cleared, otherwise it is set; | |
81 | * | |
82 | * - the task returns to userland; if TIF_FOREIGN_FPSTATE is set, the task's | |
83 | * userland FPSIMD state is copied from memory to the registers, the task's | |
84 | * fpsimd_state.cpu field is set to the id of the current CPU, the current | |
85 | * CPU's fpsimd_last_state pointer is set to this task's fpsimd_state and the | |
86 | * TIF_FOREIGN_FPSTATE flag is cleared; | |
87 | * | |
88 | * - the task executes an ordinary syscall; upon return to userland, the | |
89 | * TIF_FOREIGN_FPSTATE flag will still be cleared, so no FPSIMD state is | |
90 | * restored; | |
91 | * | |
92 | * - the task executes a syscall which executes some NEON instructions; this is | |
93 | * preceded by a call to kernel_neon_begin(), which copies the task's FPSIMD | |
94 | * register contents to memory, clears the fpsimd_last_state per-cpu variable | |
95 | * and sets the TIF_FOREIGN_FPSTATE flag; | |
96 | * | |
97 | * - the task gets preempted after kernel_neon_end() is called; as we have not | |
98 | * returned from the 2nd syscall yet, TIF_FOREIGN_FPSTATE is still set so | |
99 | * whatever is in the FPSIMD registers is not saved to memory, but discarded. | |
100 | */ | |
101 | static DEFINE_PER_CPU(struct fpsimd_state *, fpsimd_last_state); | |
102 | ||
53631b54 CM |
103 | /* |
104 | * Trapped FP/ASIMD access. | |
105 | */ | |
94ef7ecb | 106 | asmlinkage void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs) |
53631b54 CM |
107 | { |
108 | /* TODO: implement lazy context saving/restoring */ | |
109 | WARN_ON(1); | |
110 | } | |
111 | ||
112 | /* | |
113 | * Raise a SIGFPE for the current process. | |
114 | */ | |
94ef7ecb | 115 | asmlinkage void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs) |
53631b54 CM |
116 | { |
117 | siginfo_t info; | |
118 | unsigned int si_code = 0; | |
119 | ||
120 | if (esr & FPEXC_IOF) | |
121 | si_code = FPE_FLTINV; | |
122 | else if (esr & FPEXC_DZF) | |
123 | si_code = FPE_FLTDIV; | |
124 | else if (esr & FPEXC_OFF) | |
125 | si_code = FPE_FLTOVF; | |
126 | else if (esr & FPEXC_UFF) | |
127 | si_code = FPE_FLTUND; | |
128 | else if (esr & FPEXC_IXF) | |
129 | si_code = FPE_FLTRES; | |
130 | ||
131 | memset(&info, 0, sizeof(info)); | |
132 | info.si_signo = SIGFPE; | |
133 | info.si_code = si_code; | |
134 | info.si_addr = (void __user *)instruction_pointer(regs); | |
135 | ||
136 | send_sig_info(SIGFPE, &info, current); | |
137 | } | |
138 | ||
139 | void fpsimd_thread_switch(struct task_struct *next) | |
140 | { | |
82e0191a SP |
141 | if (!system_supports_fpsimd()) |
142 | return; | |
005f78cd AB |
143 | /* |
144 | * Save the current FPSIMD state to memory, but only if whatever is in | |
145 | * the registers is in fact the most recent userland FPSIMD state of | |
146 | * 'current'. | |
147 | */ | |
148 | if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE)) | |
53631b54 | 149 | fpsimd_save_state(¤t->thread.fpsimd_state); |
005f78cd AB |
150 | |
151 | if (next->mm) { | |
152 | /* | |
153 | * If we are switching to a task whose most recent userland | |
154 | * FPSIMD state is already in the registers of *this* cpu, | |
155 | * we can skip loading the state from memory. Otherwise, set | |
156 | * the TIF_FOREIGN_FPSTATE flag so the state will be loaded | |
157 | * upon the next return to userland. | |
158 | */ | |
159 | struct fpsimd_state *st = &next->thread.fpsimd_state; | |
160 | ||
161 | if (__this_cpu_read(fpsimd_last_state) == st | |
162 | && st->cpu == smp_processor_id()) | |
9cf5b54f | 163 | clear_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE); |
005f78cd | 164 | else |
9cf5b54f | 165 | set_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE); |
005f78cd | 166 | } |
53631b54 CM |
167 | } |
168 | ||
169 | void fpsimd_flush_thread(void) | |
170 | { | |
82e0191a SP |
171 | if (!system_supports_fpsimd()) |
172 | return; | |
cb84d11e DM |
173 | |
174 | local_bh_disable(); | |
175 | ||
53631b54 | 176 | memset(¤t->thread.fpsimd_state, 0, sizeof(struct fpsimd_state)); |
674c242c | 177 | fpsimd_flush_task_state(current); |
005f78cd | 178 | set_thread_flag(TIF_FOREIGN_FPSTATE); |
cb84d11e DM |
179 | |
180 | local_bh_enable(); | |
53631b54 CM |
181 | } |
182 | ||
c51f9269 | 183 | /* |
005f78cd AB |
184 | * Save the userland FPSIMD state of 'current' to memory, but only if the state |
185 | * currently held in the registers does in fact belong to 'current' | |
c51f9269 AB |
186 | */ |
187 | void fpsimd_preserve_current_state(void) | |
188 | { | |
82e0191a SP |
189 | if (!system_supports_fpsimd()) |
190 | return; | |
cb84d11e DM |
191 | |
192 | local_bh_disable(); | |
193 | ||
005f78cd AB |
194 | if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) |
195 | fpsimd_save_state(¤t->thread.fpsimd_state); | |
cb84d11e DM |
196 | |
197 | local_bh_enable(); | |
c51f9269 AB |
198 | } |
199 | ||
200 | /* | |
005f78cd AB |
201 | * Load the userland FPSIMD state of 'current' from memory, but only if the |
202 | * FPSIMD state already held in the registers is /not/ the most recent FPSIMD | |
203 | * state of 'current' | |
204 | */ | |
205 | void fpsimd_restore_current_state(void) | |
206 | { | |
82e0191a SP |
207 | if (!system_supports_fpsimd()) |
208 | return; | |
cb84d11e DM |
209 | |
210 | local_bh_disable(); | |
211 | ||
005f78cd AB |
212 | if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) { |
213 | struct fpsimd_state *st = ¤t->thread.fpsimd_state; | |
214 | ||
215 | fpsimd_load_state(st); | |
50464185 | 216 | __this_cpu_write(fpsimd_last_state, st); |
005f78cd AB |
217 | st->cpu = smp_processor_id(); |
218 | } | |
cb84d11e DM |
219 | |
220 | local_bh_enable(); | |
005f78cd AB |
221 | } |
222 | ||
223 | /* | |
224 | * Load an updated userland FPSIMD state for 'current' from memory and set the | |
225 | * flag that indicates that the FPSIMD register contents are the most recent | |
226 | * FPSIMD state of 'current' | |
c51f9269 AB |
227 | */ |
228 | void fpsimd_update_current_state(struct fpsimd_state *state) | |
229 | { | |
82e0191a SP |
230 | if (!system_supports_fpsimd()) |
231 | return; | |
cb84d11e DM |
232 | |
233 | local_bh_disable(); | |
234 | ||
c51f9269 | 235 | fpsimd_load_state(state); |
005f78cd AB |
236 | if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) { |
237 | struct fpsimd_state *st = ¤t->thread.fpsimd_state; | |
238 | ||
50464185 | 239 | __this_cpu_write(fpsimd_last_state, st); |
005f78cd AB |
240 | st->cpu = smp_processor_id(); |
241 | } | |
cb84d11e DM |
242 | |
243 | local_bh_enable(); | |
c51f9269 AB |
244 | } |
245 | ||
005f78cd AB |
246 | /* |
247 | * Invalidate live CPU copies of task t's FPSIMD state | |
248 | */ | |
249 | void fpsimd_flush_task_state(struct task_struct *t) | |
250 | { | |
251 | t->thread.fpsimd_state.cpu = NR_CPUS; | |
252 | } | |
253 | ||
4cfb3613 AB |
254 | #ifdef CONFIG_KERNEL_MODE_NEON |
255 | ||
cb84d11e | 256 | DEFINE_PER_CPU(bool, kernel_neon_busy); |
11cefd5a | 257 | EXPORT_PER_CPU_SYMBOL(kernel_neon_busy); |
190f1ca8 | 258 | |
4cfb3613 AB |
259 | /* |
260 | * Kernel-side NEON support functions | |
261 | */ | |
cb84d11e DM |
262 | |
263 | /* | |
264 | * kernel_neon_begin(): obtain the CPU FPSIMD registers for use by the calling | |
265 | * context | |
266 | * | |
267 | * Must not be called unless may_use_simd() returns true. | |
268 | * Task context in the FPSIMD registers is saved back to memory as necessary. | |
269 | * | |
270 | * A matching call to kernel_neon_end() must be made before returning from the | |
271 | * calling context. | |
272 | * | |
273 | * The caller may freely use the FPSIMD registers until kernel_neon_end() is | |
274 | * called. | |
275 | */ | |
276 | void kernel_neon_begin(void) | |
4cfb3613 | 277 | { |
82e0191a SP |
278 | if (WARN_ON(!system_supports_fpsimd())) |
279 | return; | |
4cfb3613 | 280 | |
cb84d11e DM |
281 | BUG_ON(!may_use_simd()); |
282 | ||
283 | local_bh_disable(); | |
284 | ||
285 | __this_cpu_write(kernel_neon_busy, true); | |
286 | ||
287 | /* Save unsaved task fpsimd state, if any: */ | |
288 | if (current->mm && !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE)) | |
289 | fpsimd_save_state(¤t->thread.fpsimd_state); | |
290 | ||
291 | /* Invalidate any task state remaining in the fpsimd regs: */ | |
292 | __this_cpu_write(fpsimd_last_state, NULL); | |
293 | ||
294 | preempt_disable(); | |
295 | ||
296 | local_bh_enable(); | |
4cfb3613 | 297 | } |
cb84d11e | 298 | EXPORT_SYMBOL(kernel_neon_begin); |
4cfb3613 | 299 | |
cb84d11e DM |
300 | /* |
301 | * kernel_neon_end(): give the CPU FPSIMD registers back to the current task | |
302 | * | |
303 | * Must be called from a context in which kernel_neon_begin() was previously | |
304 | * called, with no call to kernel_neon_end() in the meantime. | |
305 | * | |
306 | * The caller must not use the FPSIMD registers after this function is called, | |
307 | * unless kernel_neon_begin() is called again in the meantime. | |
308 | */ | |
4cfb3613 AB |
309 | void kernel_neon_end(void) |
310 | { | |
cb84d11e DM |
311 | bool busy; |
312 | ||
82e0191a SP |
313 | if (!system_supports_fpsimd()) |
314 | return; | |
cb84d11e DM |
315 | |
316 | busy = __this_cpu_xchg(kernel_neon_busy, false); | |
317 | WARN_ON(!busy); /* No matching kernel_neon_begin()? */ | |
318 | ||
319 | preempt_enable(); | |
4cfb3613 AB |
320 | } |
321 | EXPORT_SYMBOL(kernel_neon_end); | |
322 | ||
e580b8bc DM |
323 | #ifdef CONFIG_EFI |
324 | ||
3b66023d DM |
325 | static DEFINE_PER_CPU(struct fpsimd_state, efi_fpsimd_state); |
326 | static DEFINE_PER_CPU(bool, efi_fpsimd_state_used); | |
4328825d DM |
327 | |
328 | /* | |
329 | * EFI runtime services support functions | |
330 | * | |
331 | * The ABI for EFI runtime services allows EFI to use FPSIMD during the call. | |
332 | * This means that for EFI (and only for EFI), we have to assume that FPSIMD | |
333 | * is always used rather than being an optional accelerator. | |
334 | * | |
335 | * These functions provide the necessary support for ensuring FPSIMD | |
336 | * save/restore in the contexts from which EFI is used. | |
337 | * | |
338 | * Do not use them for any other purpose -- if tempted to do so, you are | |
339 | * either doing something wrong or you need to propose some refactoring. | |
340 | */ | |
341 | ||
342 | /* | |
343 | * __efi_fpsimd_begin(): prepare FPSIMD for making an EFI runtime services call | |
344 | */ | |
345 | void __efi_fpsimd_begin(void) | |
346 | { | |
347 | if (!system_supports_fpsimd()) | |
348 | return; | |
349 | ||
350 | WARN_ON(preemptible()); | |
351 | ||
352 | if (may_use_simd()) | |
353 | kernel_neon_begin(); | |
354 | else { | |
355 | fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state)); | |
356 | __this_cpu_write(efi_fpsimd_state_used, true); | |
357 | } | |
358 | } | |
359 | ||
360 | /* | |
361 | * __efi_fpsimd_end(): clean up FPSIMD after an EFI runtime services call | |
362 | */ | |
363 | void __efi_fpsimd_end(void) | |
364 | { | |
365 | if (!system_supports_fpsimd()) | |
366 | return; | |
367 | ||
368 | if (__this_cpu_xchg(efi_fpsimd_state_used, false)) | |
369 | fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state)); | |
370 | else | |
371 | kernel_neon_end(); | |
372 | } | |
373 | ||
e580b8bc DM |
374 | #endif /* CONFIG_EFI */ |
375 | ||
4cfb3613 AB |
376 | #endif /* CONFIG_KERNEL_MODE_NEON */ |
377 | ||
fb1ab1ab LP |
378 | #ifdef CONFIG_CPU_PM |
379 | static int fpsimd_cpu_pm_notifier(struct notifier_block *self, | |
380 | unsigned long cmd, void *v) | |
381 | { | |
382 | switch (cmd) { | |
383 | case CPU_PM_ENTER: | |
005f78cd | 384 | if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE)) |
fb1ab1ab | 385 | fpsimd_save_state(¤t->thread.fpsimd_state); |
7c68a9cc | 386 | this_cpu_write(fpsimd_last_state, NULL); |
fb1ab1ab LP |
387 | break; |
388 | case CPU_PM_EXIT: | |
389 | if (current->mm) | |
005f78cd | 390 | set_thread_flag(TIF_FOREIGN_FPSTATE); |
fb1ab1ab LP |
391 | break; |
392 | case CPU_PM_ENTER_FAILED: | |
393 | default: | |
394 | return NOTIFY_DONE; | |
395 | } | |
396 | return NOTIFY_OK; | |
397 | } | |
398 | ||
399 | static struct notifier_block fpsimd_cpu_pm_notifier_block = { | |
400 | .notifier_call = fpsimd_cpu_pm_notifier, | |
401 | }; | |
402 | ||
a7c61a34 | 403 | static void __init fpsimd_pm_init(void) |
fb1ab1ab LP |
404 | { |
405 | cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block); | |
406 | } | |
407 | ||
408 | #else | |
409 | static inline void fpsimd_pm_init(void) { } | |
410 | #endif /* CONFIG_CPU_PM */ | |
411 | ||
32365e64 | 412 | #ifdef CONFIG_HOTPLUG_CPU |
c23a7266 | 413 | static int fpsimd_cpu_dead(unsigned int cpu) |
32365e64 | 414 | { |
c23a7266 SAS |
415 | per_cpu(fpsimd_last_state, cpu) = NULL; |
416 | return 0; | |
32365e64 JL |
417 | } |
418 | ||
32365e64 JL |
419 | static inline void fpsimd_hotplug_init(void) |
420 | { | |
c23a7266 SAS |
421 | cpuhp_setup_state_nocalls(CPUHP_ARM64_FPSIMD_DEAD, "arm64/fpsimd:dead", |
422 | NULL, fpsimd_cpu_dead); | |
32365e64 JL |
423 | } |
424 | ||
425 | #else | |
426 | static inline void fpsimd_hotplug_init(void) { } | |
427 | #endif | |
428 | ||
53631b54 CM |
429 | /* |
430 | * FP/SIMD support code initialisation. | |
431 | */ | |
432 | static int __init fpsimd_init(void) | |
433 | { | |
fe80f9f2 SP |
434 | if (elf_hwcap & HWCAP_FP) { |
435 | fpsimd_pm_init(); | |
436 | fpsimd_hotplug_init(); | |
437 | } else { | |
53631b54 | 438 | pr_notice("Floating-point is not implemented\n"); |
53631b54 | 439 | } |
53631b54 | 440 | |
fe80f9f2 | 441 | if (!(elf_hwcap & HWCAP_ASIMD)) |
53631b54 | 442 | pr_notice("Advanced SIMD is not implemented\n"); |
fb1ab1ab | 443 | |
53631b54 CM |
444 | return 0; |
445 | } | |
446 | late_initcall(fpsimd_init); |