Commit | Line | Data |
---|---|---|
caab277b | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
0be7320a | 2 | /* |
7b7293ae | 3 | * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S |
0be7320a CM |
4 | * |
5 | * Copyright (C) 1996-2000 Russell King | |
6 | * Copyright (C) 2012 ARM Ltd. | |
0be7320a CM |
7 | */ |
8 | #ifndef __ASSEMBLY__ | |
9 | #error "Only include this from assembly code" | |
10 | #endif | |
11 | ||
f3e39273 MZ |
12 | #ifndef __ASM_ASSEMBLER_H |
13 | #define __ASM_ASSEMBLER_H | |
14 | ||
8fd7588f | 15 | #include <linux/export.h> |
386b3c7b | 16 | |
13150149 | 17 | #include <asm/alternative.h> |
755db234 | 18 | #include <asm/asm-bug.h> |
819771cc MR |
19 | #include <asm/asm-extable.h> |
20 | #include <asm/asm-offsets.h> | |
823066d9 | 21 | #include <asm/cpufeature.h> |
3e32131a | 22 | #include <asm/cputype.h> |
e28cc025 | 23 | #include <asm/debug-monitors.h> |
5003dbde | 24 | #include <asm/page.h> |
7b7293ae | 25 | #include <asm/pgtable-hwdef.h> |
0be7320a | 26 | #include <asm/ptrace.h> |
2a283070 | 27 | #include <asm/thread_info.h> |
0be7320a | 28 | |
4c4dcd35 AB |
29 | /* |
30 | * Provide a wxN alias for each wN register so what we can paste a xN | |
31 | * reference after a 'w' to obtain the 32-bit version. | |
32 | */ | |
33 | .irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 | |
34 | wx\n .req w\n | |
35 | .endr | |
36 | ||
0fbeb318 JM |
37 | .macro disable_daif |
38 | msr daifset, #0xf | |
39 | .endm | |
40 | ||
0be7320a | 41 | /* |
a82785a9 | 42 | * Save/restore interrupts. |
0be7320a | 43 | */ |
4b65a5db CM |
44 | .macro save_and_disable_irq, flags |
45 | mrs \flags, daif | |
f0098155 | 46 | msr daifset, #3 |
4b65a5db CM |
47 | .endm |
48 | ||
49 | .macro restore_irq, flags | |
50 | msr daif, \flags | |
51 | .endm | |
52 | ||
2a283070 WD |
53 | .macro disable_step_tsk, flgs, tmp |
54 | tbz \flgs, #TIF_SINGLESTEP, 9990f | |
0be7320a | 55 | mrs \tmp, mdscr_el1 |
e28cc025 | 56 | bic \tmp, \tmp, #DBG_MDSCR_SS |
0be7320a | 57 | msr mdscr_el1, \tmp |
3a2d2ca4 | 58 | isb // Take effect before a subsequent clear of DAIF.D |
2a283070 | 59 | 9990: |
0be7320a CM |
60 | .endm |
61 | ||
84d0fb1b | 62 | /* call with daif masked */ |
2a283070 WD |
63 | .macro enable_step_tsk, flgs, tmp |
64 | tbz \flgs, #TIF_SINGLESTEP, 9990f | |
0be7320a | 65 | mrs \tmp, mdscr_el1 |
e28cc025 | 66 | orr \tmp, \tmp, #DBG_MDSCR_SS |
0be7320a | 67 | msr mdscr_el1, \tmp |
2a283070 | 68 | 9990: |
0be7320a CM |
69 | .endm |
70 | ||
68ddbf09 JM |
71 | /* |
72 | * RAS Error Synchronization barrier | |
73 | */ | |
74 | .macro esb | |
2b68a2a9 | 75 | #ifdef CONFIG_ARM64_RAS_EXTN |
68ddbf09 | 76 | hint #16 |
2b68a2a9 JM |
77 | #else |
78 | nop | |
79 | #endif | |
68ddbf09 JM |
80 | .endm |
81 | ||
669474e7 WD |
82 | /* |
83 | * Value prediction barrier | |
84 | */ | |
85 | .macro csdb | |
86 | hint #20 | |
87 | .endm | |
88 | ||
228a26b9 JM |
89 | /* |
90 | * Clear Branch History instruction | |
91 | */ | |
92 | .macro clearbhb | |
93 | hint #22 | |
94 | .endm | |
95 | ||
bd4fb6d2 WD |
96 | /* |
97 | * Speculation barrier | |
98 | */ | |
99 | .macro sb | |
100 | alternative_if_not ARM64_HAS_SB | |
101 | dsb nsh | |
102 | isb | |
103 | alternative_else | |
104 | SB_BARRIER_INSN | |
105 | nop | |
106 | alternative_endif | |
107 | .endm | |
108 | ||
f99a250c WD |
109 | /* |
110 | * NOP sequence | |
111 | */ | |
112 | .macro nops, num | |
113 | .rept \num | |
114 | nop | |
115 | .endr | |
116 | .endm | |
117 | ||
0be7320a CM |
118 | /* |
119 | * Register aliases. | |
120 | */ | |
121 | lr .req x30 // link register | |
dc637f1f MZ |
122 | |
123 | /* | |
124 | * Vector entry | |
125 | */ | |
126 | .macro ventry label | |
127 | .align 7 | |
128 | b \label | |
129 | .endm | |
e68bedaa ML |
130 | |
131 | /* | |
132 | * Select code when configured for BE. | |
133 | */ | |
134 | #ifdef CONFIG_CPU_BIG_ENDIAN | |
135 | #define CPU_BE(code...) code | |
136 | #else | |
137 | #define CPU_BE(code...) | |
138 | #endif | |
139 | ||
140 | /* | |
141 | * Select code when configured for LE. | |
142 | */ | |
143 | #ifdef CONFIG_CPU_BIG_ENDIAN | |
144 | #define CPU_LE(code...) | |
145 | #else | |
146 | #define CPU_LE(code...) code | |
147 | #endif | |
148 | ||
55b89540 ML |
149 | /* |
150 | * Define a macro that constructs a 64-bit value by concatenating two | |
151 | * 32-bit registers. Note that on big endian systems the order of the | |
152 | * registers is swapped. | |
153 | */ | |
154 | #ifndef CONFIG_CPU_BIG_ENDIAN | |
155 | .macro regs_to_64, rd, lbits, hbits | |
156 | #else | |
157 | .macro regs_to_64, rd, hbits, lbits | |
158 | #endif | |
159 | orr \rd, \lbits, \hbits, lsl #32 | |
160 | .endm | |
f3e39273 | 161 | |
b784a5d9 AB |
162 | /* |
163 | * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where | |
350e1dad | 164 | * <symbol> is within the range +/- 4 GB of the PC. |
b784a5d9 AB |
165 | */ |
166 | /* | |
167 | * @dst: destination register (64 bit wide) | |
168 | * @sym: name of the symbol | |
b784a5d9 | 169 | */ |
41c066f2 | 170 | .macro adr_l, dst, sym |
b784a5d9 AB |
171 | adrp \dst, \sym |
172 | add \dst, \dst, :lo12:\sym | |
b784a5d9 AB |
173 | .endm |
174 | ||
175 | /* | |
176 | * @dst: destination register (32 or 64 bit wide) | |
177 | * @sym: name of the symbol | |
178 | * @tmp: optional 64-bit scratch register to be used if <dst> is a | |
179 | * 32-bit wide register, in which case it cannot be used to hold | |
180 | * the address | |
181 | */ | |
182 | .macro ldr_l, dst, sym, tmp= | |
183 | .ifb \tmp | |
184 | adrp \dst, \sym | |
185 | ldr \dst, [\dst, :lo12:\sym] | |
186 | .else | |
187 | adrp \tmp, \sym | |
188 | ldr \dst, [\tmp, :lo12:\sym] | |
189 | .endif | |
190 | .endm | |
191 | ||
192 | /* | |
193 | * @src: source register (32 or 64 bit wide) | |
194 | * @sym: name of the symbol | |
195 | * @tmp: mandatory 64-bit scratch register to calculate the address | |
196 | * while <src> needs to be preserved. | |
197 | */ | |
198 | .macro str_l, src, sym, tmp | |
199 | adrp \tmp, \sym | |
200 | str \src, [\tmp, :lo12:\sym] | |
201 | .endm | |
202 | ||
ea391027 DB |
203 | /* |
204 | * @dst: destination register | |
205 | */ | |
206 | #if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__) | |
e176e267 | 207 | .macro get_this_cpu_offset, dst |
ea391027 DB |
208 | mrs \dst, tpidr_el2 |
209 | .endm | |
210 | #else | |
e176e267 | 211 | .macro get_this_cpu_offset, dst |
ea391027 DB |
212 | alternative_if_not ARM64_HAS_VIRT_HOST_EXTN |
213 | mrs \dst, tpidr_el1 | |
214 | alternative_else | |
215 | mrs \dst, tpidr_el2 | |
e176e267 MR |
216 | alternative_endif |
217 | .endm | |
218 | ||
219 | .macro set_this_cpu_offset, src | |
220 | alternative_if_not ARM64_HAS_VIRT_HOST_EXTN | |
221 | msr tpidr_el1, \src | |
222 | alternative_else | |
223 | msr tpidr_el2, \src | |
ea391027 DB |
224 | alternative_endif |
225 | .endm | |
226 | #endif | |
227 | ||
aa4d5d3c | 228 | /* |
350e1dad | 229 | * @dst: Result of per_cpu(sym, smp_processor_id()) (can be SP) |
aa4d5d3c | 230 | * @sym: The name of the per-cpu variable |
aa4d5d3c JM |
231 | * @tmp: scratch register |
232 | */ | |
1b7e2296 | 233 | .macro adr_this_cpu, dst, sym, tmp |
8ea41b11 AB |
234 | adrp \tmp, \sym |
235 | add \dst, \tmp, #:lo12:\sym | |
e176e267 | 236 | get_this_cpu_offset \tmp |
1b7e2296 MR |
237 | add \dst, \dst, \tmp |
238 | .endm | |
239 | ||
240 | /* | |
241 | * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id())) | |
242 | * @sym: The name of the per-cpu variable | |
243 | * @tmp: scratch register | |
244 | */ | |
245 | .macro ldr_this_cpu dst, sym, tmp | |
246 | adr_l \dst, \sym | |
e176e267 | 247 | get_this_cpu_offset \tmp |
1b7e2296 | 248 | ldr \dst, [\dst, \tmp] |
aa4d5d3c JM |
249 | .endm |
250 | ||
7b7293ae GL |
251 | /* |
252 | * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm) | |
253 | */ | |
254 | .macro vma_vm_mm, rd, rn | |
255 | ldr \rd, [\rn, #VMA_VM_MM] | |
256 | .endm | |
257 | ||
116c81f4 | 258 | /* |
880f7cc4 WD |
259 | * read_ctr - read CTR_EL0. If the system has mismatched register fields, |
260 | * provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val | |
116c81f4 SP |
261 | */ |
262 | .macro read_ctr, reg | |
755db234 | 263 | #ifndef __KVM_NVHE_HYPERVISOR__ |
880f7cc4 | 264 | alternative_if_not ARM64_MISMATCHED_CACHE_TYPE |
116c81f4 SP |
265 | mrs \reg, ctr_el0 // read CTR |
266 | nop | |
267 | alternative_else | |
268 | ldr_l \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL | |
269 | alternative_endif | |
755db234 MZ |
270 | #else |
271 | alternative_if_not ARM64_KVM_PROTECTED_MODE | |
272 | ASM_BUG() | |
273 | alternative_else_nop_endif | |
4c0bd995 | 274 | alternative_cb ARM64_ALWAYS_SYSTEM, kvm_compute_final_ctr_el0 |
755db234 MZ |
275 | movz \reg, #0 |
276 | movk \reg, #0, lsl #16 | |
277 | movk \reg, #0, lsl #32 | |
278 | movk \reg, #0, lsl #48 | |
279 | alternative_cb_end | |
280 | #endif | |
116c81f4 SP |
281 | .endm |
282 | ||
7b7293ae GL |
283 | |
284 | /* | |
072f0a63 SP |
285 | * raw_dcache_line_size - get the minimum D-cache line size on this CPU |
286 | * from the CTR register. | |
7b7293ae | 287 | */ |
072f0a63 | 288 | .macro raw_dcache_line_size, reg, tmp |
7b7293ae GL |
289 | mrs \tmp, ctr_el0 // read CTR |
290 | ubfm \tmp, \tmp, #16, #19 // cache line size encoding | |
291 | mov \reg, #4 // bytes per word | |
292 | lsl \reg, \reg, \tmp // actual cache line size | |
293 | .endm | |
294 | ||
295 | /* | |
072f0a63 | 296 | * dcache_line_size - get the safe D-cache line size across all CPUs |
7b7293ae | 297 | */ |
072f0a63 | 298 | .macro dcache_line_size, reg, tmp |
116c81f4 SP |
299 | read_ctr \tmp |
300 | ubfm \tmp, \tmp, #16, #19 // cache line size encoding | |
301 | mov \reg, #4 // bytes per word | |
302 | lsl \reg, \reg, \tmp // actual cache line size | |
072f0a63 SP |
303 | .endm |
304 | ||
305 | /* | |
306 | * raw_icache_line_size - get the minimum I-cache line size on this CPU | |
307 | * from the CTR register. | |
308 | */ | |
309 | .macro raw_icache_line_size, reg, tmp | |
7b7293ae GL |
310 | mrs \tmp, ctr_el0 // read CTR |
311 | and \tmp, \tmp, #0xf // cache line size encoding | |
312 | mov \reg, #4 // bytes per word | |
313 | lsl \reg, \reg, \tmp // actual cache line size | |
314 | .endm | |
315 | ||
072f0a63 SP |
316 | /* |
317 | * icache_line_size - get the safe I-cache line size across all CPUs | |
318 | */ | |
319 | .macro icache_line_size, reg, tmp | |
116c81f4 SP |
320 | read_ctr \tmp |
321 | and \tmp, \tmp, #0xf // cache line size encoding | |
322 | mov \reg, #4 // bytes per word | |
323 | lsl \reg, \reg, \tmp // actual cache line size | |
072f0a63 SP |
324 | .endm |
325 | ||
7b7293ae | 326 | /* |
67e7fdfc | 327 | * tcr_set_t0sz - update TCR.T0SZ so that we can load the ID map |
7b7293ae | 328 | */ |
67e7fdfc SC |
329 | .macro tcr_set_t0sz, valreg, t0sz |
330 | bfi \valreg, \t0sz, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH | |
7b7293ae GL |
331 | .endm |
332 | ||
b6d00d47 SC |
333 | /* |
334 | * tcr_set_t1sz - update TCR.T1SZ | |
335 | */ | |
336 | .macro tcr_set_t1sz, valreg, t1sz | |
337 | bfi \valreg, \t1sz, #TCR_T1SZ_OFFSET, #TCR_TxSZ_WIDTH | |
338 | .endm | |
339 | ||
787fd1d0 KM |
340 | /* |
341 | * tcr_compute_pa_size - set TCR.(I)PS to the highest supported | |
342 | * ID_AA64MMFR0_EL1.PARange value | |
343 | * | |
344 | * tcr: register with the TCR_ELx value to be updated | |
39610a68 | 345 | * pos: IPS or PS bitfield position |
787fd1d0 KM |
346 | * tmp{0,1}: temporary registers |
347 | */ | |
348 | .macro tcr_compute_pa_size, tcr, pos, tmp0, tmp1 | |
349 | mrs \tmp0, ID_AA64MMFR0_EL1 | |
350 | // Narrow PARange to fit the PS field in TCR_ELx | |
2d987e64 MB |
351 | ubfx \tmp0, \tmp0, #ID_AA64MMFR0_EL1_PARANGE_SHIFT, #3 |
352 | mov \tmp1, #ID_AA64MMFR0_EL1_PARANGE_MAX | |
787fd1d0 KM |
353 | cmp \tmp0, \tmp1 |
354 | csel \tmp0, \tmp1, \tmp0, hi | |
355 | bfi \tcr, \tmp0, \pos, #3 | |
7b7293ae GL |
356 | .endm |
357 | ||
06b7a568 FT |
358 | .macro __dcache_op_workaround_clean_cache, op, addr |
359 | alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE | |
360 | dc \op, \addr | |
361 | alternative_else | |
362 | dc civac, \addr | |
363 | alternative_endif | |
364 | .endm | |
365 | ||
7b7293ae GL |
366 | /* |
367 | * Macro to perform a data cache maintenance for the interval | |
3036ec59 | 368 | * [start, end) with dcache line size explicitly provided. |
7b7293ae GL |
369 | * |
370 | * op: operation passed to dc instruction | |
371 | * domain: domain used in dsb instruciton | |
163d3f80 FT |
372 | * start: starting virtual address of the region |
373 | * end: end virtual address of the region | |
3036ec59 | 374 | * linesz: dcache line size |
d11b1877 | 375 | * fixup: optional label to branch to on user fault |
3036ec59 | 376 | * Corrupts: start, end, tmp |
7b7293ae | 377 | */ |
3036ec59 PT |
378 | .macro dcache_by_myline_op op, domain, start, end, linesz, tmp, fixup |
379 | sub \tmp, \linesz, #1 | |
380 | bic \start, \start, \tmp | |
d11b1877 | 381 | .Ldcache_op\@: |
33309ecd | 382 | .ifc \op, cvau |
163d3f80 | 383 | __dcache_op_workaround_clean_cache \op, \start |
33309ecd WD |
384 | .else |
385 | .ifc \op, cvac | |
163d3f80 | 386 | __dcache_op_workaround_clean_cache \op, \start |
33309ecd WD |
387 | .else |
388 | .ifc \op, cvap | |
163d3f80 | 389 | sys 3, c7, c12, 1, \start // dc cvap |
823066d9 | 390 | .else |
04a1438e | 391 | .ifc \op, cvadp |
163d3f80 | 392 | sys 3, c7, c13, 1, \start // dc cvadp |
04a1438e | 393 | .else |
163d3f80 | 394 | dc \op, \start |
823066d9 | 395 | .endif |
33309ecd WD |
396 | .endif |
397 | .endif | |
04a1438e | 398 | .endif |
3036ec59 | 399 | add \start, \start, \linesz |
163d3f80 | 400 | cmp \start, \end |
d11b1877 | 401 | b.lo .Ldcache_op\@ |
7b7293ae | 402 | dsb \domain |
d11b1877 | 403 | |
e4208e80 | 404 | _cond_uaccess_extable .Ldcache_op\@, \fixup |
7b7293ae GL |
405 | .endm |
406 | ||
3036ec59 PT |
407 | /* |
408 | * Macro to perform a data cache maintenance for the interval | |
409 | * [start, end) | |
410 | * | |
411 | * op: operation passed to dc instruction | |
412 | * domain: domain used in dsb instruciton | |
413 | * start: starting virtual address of the region | |
414 | * end: end virtual address of the region | |
415 | * fixup: optional label to branch to on user fault | |
416 | * Corrupts: start, end, tmp1, tmp2 | |
417 | */ | |
418 | .macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup | |
419 | dcache_line_size \tmp1, \tmp2 | |
420 | dcache_by_myline_op \op, \domain, \start, \end, \tmp1, \tmp2, \fixup | |
421 | .endm | |
422 | ||
4fee9473 MZ |
423 | /* |
424 | * Macro to perform an instruction cache maintenance for the interval | |
425 | * [start, end) | |
426 | * | |
427 | * start, end: virtual addresses describing the region | |
d11b1877 | 428 | * fixup: optional label to branch to on user fault |
4fee9473 MZ |
429 | * Corrupts: tmp1, tmp2 |
430 | */ | |
d11b1877 | 431 | .macro invalidate_icache_by_line start, end, tmp1, tmp2, fixup |
4fee9473 MZ |
432 | icache_line_size \tmp1, \tmp2 |
433 | sub \tmp2, \tmp1, #1 | |
434 | bic \tmp2, \start, \tmp2 | |
d11b1877 MR |
435 | .Licache_op\@: |
436 | ic ivau, \tmp2 // invalidate I line PoU | |
4fee9473 MZ |
437 | add \tmp2, \tmp2, \tmp1 |
438 | cmp \tmp2, \end | |
d11b1877 | 439 | b.lo .Licache_op\@ |
4fee9473 MZ |
440 | dsb ish |
441 | isb | |
d11b1877 | 442 | |
e4208e80 | 443 | _cond_uaccess_extable .Licache_op\@, \fixup |
4fee9473 MZ |
444 | .endm |
445 | ||
c0be8f18 AB |
446 | /* |
447 | * load_ttbr1 - install @pgtbl as a TTBR1 page table | |
448 | * pgtbl preserved | |
449 | * tmp1/tmp2 clobbered, either may overlap with pgtbl | |
450 | */ | |
451 | .macro load_ttbr1, pgtbl, tmp1, tmp2 | |
452 | phys_to_ttbr \tmp1, \pgtbl | |
453 | offset_ttbr1 \tmp1, \tmp2 | |
454 | msr ttbr1_el1, \tmp1 | |
455 | isb | |
456 | .endm | |
457 | ||
3744b528 PT |
458 | /* |
459 | * To prevent the possibility of old and new partial table walks being visible | |
460 | * in the tlb, switch the ttbr to a zero page when we invalidate the old | |
461 | * records. D4.7.1 'General TLB maintenance requirements' in ARM DDI 0487A.i | |
462 | * Even switching to our copied tables will cause a changed output address at | |
463 | * each stage of the walk. | |
464 | */ | |
465 | .macro break_before_make_ttbr_switch zero_page, page_table, tmp, tmp2 | |
466 | phys_to_ttbr \tmp, \zero_page | |
467 | msr ttbr1_el1, \tmp | |
468 | isb | |
469 | tlbi vmalle1 | |
470 | dsb nsh | |
c0be8f18 | 471 | load_ttbr1 \page_table, \tmp, \tmp2 |
3744b528 PT |
472 | .endm |
473 | ||
7b7293ae GL |
474 | /* |
475 | * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present | |
476 | */ | |
477 | .macro reset_pmuserenr_el0, tmpreg | |
f6e56435 | 478 | mrs \tmpreg, id_aa64dfr0_el1 |
b782e8d0 YY |
479 | ubfx \tmpreg, \tmpreg, #ID_AA64DFR0_EL1_PMUVer_SHIFT, #4 |
480 | cmp \tmpreg, #ID_AA64DFR0_EL1_PMUVer_NI | |
481 | ccmp \tmpreg, #ID_AA64DFR0_EL1_PMUVer_IMP_DEF, #4, ne | |
482 | b.eq 9000f // Skip if no PMU present or IMP_DEF | |
7b7293ae GL |
483 | msr pmuserenr_el0, xzr // Disable PMU access from EL0 |
484 | 9000: | |
485 | .endm | |
486 | ||
87a1f063 IV |
487 | /* |
488 | * reset_amuserenr_el0 - reset AMUSERENR_EL0 if AMUv1 present | |
489 | */ | |
490 | .macro reset_amuserenr_el0, tmpreg | |
491 | mrs \tmpreg, id_aa64pfr0_el1 // Check ID_AA64PFR0_EL1 | |
55adc08d | 492 | ubfx \tmpreg, \tmpreg, #ID_AA64PFR0_EL1_AMU_SHIFT, #4 |
87a1f063 IV |
493 | cbz \tmpreg, .Lskip_\@ // Skip if no AMU present |
494 | msr_s SYS_AMUSERENR_EL0, xzr // Disable AMU access from EL0 | |
495 | .Lskip_\@: | |
496 | .endm | |
5003dbde GL |
497 | /* |
498 | * copy_page - copy src to dest using temp registers t1-t8 | |
499 | */ | |
500 | .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req | |
501 | 9998: ldp \t1, \t2, [\src] | |
502 | ldp \t3, \t4, [\src, #16] | |
503 | ldp \t5, \t6, [\src, #32] | |
504 | ldp \t7, \t8, [\src, #48] | |
505 | add \src, \src, #64 | |
506 | stnp \t1, \t2, [\dest] | |
507 | stnp \t3, \t4, [\dest, #16] | |
508 | stnp \t5, \t6, [\dest, #32] | |
509 | stnp \t7, \t8, [\dest, #48] | |
510 | add \dest, \dest, #64 | |
511 | tst \src, #(PAGE_SIZE - 1) | |
512 | b.ne 9998b | |
513 | .endm | |
514 | ||
ed84b4e9 MR |
515 | /* |
516 | * Annotate a function as being unsuitable for kprobes. | |
517 | */ | |
518 | #ifdef CONFIG_KPROBES | |
519 | #define NOKPROBE(x) \ | |
520 | .pushsection "_kprobe_blacklist", "aw"; \ | |
521 | .quad x; \ | |
522 | .popsection; | |
523 | #else | |
524 | #define NOKPROBE(x) | |
525 | #endif | |
386b3c7b | 526 | |
0fea6e9a | 527 | #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) |
386b3c7b MR |
528 | #define EXPORT_SYMBOL_NOKASAN(name) |
529 | #else | |
530 | #define EXPORT_SYMBOL_NOKASAN(name) EXPORT_SYMBOL(name) | |
531 | #endif | |
532 | ||
6ad1fe5d AB |
533 | /* |
534 | * Emit a 64-bit absolute little endian symbol reference in a way that | |
535 | * ensures that it will be resolved at build time, even when building a | |
536 | * PIE binary. This requires cooperation from the linker script, which | |
537 | * must emit the lo32/hi32 halves individually. | |
538 | */ | |
539 | .macro le64sym, sym | |
540 | .long \sym\()_lo32 | |
541 | .long \sym\()_hi32 | |
542 | .endm | |
543 | ||
30b5ba5c AB |
544 | /* |
545 | * mov_q - move an immediate constant into a 64-bit register using | |
546 | * between 2 and 4 movz/movk instructions (depending on the | |
547 | * magnitude and sign of the operand) | |
548 | */ | |
549 | .macro mov_q, reg, val | |
550 | .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff) | |
551 | movz \reg, :abs_g1_s:\val | |
552 | .else | |
553 | .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff) | |
554 | movz \reg, :abs_g2_s:\val | |
555 | .else | |
556 | movz \reg, :abs_g3:\val | |
557 | movk \reg, :abs_g2_nc:\val | |
558 | .endif | |
559 | movk \reg, :abs_g1_nc:\val | |
560 | .endif | |
561 | movk \reg, :abs_g0_nc:\val | |
562 | .endm | |
563 | ||
4b65a5db | 564 | /* |
4caf8758 | 565 | * Return the current task_struct. |
4b65a5db | 566 | */ |
4caf8758 | 567 | .macro get_current_task, rd |
4b65a5db CM |
568 | mrs \rd, sp_el0 |
569 | .endm | |
570 | ||
e842dfb5 | 571 | /* |
68aec33f AB |
572 | * If the kernel is built for 52-bit virtual addressing but the hardware only |
573 | * supports 48 bits, we cannot program the pgdir address into TTBR1 directly, | |
574 | * but we have to add an offset so that the TTBR1 address corresponds with the | |
575 | * pgdir entry that covers the lowest 48-bit addressable VA. | |
576 | * | |
9684ec18 AB |
577 | * Note that this trick is only used for LVA/64k pages - LPA2/4k pages uses an |
578 | * additional paging level, and on LPA2/16k pages, we would end up with a root | |
579 | * level table with only 2 entries, which is suboptimal in terms of TLB | |
580 | * utilization, so there we fall back to 47 bits of translation if LPA2 is not | |
581 | * supported. | |
582 | * | |
e842dfb5 | 583 | * orr is used as it can cover the immediate value (and is idempotent). |
e842dfb5 SC |
584 | * ttbr: Value of ttbr to set, modified. |
585 | */ | |
c812026c | 586 | .macro offset_ttbr1, ttbr, tmp |
9684ec18 | 587 | #if defined(CONFIG_ARM64_VA_BITS_52) && !defined(CONFIG_ARM64_LPA2) |
68aec33f AB |
588 | mrs \tmp, tcr_el1 |
589 | and \tmp, \tmp, #TCR_T1SZ_MASK | |
590 | cmp \tmp, #TCR_T1SZ(VA_BITS_MIN) | |
591 | orr \tmp, \ttbr, #TTBR1_BADDR_4852_OFFSET | |
592 | csel \ttbr, \tmp, \ttbr, eq | |
e842dfb5 SC |
593 | #endif |
594 | .endm | |
595 | ||
529c4b05 KM |
596 | /* |
597 | * Arrange a physical address in a TTBR register, taking care of 52-bit | |
598 | * addresses. | |
599 | * | |
600 | * phys: physical address, preserved | |
601 | * ttbr: returns the TTBR value | |
602 | */ | |
fa0465fc | 603 | .macro phys_to_ttbr, ttbr, phys |
529c4b05 KM |
604 | #ifdef CONFIG_ARM64_PA_BITS_52 |
605 | orr \ttbr, \phys, \phys, lsr #46 | |
606 | and \ttbr, \ttbr, #TTBR_BADDR_MASK_52 | |
607 | #else | |
608 | mov \ttbr, \phys | |
609 | #endif | |
610 | .endm | |
611 | ||
79ddab3b WD |
612 | .macro phys_to_pte, pte, phys |
613 | #ifdef CONFIG_ARM64_PA_BITS_52 | |
925a0eb4 AB |
614 | orr \pte, \phys, \phys, lsr #PTE_ADDR_HIGH_SHIFT |
615 | and \pte, \pte, #PHYS_TO_PTE_ADDR_MASK | |
79ddab3b WD |
616 | #else |
617 | mov \pte, \phys | |
618 | #endif | |
619 | .endm | |
620 | ||
3e32131a ZL |
621 | /* |
622 | * tcr_clear_errata_bits - Clear TCR bits that trigger an errata on this CPU. | |
623 | */ | |
624 | .macro tcr_clear_errata_bits, tcr, tmp1, tmp2 | |
625 | #ifdef CONFIG_FUJITSU_ERRATUM_010001 | |
626 | mrs \tmp1, midr_el1 | |
627 | ||
628 | mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001_MASK | |
629 | and \tmp1, \tmp1, \tmp2 | |
630 | mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001 | |
631 | cmp \tmp1, \tmp2 | |
632 | b.ne 10f | |
633 | ||
634 | mov_q \tmp2, TCR_CLEAR_FUJITSU_ERRATUM_010001 | |
635 | bic \tcr, \tcr, \tmp2 | |
636 | 10: | |
637 | #endif /* CONFIG_FUJITSU_ERRATUM_010001 */ | |
638 | .endm | |
639 | ||
3060e9f0 SD |
640 | /** |
641 | * Errata workaround prior to disable MMU. Insert an ISB immediately prior | |
642 | * to executing the MSR that will change SCTLR_ELn[M] from a value of 1 to 0. | |
643 | */ | |
644 | .macro pre_disable_mmu_workaround | |
645 | #ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1041 | |
646 | isb | |
647 | #endif | |
648 | .endm | |
649 | ||
0f468e22 AB |
650 | /* |
651 | * frame_push - Push @regcount callee saved registers to the stack, | |
652 | * starting at x19, as well as x29/x30, and set x29 to | |
653 | * the new value of sp. Add @extra bytes of stack space | |
654 | * for locals. | |
655 | */ | |
656 | .macro frame_push, regcount:req, extra | |
657 | __frame st, \regcount, \extra | |
658 | .endm | |
659 | ||
660 | /* | |
661 | * frame_pop - Pop the callee saved registers from the stack that were | |
662 | * pushed in the most recent call to frame_push, as well | |
663 | * as x29/x30 and any extra stack space that may have been | |
664 | * allocated. | |
665 | */ | |
666 | .macro frame_pop | |
667 | __frame ld | |
668 | .endm | |
669 | ||
670 | .macro __frame_regs, reg1, reg2, op, num | |
671 | .if .Lframe_regcount == \num | |
672 | \op\()r \reg1, [sp, #(\num + 1) * 8] | |
673 | .elseif .Lframe_regcount > \num | |
674 | \op\()p \reg1, \reg2, [sp, #(\num + 1) * 8] | |
675 | .endif | |
676 | .endm | |
677 | ||
678 | .macro __frame, op, regcount, extra=0 | |
679 | .ifc \op, st | |
680 | .if (\regcount) < 0 || (\regcount) > 10 | |
681 | .error "regcount should be in the range [0 ... 10]" | |
682 | .endif | |
683 | .if ((\extra) % 16) != 0 | |
684 | .error "extra should be a multiple of 16 bytes" | |
685 | .endif | |
686 | .ifdef .Lframe_regcount | |
687 | .if .Lframe_regcount != -1 | |
688 | .error "frame_push/frame_pop may not be nested" | |
689 | .endif | |
690 | .endif | |
691 | .set .Lframe_regcount, \regcount | |
692 | .set .Lframe_extra, \extra | |
693 | .set .Lframe_local_offset, ((\regcount + 3) / 2) * 16 | |
694 | stp x29, x30, [sp, #-.Lframe_local_offset - .Lframe_extra]! | |
695 | mov x29, sp | |
696 | .endif | |
697 | ||
698 | __frame_regs x19, x20, \op, 1 | |
699 | __frame_regs x21, x22, \op, 3 | |
700 | __frame_regs x23, x24, \op, 5 | |
701 | __frame_regs x25, x26, \op, 7 | |
702 | __frame_regs x27, x28, \op, 9 | |
703 | ||
704 | .ifc \op, ld | |
705 | .if .Lframe_regcount == -1 | |
706 | .error "frame_push/frame_pop may not be nested" | |
707 | .endif | |
708 | ldp x29, x30, [sp], #.Lframe_local_offset + .Lframe_extra | |
709 | .set .Lframe_regcount, -1 | |
710 | .endif | |
711 | .endm | |
712 | ||
8cc8a324 | 713 | /* |
8f4de66e | 714 | * Set SCTLR_ELx to the @reg value, and invalidate the local icache |
8cc8a324 MZ |
715 | * in the process. This is called when setting the MMU on. |
716 | */ | |
8f4de66e QP |
717 | .macro set_sctlr, sreg, reg |
718 | msr \sreg, \reg | |
8cc8a324 MZ |
719 | isb |
720 | /* | |
721 | * Invalidate the local I-cache so that any instructions fetched | |
722 | * speculatively from the PoC are discarded, since they may have | |
723 | * been dynamically patched at the PoU. | |
724 | */ | |
725 | ic iallu | |
726 | dsb nsh | |
727 | isb | |
728 | .endm | |
729 | ||
8f4de66e QP |
730 | .macro set_sctlr_el1, reg |
731 | set_sctlr sctlr_el1, \reg | |
732 | .endm | |
733 | ||
734 | .macro set_sctlr_el2, reg | |
735 | set_sctlr sctlr_el2, \reg | |
736 | .endm | |
737 | ||
d13c613f | 738 | /* |
3931261e AB |
739 | * Check whether asm code should yield as soon as it is able. This is |
740 | * the case if we are currently running in task context, and the | |
741 | * TIF_NEED_RESCHED flag is set. (Note that the TIF_NEED_RESCHED flag | |
742 | * is stored negated in the top word of the thread_info::preempt_count | |
13150149 | 743 | * field) |
d13c613f | 744 | */ |
3931261e AB |
745 | .macro cond_yield, lbl:req, tmp:req, tmp2 |
746 | #ifdef CONFIG_PREEMPT_VOLUNTARY | |
d13c613f AB |
747 | get_current_task \tmp |
748 | ldr \tmp, [\tmp, #TSK_TI_PREEMPT] | |
13150149 AB |
749 | /* |
750 | * If we are serving a softirq, there is no point in yielding: the | |
751 | * softirq will not be preempted no matter what we do, so we should | |
3931261e AB |
752 | * run to completion as quickly as we can. The preempt_count field will |
753 | * have BIT(SOFTIRQ_SHIFT) set in this case, so the zero check will | |
754 | * catch this case too. | |
13150149 | 755 | */ |
d13c613f AB |
756 | cbz \tmp, \lbl |
757 | #endif | |
758 | .endm | |
759 | ||
9be34be8 MB |
760 | /* |
761 | * Branch Target Identifier (BTI) | |
762 | */ | |
763 | .macro bti, targets | |
764 | .equ .L__bti_targets_c, 34 | |
765 | .equ .L__bti_targets_j, 36 | |
766 | .equ .L__bti_targets_jc,38 | |
767 | hint #.L__bti_targets_\targets | |
768 | .endm | |
769 | ||
3a9b136c MB |
770 | /* |
771 | * This macro emits a program property note section identifying | |
772 | * architecture features which require special handling, mainly for | |
773 | * use in assembly files included in the VDSO. | |
774 | */ | |
775 | ||
776 | #define NT_GNU_PROPERTY_TYPE_0 5 | |
777 | #define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000 | |
778 | ||
779 | #define GNU_PROPERTY_AARCH64_FEATURE_1_BTI (1U << 0) | |
780 | #define GNU_PROPERTY_AARCH64_FEATURE_1_PAC (1U << 1) | |
781 | ||
782 | #ifdef CONFIG_ARM64_BTI_KERNEL | |
783 | #define GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT \ | |
784 | ((GNU_PROPERTY_AARCH64_FEATURE_1_BTI | \ | |
785 | GNU_PROPERTY_AARCH64_FEATURE_1_PAC)) | |
786 | #endif | |
787 | ||
788 | #ifdef GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT | |
789 | .macro emit_aarch64_feature_1_and, feat=GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT | |
790 | .pushsection .note.gnu.property, "a" | |
791 | .align 3 | |
792 | .long 2f - 1f | |
793 | .long 6f - 3f | |
794 | .long NT_GNU_PROPERTY_TYPE_0 | |
795 | 1: .string "GNU" | |
796 | 2: | |
797 | .align 3 | |
798 | 3: .long GNU_PROPERTY_AARCH64_FEATURE_1_AND | |
799 | .long 5f - 4f | |
800 | 4: | |
801 | /* | |
802 | * This is described with an array of char in the Linux API | |
803 | * spec but the text and all other usage (including binutils, | |
804 | * clang and GCC) treat this as a 32 bit value so no swizzling | |
805 | * is required for big endian. | |
806 | */ | |
807 | .long \feat | |
808 | 5: | |
809 | .align 3 | |
810 | 6: | |
811 | .popsection | |
812 | .endm | |
813 | ||
814 | #else | |
815 | .macro emit_aarch64_feature_1_and, feat=0 | |
816 | .endm | |
817 | ||
818 | #endif /* GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT */ | |
819 | ||
ba268923 JM |
820 | .macro __mitigate_spectre_bhb_loop tmp |
821 | #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY | |
4c0bd995 | 822 | alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_loop_iter |
558c303c JM |
823 | mov \tmp, #32 // Patched to correct the immediate |
824 | alternative_cb_end | |
ba268923 JM |
825 | .Lspectre_bhb_loop\@: |
826 | b . + 4 | |
827 | subs \tmp, \tmp, #1 | |
828 | b.ne .Lspectre_bhb_loop\@ | |
829 | sb | |
830 | #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ | |
831 | .endm | |
832 | ||
558c303c JM |
833 | .macro mitigate_spectre_bhb_loop tmp |
834 | #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY | |
4c0bd995 | 835 | alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_loop_mitigation_enable |
558c303c JM |
836 | b .L_spectre_bhb_loop_done\@ // Patched to NOP |
837 | alternative_cb_end | |
838 | __mitigate_spectre_bhb_loop \tmp | |
839 | .L_spectre_bhb_loop_done\@: | |
840 | #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ | |
841 | .endm | |
842 | ||
ba268923 JM |
843 | /* Save/restores x0-x3 to the stack */ |
844 | .macro __mitigate_spectre_bhb_fw | |
845 | #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY | |
846 | stp x0, x1, [sp, #-16]! | |
847 | stp x2, x3, [sp, #-16]! | |
848 | mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3 | |
4c0bd995 | 849 | alternative_cb ARM64_ALWAYS_SYSTEM, smccc_patch_fw_mitigation_conduit |
ba268923 JM |
850 | nop // Patched to SMC/HVC #0 |
851 | alternative_cb_end | |
852 | ldp x2, x3, [sp], #16 | |
853 | ldp x0, x1, [sp], #16 | |
228a26b9 JM |
854 | #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ |
855 | .endm | |
856 | ||
857 | .macro mitigate_spectre_bhb_clear_insn | |
858 | #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY | |
4c0bd995 | 859 | alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_clearbhb |
228a26b9 JM |
860 | /* Patched to NOP when not supported */ |
861 | clearbhb | |
862 | isb | |
863 | alternative_cb_end | |
ba268923 JM |
864 | #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ |
865 | .endm | |
f3e39273 | 866 | #endif /* __ASM_ASSEMBLER_H */ |