Commit | Line | Data |
---|---|---|
9cce7a43 CM |
1 | /* |
2 | * Based on arch/arm/mm/proc.S | |
3 | * | |
4 | * Copyright (C) 2001 Deep Blue Solutions Ltd. | |
5 | * Copyright (C) 2012 ARM Ltd. | |
6 | * Author: Catalin Marinas <catalin.marinas@arm.com> | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or modify | |
9 | * it under the terms of the GNU General Public License version 2 as | |
10 | * published by the Free Software Foundation. | |
11 | * | |
12 | * This program is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | * GNU General Public License for more details. | |
16 | * | |
17 | * You should have received a copy of the GNU General Public License | |
18 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
19 | */ | |
20 | ||
21 | #include <linux/init.h> | |
22 | #include <linux/linkage.h> | |
23 | #include <asm/assembler.h> | |
24 | #include <asm/asm-offsets.h> | |
25 | #include <asm/hwcap.h> | |
9cce7a43 | 26 | #include <asm/pgtable.h> |
cabe1c81 | 27 | #include <asm/pgtable-hwdef.h> |
104a0c02 AP |
28 | #include <asm/cpufeature.h> |
29 | #include <asm/alternative.h> | |
9cce7a43 | 30 | |
35a86976 CM |
31 | #ifdef CONFIG_ARM64_64K_PAGES |
32 | #define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K | |
44eaacf1 SP |
33 | #elif defined(CONFIG_ARM64_16K_PAGES) |
34 | #define TCR_TG_FLAGS TCR_TG0_16K | TCR_TG1_16K | |
35 | #else /* CONFIG_ARM64_4K_PAGES */ | |
35a86976 CM |
36 | #define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K |
37 | #endif | |
38 | ||
35a86976 | 39 | #define TCR_SMP_FLAGS TCR_SHARED |
9cce7a43 | 40 | |
35a86976 CM |
41 | /* PTWs cacheable, inner/outer WBWA */ |
42 | #define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA | |
43 | ||
9cce7a43 CM |
44 | #define MAIR(attr, mt) ((attr) << ((mt) * 8)) |
45 | ||
9cce7a43 CM |
46 | /* |
47 | * cpu_do_idle() | |
48 | * | |
49 | * Idle the processor (wait for interrupt). | |
50 | */ | |
51 | ENTRY(cpu_do_idle) | |
52 | dsb sy // WFI may enter a low-power mode | |
53 | wfi | |
54 | ret | |
55 | ENDPROC(cpu_do_idle) | |
56 | ||
af3cfdbf | 57 | #ifdef CONFIG_CPU_PM |
6732bc65 LP |
58 | /** |
59 | * cpu_do_suspend - save CPU registers context | |
60 | * | |
61 | * x0: virtual address of context pointer | |
62 | */ | |
63 | ENTRY(cpu_do_suspend) | |
64 | mrs x2, tpidr_el0 | |
65 | mrs x3, tpidrro_el0 | |
66 | mrs x4, contextidr_el1 | |
cabe1c81 JM |
67 | mrs x5, cpacr_el1 |
68 | mrs x6, tcr_el1 | |
69 | mrs x7, vbar_el1 | |
70 | mrs x8, mdscr_el1 | |
71 | mrs x9, oslsr_el1 | |
72 | mrs x10, sctlr_el1 | |
6d99b689 | 73 | alternative_if_not ARM64_HAS_VIRT_HOST_EXTN |
623b476f | 74 | mrs x11, tpidr_el1 |
6d99b689 JM |
75 | alternative_else |
76 | mrs x11, tpidr_el2 | |
77 | alternative_endif | |
623b476f | 78 | mrs x12, sp_el0 |
6732bc65 | 79 | stp x2, x3, [x0] |
cabe1c81 JM |
80 | stp x4, xzr, [x0, #16] |
81 | stp x5, x6, [x0, #32] | |
82 | stp x7, x8, [x0, #48] | |
83 | stp x9, x10, [x0, #64] | |
623b476f | 84 | stp x11, x12, [x0, #80] |
6732bc65 LP |
85 | ret |
86 | ENDPROC(cpu_do_suspend) | |
87 | ||
88 | /** | |
89 | * cpu_do_resume - restore CPU register context | |
90 | * | |
cabe1c81 | 91 | * x0: Address of context pointer |
6732bc65 | 92 | */ |
439e70e2 | 93 | .pushsection ".idmap.text", "awx" |
6732bc65 | 94 | ENTRY(cpu_do_resume) |
6732bc65 LP |
95 | ldp x2, x3, [x0] |
96 | ldp x4, x5, [x0, #16] | |
cabe1c81 JM |
97 | ldp x6, x8, [x0, #32] |
98 | ldp x9, x10, [x0, #48] | |
99 | ldp x11, x12, [x0, #64] | |
623b476f | 100 | ldp x13, x14, [x0, #80] |
6732bc65 LP |
101 | msr tpidr_el0, x2 |
102 | msr tpidrro_el0, x3 | |
103 | msr contextidr_el1, x4 | |
6732bc65 | 104 | msr cpacr_el1, x6 |
cabe1c81 JM |
105 | |
106 | /* Don't change t0sz here, mask those bits when restoring */ | |
107 | mrs x5, tcr_el1 | |
108 | bfi x8, x5, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH | |
109 | ||
6732bc65 LP |
110 | msr tcr_el1, x8 |
111 | msr vbar_el1, x9 | |
744c6c37 JM |
112 | |
113 | /* | |
114 | * __cpu_setup() cleared MDSCR_EL1.MDE and friends, before unmasking | |
115 | * debug exceptions. By restoring MDSCR_EL1 here, we may take a debug | |
0fbeb318 | 116 | * exception. Mask them until local_daif_restore() in cpu_suspend() |
744c6c37 JM |
117 | * resets them. |
118 | */ | |
0fbeb318 | 119 | disable_daif |
6732bc65 | 120 | msr mdscr_el1, x10 |
744c6c37 | 121 | |
cabe1c81 | 122 | msr sctlr_el1, x12 |
6d99b689 | 123 | alternative_if_not ARM64_HAS_VIRT_HOST_EXTN |
623b476f | 124 | msr tpidr_el1, x13 |
6d99b689 JM |
125 | alternative_else |
126 | msr tpidr_el2, x13 | |
127 | alternative_endif | |
623b476f | 128 | msr sp_el0, x14 |
6732bc65 LP |
129 | /* |
130 | * Restore oslsr_el1 by writing oslar_el1 | |
131 | */ | |
132 | ubfx x11, x11, #1, #1 | |
133 | msr oslar_el1, x11 | |
f436b2ac | 134 | reset_pmuserenr_el0 x0 // Disable PMU access from EL0 |
68ddbf09 JM |
135 | |
136 | alternative_if ARM64_HAS_RAS_EXTN | |
137 | msr_s SYS_DISR_EL1, xzr | |
138 | alternative_else_nop_endif | |
139 | ||
6732bc65 LP |
140 | isb |
141 | ret | |
142 | ENDPROC(cpu_do_resume) | |
b6113038 | 143 | .popsection |
6732bc65 LP |
144 | #endif |
145 | ||
9cce7a43 | 146 | /* |
812944e9 | 147 | * cpu_do_switch_mm(pgd_phys, tsk) |
9cce7a43 CM |
148 | * |
149 | * Set the translation table base pointer to be pgd_phys. | |
150 | * | |
151 | * - pgd_phys - physical address of new TTB | |
152 | */ | |
153 | ENTRY(cpu_do_switch_mm) | |
7655abb9 | 154 | mrs x2, ttbr1_el1 |
5aec715d | 155 | mmid x1, x1 // get mm->context.id |
fa0465fc | 156 | phys_to_ttbr x3, x0 |
6b88a32c | 157 | #ifdef CONFIG_ARM64_SW_TTBR0_PAN |
ec89ab50 | 158 | bfi x3, x1, #48, #16 // set the ASID field in TTBR0 |
6b88a32c | 159 | #endif |
7655abb9 WD |
160 | bfi x2, x1, #48, #16 // set the ASID |
161 | msr ttbr1_el1, x2 // in TTBR1 (since TCR.A1 is set) | |
162 | isb | |
ec89ab50 | 163 | msr ttbr0_el1, x3 // now update TTBR0 |
9cce7a43 | 164 | isb |
95e3de35 | 165 | b post_ttbr_update_workaround // Back to C code... |
9cce7a43 CM |
166 | ENDPROC(cpu_do_switch_mm) |
167 | ||
439e70e2 | 168 | .pushsection ".idmap.text", "awx" |
f992b4df WD |
169 | |
170 | .macro __idmap_cpu_set_reserved_ttbr1, tmp1, tmp2 | |
171 | adrp \tmp1, empty_zero_page | |
fa0465fc | 172 | phys_to_ttbr \tmp2, \tmp1 |
f992b4df WD |
173 | msr ttbr1_el1, \tmp2 |
174 | isb | |
175 | tlbi vmalle1 | |
176 | dsb nsh | |
177 | isb | |
178 | .endm | |
179 | ||
50e1881d MR |
180 | /* |
181 | * void idmap_cpu_replace_ttbr1(phys_addr_t new_pgd) | |
182 | * | |
183 | * This is the low-level counterpart to cpu_replace_ttbr1, and should not be | |
184 | * called by anything else. It can only be executed from a TTBR0 mapping. | |
185 | */ | |
186 | ENTRY(idmap_cpu_replace_ttbr1) | |
0fbeb318 | 187 | save_and_disable_daif flags=x2 |
50e1881d | 188 | |
f992b4df | 189 | __idmap_cpu_set_reserved_ttbr1 x1, x3 |
50e1881d | 190 | |
fa0465fc | 191 | phys_to_ttbr x3, x0 |
529c4b05 | 192 | msr ttbr1_el1, x3 |
50e1881d MR |
193 | isb |
194 | ||
0fbeb318 | 195 | restore_daif x2 |
50e1881d MR |
196 | |
197 | ret | |
198 | ENDPROC(idmap_cpu_replace_ttbr1) | |
199 | .popsection | |
200 | ||
f992b4df | 201 | #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
439e70e2 | 202 | .pushsection ".idmap.text", "awx" |
f992b4df WD |
203 | |
204 | .macro __idmap_kpti_get_pgtable_ent, type | |
205 | dc cvac, cur_\()\type\()p // Ensure any existing dirty | |
206 | dmb sy // lines are written back before | |
207 | ldr \type, [cur_\()\type\()p] // loading the entry | |
2ce77f6d WD |
208 | tbz \type, #0, skip_\()\type // Skip invalid and |
209 | tbnz \type, #11, skip_\()\type // non-global entries | |
f992b4df WD |
210 | .endm |
211 | ||
212 | .macro __idmap_kpti_put_pgtable_ent_ng, type | |
213 | orr \type, \type, #PTE_NG // Same bit for blocks and pages | |
214 | str \type, [cur_\()\type\()p] // Update the entry and ensure it | |
215 | dc civac, cur_\()\type\()p // is visible to all CPUs. | |
216 | .endm | |
217 | ||
218 | /* | |
219 | * void __kpti_install_ng_mappings(int cpu, int num_cpus, phys_addr_t swapper) | |
220 | * | |
221 | * Called exactly once from stop_machine context by each CPU found during boot. | |
222 | */ | |
223 | __idmap_kpti_flag: | |
224 | .long 1 | |
225 | ENTRY(idmap_kpti_install_ng_mappings) | |
226 | cpu .req w0 | |
227 | num_cpus .req w1 | |
228 | swapper_pa .req x2 | |
229 | swapper_ttb .req x3 | |
230 | flag_ptr .req x4 | |
231 | cur_pgdp .req x5 | |
232 | end_pgdp .req x6 | |
233 | pgd .req x7 | |
234 | cur_pudp .req x8 | |
235 | end_pudp .req x9 | |
236 | pud .req x10 | |
237 | cur_pmdp .req x11 | |
238 | end_pmdp .req x12 | |
239 | pmd .req x13 | |
240 | cur_ptep .req x14 | |
241 | end_ptep .req x15 | |
242 | pte .req x16 | |
243 | ||
244 | mrs swapper_ttb, ttbr1_el1 | |
245 | adr flag_ptr, __idmap_kpti_flag | |
246 | ||
247 | cbnz cpu, __idmap_kpti_secondary | |
248 | ||
249 | /* We're the boot CPU. Wait for the others to catch up */ | |
250 | sevl | |
251 | 1: wfe | |
252 | ldaxr w18, [flag_ptr] | |
253 | eor w18, w18, num_cpus | |
254 | cbnz w18, 1b | |
255 | ||
256 | /* We need to walk swapper, so turn off the MMU. */ | |
257 | pre_disable_mmu_workaround | |
258 | mrs x18, sctlr_el1 | |
259 | bic x18, x18, #SCTLR_ELx_M | |
260 | msr sctlr_el1, x18 | |
261 | isb | |
262 | ||
263 | /* Everybody is enjoying the idmap, so we can rewrite swapper. */ | |
264 | /* PGD */ | |
265 | mov cur_pgdp, swapper_pa | |
266 | add end_pgdp, cur_pgdp, #(PTRS_PER_PGD * 8) | |
267 | do_pgd: __idmap_kpti_get_pgtable_ent pgd | |
268 | tbnz pgd, #1, walk_puds | |
f992b4df | 269 | next_pgd: |
2ce77f6d WD |
270 | __idmap_kpti_put_pgtable_ent_ng pgd |
271 | skip_pgd: | |
f992b4df WD |
272 | add cur_pgdp, cur_pgdp, #8 |
273 | cmp cur_pgdp, end_pgdp | |
274 | b.ne do_pgd | |
275 | ||
276 | /* Publish the updated tables and nuke all the TLBs */ | |
277 | dsb sy | |
278 | tlbi vmalle1is | |
279 | dsb ish | |
280 | isb | |
281 | ||
282 | /* We're done: fire up the MMU again */ | |
283 | mrs x18, sctlr_el1 | |
284 | orr x18, x18, #SCTLR_ELx_M | |
285 | msr sctlr_el1, x18 | |
286 | isb | |
287 | ||
288 | /* Set the flag to zero to indicate that we're all done */ | |
289 | str wzr, [flag_ptr] | |
290 | ret | |
291 | ||
292 | /* PUD */ | |
293 | walk_puds: | |
294 | .if CONFIG_PGTABLE_LEVELS > 3 | |
295 | pte_to_phys cur_pudp, pgd | |
296 | add end_pudp, cur_pudp, #(PTRS_PER_PUD * 8) | |
297 | do_pud: __idmap_kpti_get_pgtable_ent pud | |
298 | tbnz pud, #1, walk_pmds | |
f992b4df | 299 | next_pud: |
2ce77f6d WD |
300 | __idmap_kpti_put_pgtable_ent_ng pud |
301 | skip_pud: | |
f992b4df WD |
302 | add cur_pudp, cur_pudp, 8 |
303 | cmp cur_pudp, end_pudp | |
304 | b.ne do_pud | |
305 | b next_pgd | |
306 | .else /* CONFIG_PGTABLE_LEVELS <= 3 */ | |
307 | mov pud, pgd | |
308 | b walk_pmds | |
309 | next_pud: | |
310 | b next_pgd | |
311 | .endif | |
312 | ||
313 | /* PMD */ | |
314 | walk_pmds: | |
315 | .if CONFIG_PGTABLE_LEVELS > 2 | |
316 | pte_to_phys cur_pmdp, pud | |
317 | add end_pmdp, cur_pmdp, #(PTRS_PER_PMD * 8) | |
318 | do_pmd: __idmap_kpti_get_pgtable_ent pmd | |
319 | tbnz pmd, #1, walk_ptes | |
f992b4df | 320 | next_pmd: |
2ce77f6d WD |
321 | __idmap_kpti_put_pgtable_ent_ng pmd |
322 | skip_pmd: | |
f992b4df WD |
323 | add cur_pmdp, cur_pmdp, #8 |
324 | cmp cur_pmdp, end_pmdp | |
325 | b.ne do_pmd | |
326 | b next_pud | |
327 | .else /* CONFIG_PGTABLE_LEVELS <= 2 */ | |
328 | mov pmd, pud | |
329 | b walk_ptes | |
330 | next_pmd: | |
331 | b next_pud | |
332 | .endif | |
333 | ||
334 | /* PTE */ | |
335 | walk_ptes: | |
336 | pte_to_phys cur_ptep, pmd | |
337 | add end_ptep, cur_ptep, #(PTRS_PER_PTE * 8) | |
338 | do_pte: __idmap_kpti_get_pgtable_ent pte | |
339 | __idmap_kpti_put_pgtable_ent_ng pte | |
2ce77f6d | 340 | skip_pte: |
f992b4df WD |
341 | add cur_ptep, cur_ptep, #8 |
342 | cmp cur_ptep, end_ptep | |
343 | b.ne do_pte | |
344 | b next_pmd | |
345 | ||
346 | /* Secondary CPUs end up here */ | |
347 | __idmap_kpti_secondary: | |
348 | /* Uninstall swapper before surgery begins */ | |
349 | __idmap_cpu_set_reserved_ttbr1 x18, x17 | |
350 | ||
351 | /* Increment the flag to let the boot CPU we're ready */ | |
352 | 1: ldxr w18, [flag_ptr] | |
353 | add w18, w18, #1 | |
354 | stxr w17, w18, [flag_ptr] | |
355 | cbnz w17, 1b | |
356 | ||
357 | /* Wait for the boot CPU to finish messing around with swapper */ | |
358 | sevl | |
359 | 1: wfe | |
360 | ldxr w18, [flag_ptr] | |
361 | cbnz w18, 1b | |
362 | ||
363 | /* All done, act like nothing happened */ | |
364 | msr ttbr1_el1, swapper_ttb | |
365 | isb | |
366 | ret | |
367 | ||
368 | .unreq cpu | |
369 | .unreq num_cpus | |
370 | .unreq swapper_pa | |
371 | .unreq swapper_ttb | |
372 | .unreq flag_ptr | |
373 | .unreq cur_pgdp | |
374 | .unreq end_pgdp | |
375 | .unreq pgd | |
376 | .unreq cur_pudp | |
377 | .unreq end_pudp | |
378 | .unreq pud | |
379 | .unreq cur_pmdp | |
380 | .unreq end_pmdp | |
381 | .unreq pmd | |
382 | .unreq cur_ptep | |
383 | .unreq end_ptep | |
384 | .unreq pte | |
385 | ENDPROC(idmap_kpti_install_ng_mappings) | |
386 | .popsection | |
387 | #endif | |
388 | ||
9cce7a43 CM |
389 | /* |
390 | * __cpu_setup | |
391 | * | |
392 | * Initialise the processor for turning the MMU on. Return in x0 the | |
393 | * value of the SCTLR_EL1 register. | |
394 | */ | |
439e70e2 | 395 | .pushsection ".idmap.text", "awx" |
9cce7a43 | 396 | ENTRY(__cpu_setup) |
fa7aae8a WD |
397 | tlbi vmalle1 // Invalidate local TLB |
398 | dsb nsh | |
9cce7a43 CM |
399 | |
400 | mov x0, #3 << 20 | |
401 | msr cpacr_el1, x0 // Enable FP/ASIMD | |
d8d23fa0 WD |
402 | mov x0, #1 << 12 // Reset mdscr_el1 and disable |
403 | msr mdscr_el1, x0 // access to the DCC from EL0 | |
2ce39ad1 WD |
404 | isb // Unmask debug exceptions now, |
405 | enable_dbg // since this is per-cpu | |
f436b2ac | 406 | reset_pmuserenr_el0 x0 // Disable PMU access from EL0 |
9cce7a43 CM |
407 | /* |
408 | * Memory region attributes for LPAE: | |
409 | * | |
410 | * n = AttrIndx[2:0] | |
411 | * n MAIR | |
412 | * DEVICE_nGnRnE 000 00000000 | |
413 | * DEVICE_nGnRE 001 00000100 | |
414 | * DEVICE_GRE 010 00001100 | |
415 | * NORMAL_NC 011 01000100 | |
416 | * NORMAL 100 11111111 | |
8d446c86 | 417 | * NORMAL_WT 101 10111011 |
9cce7a43 CM |
418 | */ |
419 | ldr x5, =MAIR(0x00, MT_DEVICE_nGnRnE) | \ | |
420 | MAIR(0x04, MT_DEVICE_nGnRE) | \ | |
421 | MAIR(0x0c, MT_DEVICE_GRE) | \ | |
422 | MAIR(0x44, MT_NORMAL_NC) | \ | |
8d446c86 JZZ |
423 | MAIR(0xff, MT_NORMAL) | \ |
424 | MAIR(0xbb, MT_NORMAL_WT) | |
9cce7a43 CM |
425 | msr mair_el1, x5 |
426 | /* | |
427 | * Prepare SCTLR | |
428 | */ | |
7a00d68e | 429 | mov_q x0, SCTLR_EL1_SET |
9cce7a43 CM |
430 | /* |
431 | * Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for | |
432 | * both user and kernel. | |
433 | */ | |
35a86976 | 434 | ldr x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \ |
7655abb9 | 435 | TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0 | TCR_A1 |
dd006da2 AB |
436 | tcr_set_idmap_t0sz x10, x9 |
437 | ||
87366d8c | 438 | /* |
787fd1d0 | 439 | * Set the IPS bits in TCR_EL1. |
87366d8c | 440 | */ |
787fd1d0 | 441 | tcr_compute_pa_size x10, #TCR_IPS_SHIFT, x5, x6 |
2f4b829c CM |
442 | #ifdef CONFIG_ARM64_HW_AFDBM |
443 | /* | |
444 | * Hardware update of the Access and Dirty bits. | |
445 | */ | |
446 | mrs x9, ID_AA64MMFR1_EL1 | |
447 | and x9, x9, #0xf | |
448 | cbz x9, 2f | |
449 | cmp x9, #2 | |
450 | b.lt 1f | |
451 | orr x10, x10, #TCR_HD // hardware Dirty flag update | |
452 | 1: orr x10, x10, #TCR_HA // hardware Access flag update | |
453 | 2: | |
454 | #endif /* CONFIG_ARM64_HW_AFDBM */ | |
9cce7a43 CM |
455 | msr tcr_el1, x10 |
456 | ret // return to head.S | |
457 | ENDPROC(__cpu_setup) |