Commit | Line | Data |
---|---|---|
ed1bbc40 TG |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | #include <linux/spinlock.h> | |
4 | #include <linux/percpu.h> | |
5 | ||
6 | #include <asm/cpu_entry_area.h> | |
7 | #include <asm/pgtable.h> | |
8 | #include <asm/fixmap.h> | |
9 | #include <asm/desc.h> | |
10 | ||
11 | static DEFINE_PER_CPU_PAGE_ALIGNED(struct entry_stack_page, entry_stack_storage); | |
12 | ||
13 | #ifdef CONFIG_X86_64 | |
14 | static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks | |
15 | [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]); | |
16 | #endif | |
17 | ||
18 | static void __init | |
19 | set_percpu_fixmap_pages(int idx, void *ptr, int pages, pgprot_t prot) | |
20 | { | |
21 | for ( ; pages; pages--, idx--, ptr += PAGE_SIZE) | |
22 | __set_fixmap(idx, per_cpu_ptr_to_phys(ptr), prot); | |
23 | } | |
24 | ||
25 | /* Setup the fixmap mappings only once per-processor */ | |
26 | static void __init setup_cpu_entry_area(int cpu) | |
27 | { | |
28 | #ifdef CONFIG_X86_64 | |
29 | extern char _entry_trampoline[]; | |
30 | ||
31 | /* On 64-bit systems, we use a read-only fixmap GDT and TSS. */ | |
32 | pgprot_t gdt_prot = PAGE_KERNEL_RO; | |
33 | pgprot_t tss_prot = PAGE_KERNEL_RO; | |
34 | #else | |
35 | /* | |
36 | * On native 32-bit systems, the GDT cannot be read-only because | |
37 | * our double fault handler uses a task gate, and entering through | |
38 | * a task gate needs to change an available TSS to busy. If the | |
39 | * GDT is read-only, that will triple fault. The TSS cannot be | |
40 | * read-only because the CPU writes to it on task switches. | |
41 | * | |
42 | * On Xen PV, the GDT must be read-only because the hypervisor | |
43 | * requires it. | |
44 | */ | |
45 | pgprot_t gdt_prot = boot_cpu_has(X86_FEATURE_XENPV) ? | |
46 | PAGE_KERNEL_RO : PAGE_KERNEL; | |
47 | pgprot_t tss_prot = PAGE_KERNEL; | |
48 | #endif | |
49 | ||
50 | __set_fixmap(get_cpu_entry_area_index(cpu, gdt), get_cpu_gdt_paddr(cpu), gdt_prot); | |
51 | set_percpu_fixmap_pages(get_cpu_entry_area_index(cpu, entry_stack_page), | |
52 | per_cpu_ptr(&entry_stack_storage, cpu), 1, | |
53 | PAGE_KERNEL); | |
54 | ||
55 | /* | |
56 | * The Intel SDM says (Volume 3, 7.2.1): | |
57 | * | |
58 | * Avoid placing a page boundary in the part of the TSS that the | |
59 | * processor reads during a task switch (the first 104 bytes). The | |
60 | * processor may not correctly perform address translations if a | |
61 | * boundary occurs in this area. During a task switch, the processor | |
62 | * reads and writes into the first 104 bytes of each TSS (using | |
63 | * contiguous physical addresses beginning with the physical address | |
64 | * of the first byte of the TSS). So, after TSS access begins, if | |
65 | * part of the 104 bytes is not physically contiguous, the processor | |
66 | * will access incorrect information without generating a page-fault | |
67 | * exception. | |
68 | * | |
69 | * There are also a lot of errata involving the TSS spanning a page | |
70 | * boundary. Assert that we're not doing that. | |
71 | */ | |
72 | BUILD_BUG_ON((offsetof(struct tss_struct, x86_tss) ^ | |
73 | offsetofend(struct tss_struct, x86_tss)) & PAGE_MASK); | |
74 | BUILD_BUG_ON(sizeof(struct tss_struct) % PAGE_SIZE != 0); | |
75 | set_percpu_fixmap_pages(get_cpu_entry_area_index(cpu, tss), | |
76 | &per_cpu(cpu_tss_rw, cpu), | |
77 | sizeof(struct tss_struct) / PAGE_SIZE, | |
78 | tss_prot); | |
79 | ||
80 | #ifdef CONFIG_X86_32 | |
81 | per_cpu(cpu_entry_area, cpu) = get_cpu_entry_area(cpu); | |
82 | #endif | |
83 | ||
84 | #ifdef CONFIG_X86_64 | |
85 | BUILD_BUG_ON(sizeof(exception_stacks) % PAGE_SIZE != 0); | |
86 | BUILD_BUG_ON(sizeof(exception_stacks) != | |
87 | sizeof(((struct cpu_entry_area *)0)->exception_stacks)); | |
88 | set_percpu_fixmap_pages(get_cpu_entry_area_index(cpu, exception_stacks), | |
89 | &per_cpu(exception_stacks, cpu), | |
90 | sizeof(exception_stacks) / PAGE_SIZE, | |
91 | PAGE_KERNEL); | |
92 | ||
93 | __set_fixmap(get_cpu_entry_area_index(cpu, entry_trampoline), | |
94 | __pa_symbol(_entry_trampoline), PAGE_KERNEL_RX); | |
95 | #endif | |
96 | } | |
97 | ||
98 | void __init setup_cpu_entry_areas(void) | |
99 | { | |
100 | unsigned int cpu; | |
101 | ||
102 | for_each_possible_cpu(cpu) | |
103 | setup_cpu_entry_area(cpu); | |
104 | } |