| 1 | /* |
| 2 | * c 2001 PPC 64 Team, IBM Corp |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or |
| 5 | * modify it under the terms of the GNU General Public License |
| 6 | * as published by the Free Software Foundation; either version |
| 7 | * 2 of the License, or (at your option) any later version. |
| 8 | */ |
| 9 | |
| 10 | #include <linux/smp.h> |
| 11 | #include <linux/export.h> |
| 12 | #include <linux/memblock.h> |
| 13 | #include <linux/sched/task.h> |
| 14 | #include <linux/numa.h> |
| 15 | |
| 16 | #include <asm/lppaca.h> |
| 17 | #include <asm/paca.h> |
| 18 | #include <asm/sections.h> |
| 19 | #include <asm/pgtable.h> |
| 20 | #include <asm/kexec.h> |
| 21 | |
| 22 | #include "setup.h" |
| 23 | |
| 24 | #ifndef CONFIG_SMP |
| 25 | #define boot_cpuid 0 |
| 26 | #endif |
| 27 | |
| 28 | static void *__init alloc_paca_data(unsigned long size, unsigned long align, |
| 29 | unsigned long limit, int cpu) |
| 30 | { |
| 31 | void *ptr; |
| 32 | int nid; |
| 33 | |
| 34 | /* |
| 35 | * boot_cpuid paca is allocated very early before cpu_to_node is up. |
| 36 | * Set bottom-up mode, because the boot CPU should be on node-0, |
| 37 | * which will put its paca in the right place. |
| 38 | */ |
| 39 | if (cpu == boot_cpuid) { |
| 40 | nid = NUMA_NO_NODE; |
| 41 | memblock_set_bottom_up(true); |
| 42 | } else { |
| 43 | nid = early_cpu_to_node(cpu); |
| 44 | } |
| 45 | |
| 46 | ptr = memblock_alloc_try_nid(size, align, MEMBLOCK_LOW_LIMIT, |
| 47 | limit, nid); |
| 48 | if (!ptr) |
| 49 | panic("cannot allocate paca data"); |
| 50 | |
| 51 | if (cpu == boot_cpuid) |
| 52 | memblock_set_bottom_up(false); |
| 53 | |
| 54 | return ptr; |
| 55 | } |
| 56 | |
| 57 | #ifdef CONFIG_PPC_PSERIES |
| 58 | |
| 59 | /* |
| 60 | * See asm/lppaca.h for more detail. |
| 61 | * |
| 62 | * lppaca structures must must be 1kB in size, L1 cache line aligned, |
| 63 | * and not cross 4kB boundary. A 1kB size and 1kB alignment will satisfy |
| 64 | * these requirements. |
| 65 | */ |
| 66 | static inline void init_lppaca(struct lppaca *lppaca) |
| 67 | { |
| 68 | BUILD_BUG_ON(sizeof(struct lppaca) != 640); |
| 69 | |
| 70 | *lppaca = (struct lppaca) { |
| 71 | .desc = cpu_to_be32(0xd397d781), /* "LpPa" */ |
| 72 | .size = cpu_to_be16(0x400), |
| 73 | .fpregs_in_use = 1, |
| 74 | .slb_count = cpu_to_be16(64), |
| 75 | .vmxregs_in_use = 0, |
| 76 | .page_ins = 0, }; |
| 77 | }; |
| 78 | |
| 79 | static struct lppaca * __init new_lppaca(int cpu, unsigned long limit) |
| 80 | { |
| 81 | struct lppaca *lp; |
| 82 | size_t size = 0x400; |
| 83 | |
| 84 | BUILD_BUG_ON(size < sizeof(struct lppaca)); |
| 85 | |
| 86 | if (early_cpu_has_feature(CPU_FTR_HVMODE)) |
| 87 | return NULL; |
| 88 | |
| 89 | lp = alloc_paca_data(size, 0x400, limit, cpu); |
| 90 | init_lppaca(lp); |
| 91 | |
| 92 | return lp; |
| 93 | } |
| 94 | #endif /* CONFIG_PPC_BOOK3S */ |
| 95 | |
| 96 | #ifdef CONFIG_PPC_BOOK3S_64 |
| 97 | |
| 98 | /* |
| 99 | * 3 persistent SLBs are allocated here. The buffer will be zero |
| 100 | * initially, hence will all be invaild until we actually write them. |
| 101 | * |
| 102 | * If you make the number of persistent SLB entries dynamic, please also |
| 103 | * update PR KVM to flush and restore them accordingly. |
| 104 | */ |
| 105 | static struct slb_shadow * __init new_slb_shadow(int cpu, unsigned long limit) |
| 106 | { |
| 107 | struct slb_shadow *s; |
| 108 | |
| 109 | if (cpu != boot_cpuid) { |
| 110 | /* |
| 111 | * Boot CPU comes here before early_radix_enabled |
| 112 | * is parsed (e.g., for disable_radix). So allocate |
| 113 | * always and this will be fixed up in free_unused_pacas. |
| 114 | */ |
| 115 | if (early_radix_enabled()) |
| 116 | return NULL; |
| 117 | } |
| 118 | |
| 119 | s = alloc_paca_data(sizeof(*s), L1_CACHE_BYTES, limit, cpu); |
| 120 | |
| 121 | s->persistent = cpu_to_be32(SLB_NUM_BOLTED); |
| 122 | s->buffer_length = cpu_to_be32(sizeof(*s)); |
| 123 | |
| 124 | return s; |
| 125 | } |
| 126 | |
| 127 | #endif /* CONFIG_PPC_BOOK3S_64 */ |
| 128 | |
| 129 | /* The Paca is an array with one entry per processor. Each contains an |
| 130 | * lppaca, which contains the information shared between the |
| 131 | * hypervisor and Linux. |
| 132 | * On systems with hardware multi-threading, there are two threads |
| 133 | * per processor. The Paca array must contain an entry for each thread. |
| 134 | * The VPD Areas will give a max logical processors = 2 * max physical |
| 135 | * processors. The processor VPD array needs one entry per physical |
| 136 | * processor (not thread). |
| 137 | */ |
| 138 | struct paca_struct **paca_ptrs __read_mostly; |
| 139 | EXPORT_SYMBOL(paca_ptrs); |
| 140 | |
| 141 | void __init initialise_paca(struct paca_struct *new_paca, int cpu) |
| 142 | { |
| 143 | #ifdef CONFIG_PPC_PSERIES |
| 144 | new_paca->lppaca_ptr = NULL; |
| 145 | #endif |
| 146 | #ifdef CONFIG_PPC_BOOK3E |
| 147 | new_paca->kernel_pgd = swapper_pg_dir; |
| 148 | #endif |
| 149 | new_paca->lock_token = 0x8000; |
| 150 | new_paca->paca_index = cpu; |
| 151 | new_paca->kernel_toc = kernel_toc_addr(); |
| 152 | new_paca->kernelbase = (unsigned long) _stext; |
| 153 | /* Only set MSR:IR/DR when MMU is initialized */ |
| 154 | new_paca->kernel_msr = MSR_KERNEL & ~(MSR_IR | MSR_DR); |
| 155 | new_paca->hw_cpu_id = 0xffff; |
| 156 | new_paca->kexec_state = KEXEC_STATE_NONE; |
| 157 | new_paca->__current = &init_task; |
| 158 | new_paca->data_offset = 0xfeeeeeeeeeeeeeeeULL; |
| 159 | #ifdef CONFIG_PPC_BOOK3S_64 |
| 160 | new_paca->slb_shadow_ptr = NULL; |
| 161 | #endif |
| 162 | |
| 163 | #ifdef CONFIG_PPC_BOOK3E |
| 164 | /* For now -- if we have threads this will be adjusted later */ |
| 165 | new_paca->tcd_ptr = &new_paca->tcd; |
| 166 | #endif |
| 167 | } |
| 168 | |
| 169 | /* Put the paca pointer into r13 and SPRG_PACA */ |
| 170 | void setup_paca(struct paca_struct *new_paca) |
| 171 | { |
| 172 | /* Setup r13 */ |
| 173 | local_paca = new_paca; |
| 174 | |
| 175 | #ifdef CONFIG_PPC_BOOK3E |
| 176 | /* On Book3E, initialize the TLB miss exception frames */ |
| 177 | mtspr(SPRN_SPRG_TLB_EXFRAME, local_paca->extlb); |
| 178 | #else |
| 179 | /* In HV mode, we setup both HPACA and PACA to avoid problems |
| 180 | * if we do a GET_PACA() before the feature fixups have been |
| 181 | * applied |
| 182 | */ |
| 183 | if (early_cpu_has_feature(CPU_FTR_HVMODE)) |
| 184 | mtspr(SPRN_SPRG_HPACA, local_paca); |
| 185 | #endif |
| 186 | mtspr(SPRN_SPRG_PACA, local_paca); |
| 187 | |
| 188 | } |
| 189 | |
| 190 | static int __initdata paca_nr_cpu_ids; |
| 191 | static int __initdata paca_ptrs_size; |
| 192 | static int __initdata paca_struct_size; |
| 193 | |
| 194 | void __init allocate_paca_ptrs(void) |
| 195 | { |
| 196 | paca_nr_cpu_ids = nr_cpu_ids; |
| 197 | |
| 198 | paca_ptrs_size = sizeof(struct paca_struct *) * nr_cpu_ids; |
| 199 | paca_ptrs = memblock_alloc_raw(paca_ptrs_size, SMP_CACHE_BYTES); |
| 200 | if (!paca_ptrs) |
| 201 | panic("Failed to allocate %d bytes for paca pointers\n", |
| 202 | paca_ptrs_size); |
| 203 | |
| 204 | memset(paca_ptrs, 0x88, paca_ptrs_size); |
| 205 | } |
| 206 | |
| 207 | void __init allocate_paca(int cpu) |
| 208 | { |
| 209 | u64 limit; |
| 210 | struct paca_struct *paca; |
| 211 | |
| 212 | BUG_ON(cpu >= paca_nr_cpu_ids); |
| 213 | |
| 214 | #ifdef CONFIG_PPC_BOOK3S_64 |
| 215 | /* |
| 216 | * We access pacas in real mode, and cannot take SLB faults |
| 217 | * on them when in virtual mode, so allocate them accordingly. |
| 218 | */ |
| 219 | limit = min(ppc64_bolted_size(), ppc64_rma_size); |
| 220 | #else |
| 221 | limit = ppc64_rma_size; |
| 222 | #endif |
| 223 | |
| 224 | paca = alloc_paca_data(sizeof(struct paca_struct), L1_CACHE_BYTES, |
| 225 | limit, cpu); |
| 226 | paca_ptrs[cpu] = paca; |
| 227 | |
| 228 | initialise_paca(paca, cpu); |
| 229 | #ifdef CONFIG_PPC_PSERIES |
| 230 | paca->lppaca_ptr = new_lppaca(cpu, limit); |
| 231 | #endif |
| 232 | #ifdef CONFIG_PPC_BOOK3S_64 |
| 233 | paca->slb_shadow_ptr = new_slb_shadow(cpu, limit); |
| 234 | #endif |
| 235 | paca_struct_size += sizeof(struct paca_struct); |
| 236 | } |
| 237 | |
| 238 | void __init free_unused_pacas(void) |
| 239 | { |
| 240 | int new_ptrs_size; |
| 241 | |
| 242 | new_ptrs_size = sizeof(struct paca_struct *) * nr_cpu_ids; |
| 243 | if (new_ptrs_size < paca_ptrs_size) |
| 244 | memblock_free(__pa(paca_ptrs) + new_ptrs_size, |
| 245 | paca_ptrs_size - new_ptrs_size); |
| 246 | |
| 247 | paca_nr_cpu_ids = nr_cpu_ids; |
| 248 | paca_ptrs_size = new_ptrs_size; |
| 249 | |
| 250 | #ifdef CONFIG_PPC_BOOK3S_64 |
| 251 | if (early_radix_enabled()) { |
| 252 | /* Ugly fixup, see new_slb_shadow() */ |
| 253 | memblock_free(__pa(paca_ptrs[boot_cpuid]->slb_shadow_ptr), |
| 254 | sizeof(struct slb_shadow)); |
| 255 | paca_ptrs[boot_cpuid]->slb_shadow_ptr = NULL; |
| 256 | } |
| 257 | #endif |
| 258 | |
| 259 | printk(KERN_DEBUG "Allocated %u bytes for %u pacas\n", |
| 260 | paca_ptrs_size + paca_struct_size, nr_cpu_ids); |
| 261 | } |
| 262 | |
| 263 | void copy_mm_to_paca(struct mm_struct *mm) |
| 264 | { |
| 265 | #ifdef CONFIG_PPC_BOOK3S |
| 266 | mm_context_t *context = &mm->context; |
| 267 | |
| 268 | get_paca()->mm_ctx_id = context->id; |
| 269 | #ifdef CONFIG_PPC_MM_SLICES |
| 270 | VM_BUG_ON(!mm_ctx_slb_addr_limit(context)); |
| 271 | get_paca()->mm_ctx_slb_addr_limit = mm_ctx_slb_addr_limit(context); |
| 272 | memcpy(&get_paca()->mm_ctx_low_slices_psize, mm_ctx_low_slices(context), |
| 273 | LOW_SLICE_ARRAY_SZ); |
| 274 | memcpy(&get_paca()->mm_ctx_high_slices_psize, mm_ctx_high_slices(context), |
| 275 | TASK_SLICE_ARRAY_SZ(context)); |
| 276 | #else /* CONFIG_PPC_MM_SLICES */ |
| 277 | get_paca()->mm_ctx_user_psize = context->user_psize; |
| 278 | get_paca()->mm_ctx_sllp = context->sllp; |
| 279 | #endif |
| 280 | #else /* !CONFIG_PPC_BOOK3S */ |
| 281 | return; |
| 282 | #endif |
| 283 | } |