Commit | Line | Data |
---|---|---|
14cf11af PM |
1 | /* |
2 | * This file contains the routines setting up the linux page tables. | |
3 | * -- paulus | |
4 | * | |
5 | * Derived from arch/ppc/mm/init.c: | |
6 | * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) | |
7 | * | |
8 | * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au) | |
9 | * and Cort Dougan (PReP) (cort@cs.nmt.edu) | |
10 | * Copyright (C) 1996 Paul Mackerras | |
14cf11af PM |
11 | * |
12 | * Derived from "arch/i386/mm/init.c" | |
13 | * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds | |
14 | * | |
15 | * This program is free software; you can redistribute it and/or | |
16 | * modify it under the terms of the GNU General Public License | |
17 | * as published by the Free Software Foundation; either version | |
18 | * 2 of the License, or (at your option) any later version. | |
19 | * | |
20 | */ | |
21 | ||
14cf11af PM |
22 | #include <linux/kernel.h> |
23 | #include <linux/module.h> | |
24 | #include <linux/types.h> | |
25 | #include <linux/mm.h> | |
26 | #include <linux/vmalloc.h> | |
27 | #include <linux/init.h> | |
28 | #include <linux/highmem.h> | |
95f72d1e | 29 | #include <linux/memblock.h> |
5a0e3ad6 | 30 | #include <linux/slab.h> |
14cf11af PM |
31 | |
32 | #include <asm/pgtable.h> | |
33 | #include <asm/pgalloc.h> | |
2c419bde | 34 | #include <asm/fixmap.h> |
14cf11af | 35 | #include <asm/io.h> |
ae3a197e | 36 | #include <asm/setup.h> |
95902e6c | 37 | #include <asm/sections.h> |
14cf11af | 38 | |
9d9f2ccc | 39 | #include <mm/mmu_decl.h> |
14cf11af | 40 | |
14cf11af | 41 | unsigned long ioremap_bot; |
920573bd | 42 | EXPORT_SYMBOL(ioremap_bot); /* aka VMALLOC_END */ |
14cf11af | 43 | |
060ef9d8 | 44 | extern char etext[], _stext[], _sinittext[], _einittext[]; |
14cf11af | 45 | |
14cf11af PM |
46 | void __iomem * |
47 | ioremap(phys_addr_t addr, unsigned long size) | |
48 | { | |
c766ee72 | 49 | pgprot_t prot = pgprot_noncached(PAGE_KERNEL); |
56f3c141 | 50 | |
c766ee72 | 51 | return __ioremap_caller(addr, size, prot, __builtin_return_address(0)); |
14cf11af | 52 | } |
920573bd | 53 | EXPORT_SYMBOL(ioremap); |
14cf11af | 54 | |
be135f40 AB |
55 | void __iomem * |
56 | ioremap_wc(phys_addr_t addr, unsigned long size) | |
57 | { | |
c766ee72 | 58 | pgprot_t prot = pgprot_noncached_wc(PAGE_KERNEL); |
56f3c141 | 59 | |
c766ee72 | 60 | return __ioremap_caller(addr, size, prot, __builtin_return_address(0)); |
be135f40 AB |
61 | } |
62 | EXPORT_SYMBOL(ioremap_wc); | |
63 | ||
86c391bd CL |
64 | void __iomem * |
65 | ioremap_wt(phys_addr_t addr, unsigned long size) | |
66 | { | |
c766ee72 | 67 | pgprot_t prot = pgprot_cached_wthru(PAGE_KERNEL); |
56f3c141 | 68 | |
c766ee72 | 69 | return __ioremap_caller(addr, size, prot, __builtin_return_address(0)); |
86c391bd CL |
70 | } |
71 | EXPORT_SYMBOL(ioremap_wt); | |
72 | ||
73 | void __iomem * | |
74 | ioremap_coherent(phys_addr_t addr, unsigned long size) | |
75 | { | |
c766ee72 | 76 | pgprot_t prot = pgprot_cached(PAGE_KERNEL); |
56f3c141 | 77 | |
c766ee72 | 78 | return __ioremap_caller(addr, size, prot, __builtin_return_address(0)); |
86c391bd CL |
79 | } |
80 | EXPORT_SYMBOL(ioremap_coherent); | |
81 | ||
68a64357 | 82 | void __iomem * |
40f1ce7f | 83 | ioremap_prot(phys_addr_t addr, unsigned long size, unsigned long flags) |
68a64357 | 84 | { |
26973fa5 CL |
85 | pte_t pte = __pte(flags); |
86 | ||
a1f242ff | 87 | /* writeable implies dirty for kernel addresses */ |
26973fa5 CL |
88 | if (pte_write(pte)) |
89 | pte = pte_mkdirty(pte); | |
a1f242ff BH |
90 | |
91 | /* we don't want to let _PAGE_USER and _PAGE_EXEC leak out */ | |
26973fa5 CL |
92 | pte = pte_exprotect(pte); |
93 | pte = pte_mkprivileged(pte); | |
55052eec | 94 | |
26973fa5 | 95 | return __ioremap_caller(addr, size, pte_pgprot(pte), __builtin_return_address(0)); |
68a64357 | 96 | } |
40f1ce7f | 97 | EXPORT_SYMBOL(ioremap_prot); |
68a64357 | 98 | |
14cf11af PM |
99 | void __iomem * |
100 | __ioremap(phys_addr_t addr, unsigned long size, unsigned long flags) | |
1cdab55d | 101 | { |
c766ee72 | 102 | return __ioremap_caller(addr, size, __pgprot(flags), __builtin_return_address(0)); |
1cdab55d BH |
103 | } |
104 | ||
105 | void __iomem * | |
c766ee72 | 106 | __ioremap_caller(phys_addr_t addr, unsigned long size, pgprot_t prot, void *caller) |
14cf11af PM |
107 | { |
108 | unsigned long v, i; | |
109 | phys_addr_t p; | |
110 | int err; | |
111 | ||
112 | /* | |
113 | * Choose an address to map it to. | |
114 | * Once the vmalloc system is running, we use it. | |
e974cd4b | 115 | * Before then, we use space going down from IOREMAP_TOP |
14cf11af PM |
116 | * (ioremap_bot records where we're up to). |
117 | */ | |
118 | p = addr & PAGE_MASK; | |
119 | size = PAGE_ALIGN(addr + size) - p; | |
120 | ||
121 | /* | |
122 | * If the address lies within the first 16 MB, assume it's in ISA | |
123 | * memory space | |
124 | */ | |
125 | if (p < 16*1024*1024) | |
126 | p += _ISA_MEM_BASE; | |
127 | ||
01695a96 | 128 | #ifndef CONFIG_CRASH_DUMP |
14cf11af PM |
129 | /* |
130 | * Don't allow anybody to remap normal RAM that we're using. | |
131 | * mem_init() sets high_memory so only do the check after that. | |
132 | */ | |
6bf752da | 133 | if (slab_is_available() && p <= virt_to_phys(high_memory - 1) && |
7e140591 | 134 | page_is_ram(__phys_to_pfn(p))) { |
cc83458d | 135 | printk("__ioremap(): phys addr 0x%llx is RAM lr %ps\n", |
37f01d64 | 136 | (unsigned long long)p, __builtin_return_address(0)); |
14cf11af PM |
137 | return NULL; |
138 | } | |
01695a96 | 139 | #endif |
14cf11af PM |
140 | |
141 | if (size == 0) | |
142 | return NULL; | |
143 | ||
144 | /* | |
145 | * Is it already mapped? Perhaps overlapped by a previous | |
3084cdb7 | 146 | * mapping. |
14cf11af | 147 | */ |
3084cdb7 CL |
148 | v = p_block_mapped(p); |
149 | if (v) | |
14cf11af PM |
150 | goto out; |
151 | ||
f691fa10 | 152 | if (slab_is_available()) { |
14cf11af | 153 | struct vm_struct *area; |
1cdab55d | 154 | area = get_vm_area_caller(size, VM_IOREMAP, caller); |
14cf11af PM |
155 | if (area == 0) |
156 | return NULL; | |
7a9d1256 | 157 | area->phys_addr = p; |
14cf11af PM |
158 | v = (unsigned long) area->addr; |
159 | } else { | |
160 | v = (ioremap_bot -= size); | |
161 | } | |
162 | ||
14cf11af PM |
163 | /* |
164 | * Should check if it is a candidate for a BAT mapping | |
165 | */ | |
166 | ||
167 | err = 0; | |
168 | for (i = 0; i < size && err == 0; i += PAGE_SIZE) | |
c766ee72 | 169 | err = map_kernel_page(v + i, p + i, prot); |
14cf11af | 170 | if (err) { |
f691fa10 | 171 | if (slab_is_available()) |
14cf11af PM |
172 | vunmap((void *)v); |
173 | return NULL; | |
174 | } | |
175 | ||
176 | out: | |
177 | return (void __iomem *) (v + ((unsigned long)addr & ~PAGE_MASK)); | |
178 | } | |
920573bd | 179 | EXPORT_SYMBOL(__ioremap); |
14cf11af PM |
180 | |
181 | void iounmap(volatile void __iomem *addr) | |
182 | { | |
183 | /* | |
184 | * If mapped by BATs then there is nothing to do. | |
185 | * Calling vfree() generates a benign warning. | |
186 | */ | |
3084cdb7 CL |
187 | if (v_block_mapped((unsigned long)addr)) |
188 | return; | |
14cf11af PM |
189 | |
190 | if (addr > high_memory && (unsigned long) addr < ioremap_bot) | |
191 | vunmap((void *) (PAGE_MASK & (unsigned long)addr)); | |
192 | } | |
920573bd | 193 | EXPORT_SYMBOL(iounmap); |
14cf11af | 194 | |
4a6d8cf9 CL |
195 | static void __init *early_alloc_pgtable(unsigned long size) |
196 | { | |
197 | void *ptr = memblock_alloc(size, size); | |
198 | ||
199 | if (!ptr) | |
200 | panic("%s: Failed to allocate %lu bytes align=0x%lx\n", | |
201 | __func__, size, size); | |
202 | ||
203 | return ptr; | |
204 | } | |
205 | ||
206 | static pte_t __init *early_pte_alloc_kernel(pmd_t *pmdp, unsigned long va) | |
207 | { | |
208 | if (pmd_none(*pmdp)) { | |
209 | pte_t *ptep = early_alloc_pgtable(PTE_FRAG_SIZE); | |
210 | ||
211 | pmd_populate_kernel(&init_mm, pmdp, ptep); | |
212 | } | |
213 | return pte_offset_kernel(pmdp, va); | |
214 | } | |
215 | ||
216 | ||
217 | int __ref map_kernel_page(unsigned long va, phys_addr_t pa, pgprot_t prot) | |
14cf11af PM |
218 | { |
219 | pmd_t *pd; | |
220 | pte_t *pg; | |
221 | int err = -ENOMEM; | |
222 | ||
14cf11af | 223 | /* Use upper 10 bits of VA to index the first level map */ |
d1953c88 | 224 | pd = pmd_offset(pud_offset(pgd_offset_k(va), va), va); |
14cf11af | 225 | /* Use middle 10 bits of VA to index the second-level map */ |
4a6d8cf9 CL |
226 | if (likely(slab_is_available())) |
227 | pg = pte_alloc_kernel(pd, va); | |
228 | else | |
229 | pg = early_pte_alloc_kernel(pd, va); | |
14cf11af PM |
230 | if (pg != 0) { |
231 | err = 0; | |
3be4e699 BH |
232 | /* The PTE should never be already set nor present in the |
233 | * hash table | |
234 | */ | |
26973fa5 | 235 | BUG_ON((pte_present(*pg) | pte_hashpte(*pg)) && pgprot_val(prot)); |
c766ee72 | 236 | set_pte_at(&init_mm, va, pg, pfn_pte(pa >> PAGE_SHIFT, prot)); |
14cf11af | 237 | } |
47ce8af4 | 238 | smp_wmb(); |
14cf11af PM |
239 | return err; |
240 | } | |
241 | ||
242 | /* | |
de32400d | 243 | * Map in a chunk of physical memory starting at start. |
14cf11af | 244 | */ |
86b19520 | 245 | static void __init __mapin_ram_chunk(unsigned long offset, unsigned long top) |
14cf11af | 246 | { |
c766ee72 | 247 | unsigned long v, s; |
99c62dd7 | 248 | phys_addr_t p; |
ee4f2ea4 | 249 | int ktext; |
14cf11af | 250 | |
de32400d | 251 | s = offset; |
ccdcef72 | 252 | v = PAGE_OFFSET + s; |
99c62dd7 | 253 | p = memstart_addr + s; |
de32400d | 254 | for (; s < top; s += PAGE_SIZE) { |
060ef9d8 CL |
255 | ktext = ((char *)v >= _stext && (char *)v < etext) || |
256 | ((char *)v >= _sinittext && (char *)v < _einittext); | |
c766ee72 | 257 | map_kernel_page(v, p, ktext ? PAGE_KERNEL_TEXT : PAGE_KERNEL); |
68289ae9 | 258 | #ifdef CONFIG_PPC_BOOK3S_32 |
ee4f2ea4 | 259 | if (ktext) |
34eb138e | 260 | hash_preload(&init_mm, v, false, 0x300); |
ee4f2ea4 | 261 | #endif |
14cf11af PM |
262 | v += PAGE_SIZE; |
263 | p += PAGE_SIZE; | |
264 | } | |
265 | } | |
266 | ||
de32400d AH |
267 | void __init mapin_ram(void) |
268 | { | |
9e849f23 CL |
269 | struct memblock_region *reg; |
270 | ||
271 | for_each_memblock(memory, reg) { | |
272 | phys_addr_t base = reg->base; | |
273 | phys_addr_t top = min(base + reg->size, total_lowmem); | |
de32400d | 274 | |
9e849f23 CL |
275 | if (base >= top) |
276 | continue; | |
277 | base = mmu_mapin_ram(base, top); | |
d2f15e09 CL |
278 | if (IS_ENABLED(CONFIG_BDI_SWITCH)) |
279 | __mapin_ram_chunk(reg->base, top); | |
280 | else | |
281 | __mapin_ram_chunk(base, top); | |
de32400d | 282 | } |
de32400d AH |
283 | } |
284 | ||
14cf11af PM |
285 | /* Scan the real Linux page tables and return a PTE pointer for |
286 | * a virtual address in a context. | |
287 | * Returns true (1) if PTE was found, zero otherwise. The pointer to | |
288 | * the PTE pointer is unmodified if PTE is not found. | |
289 | */ | |
86b19520 | 290 | static int |
bab70a4a | 291 | get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep, pmd_t **pmdp) |
14cf11af PM |
292 | { |
293 | pgd_t *pgd; | |
d1953c88 | 294 | pud_t *pud; |
14cf11af PM |
295 | pmd_t *pmd; |
296 | pte_t *pte; | |
297 | int retval = 0; | |
298 | ||
299 | pgd = pgd_offset(mm, addr & PAGE_MASK); | |
300 | if (pgd) { | |
d1953c88 DG |
301 | pud = pud_offset(pgd, addr & PAGE_MASK); |
302 | if (pud && pud_present(*pud)) { | |
303 | pmd = pmd_offset(pud, addr & PAGE_MASK); | |
304 | if (pmd_present(*pmd)) { | |
305 | pte = pte_offset_map(pmd, addr & PAGE_MASK); | |
306 | if (pte) { | |
307 | retval = 1; | |
308 | *ptep = pte; | |
309 | if (pmdp) | |
310 | *pmdp = pmd; | |
311 | /* XXX caller needs to do pte_unmap, yuck */ | |
312 | } | |
313 | } | |
314 | } | |
14cf11af PM |
315 | } |
316 | return(retval); | |
317 | } | |
318 | ||
e611939f | 319 | static int __change_page_attr_noflush(struct page *page, pgprot_t prot) |
88df6e90 BH |
320 | { |
321 | pte_t *kpte; | |
322 | pmd_t *kpmd; | |
323 | unsigned long address; | |
324 | ||
325 | BUG_ON(PageHighMem(page)); | |
326 | address = (unsigned long)page_address(page); | |
327 | ||
3084cdb7 | 328 | if (v_block_mapped(address)) |
88df6e90 BH |
329 | return 0; |
330 | if (!get_pteptr(&init_mm, address, &kpte, &kpmd)) | |
331 | return -EINVAL; | |
50891457 | 332 | __set_pte_at(&init_mm, address, kpte, mk_pte(page, prot), 0); |
88df6e90 BH |
333 | pte_unmap(kpte); |
334 | ||
335 | return 0; | |
336 | } | |
337 | ||
338 | /* | |
339 | * Change the page attributes of an page in the linear mapping. | |
340 | * | |
3184cc4b | 341 | * THIS DOES NOTHING WITH BAT MAPPINGS, DEBUG USE ONLY |
88df6e90 BH |
342 | */ |
343 | static int change_page_attr(struct page *page, int numpages, pgprot_t prot) | |
344 | { | |
345 | int i, err = 0; | |
346 | unsigned long flags; | |
e611939f | 347 | struct page *start = page; |
88df6e90 BH |
348 | |
349 | local_irq_save(flags); | |
350 | for (i = 0; i < numpages; i++, page++) { | |
e611939f | 351 | err = __change_page_attr_noflush(page, prot); |
88df6e90 BH |
352 | if (err) |
353 | break; | |
354 | } | |
e611939f | 355 | wmb(); |
7c6a4f3b | 356 | local_irq_restore(flags); |
e611939f CL |
357 | flush_tlb_kernel_range((unsigned long)page_address(start), |
358 | (unsigned long)page_address(page)); | |
88df6e90 BH |
359 | return err; |
360 | } | |
361 | ||
3184cc4b CL |
362 | void mark_initmem_nx(void) |
363 | { | |
364 | struct page *page = virt_to_page(_sinittext); | |
365 | unsigned long numpages = PFN_UP((unsigned long)_einittext) - | |
366 | PFN_DOWN((unsigned long)_sinittext); | |
367 | ||
63b2bc61 CL |
368 | if (v_block_mapped((unsigned long)_stext) + 1) |
369 | mmu_mark_initmem_nx(); | |
370 | else | |
371 | change_page_attr(page, numpages, PAGE_KERNEL); | |
3184cc4b | 372 | } |
88df6e90 | 373 | |
95902e6c CL |
374 | #ifdef CONFIG_STRICT_KERNEL_RWX |
375 | void mark_rodata_ro(void) | |
376 | { | |
377 | struct page *page; | |
378 | unsigned long numpages; | |
379 | ||
63b2bc61 CL |
380 | if (v_block_mapped((unsigned long)_sinittext)) { |
381 | mmu_mark_rodata_ro(); | |
382 | return; | |
383 | } | |
384 | ||
95902e6c CL |
385 | page = virt_to_page(_stext); |
386 | numpages = PFN_UP((unsigned long)_etext) - | |
387 | PFN_DOWN((unsigned long)_stext); | |
388 | ||
389 | change_page_attr(page, numpages, PAGE_KERNEL_ROX); | |
390 | /* | |
391 | * mark .rodata as read only. Use __init_begin rather than __end_rodata | |
392 | * to cover NOTES and EXCEPTION_TABLE. | |
393 | */ | |
394 | page = virt_to_page(__start_rodata); | |
395 | numpages = PFN_UP((unsigned long)__init_begin) - | |
396 | PFN_DOWN((unsigned long)__start_rodata); | |
397 | ||
398 | change_page_attr(page, numpages, PAGE_KERNEL_RO); | |
453d87f6 RC |
399 | |
400 | // mark_initmem_nx() should have already run by now | |
401 | ptdump_check_wx(); | |
95902e6c CL |
402 | } |
403 | #endif | |
404 | ||
3184cc4b | 405 | #ifdef CONFIG_DEBUG_PAGEALLOC |
031bc574 | 406 | void __kernel_map_pages(struct page *page, int numpages, int enable) |
88df6e90 BH |
407 | { |
408 | if (PageHighMem(page)) | |
409 | return; | |
410 | ||
411 | change_page_attr(page, numpages, enable ? PAGE_KERNEL : __pgprot(0)); | |
412 | } | |
413 | #endif /* CONFIG_DEBUG_PAGEALLOC */ |