Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
305b1523 JG |
2 | /* |
3 | * Copyright IBM Corp. 2011 | |
4 | * Author(s): Jan Glauber <jang@linux.vnet.ibm.com> | |
5 | */ | |
6b70a920 | 6 | #include <linux/hugetlb.h> |
305b1523 | 7 | #include <linux/mm.h> |
638ad34a | 8 | #include <asm/cacheflush.h> |
cfb0b241 | 9 | #include <asm/facility.h> |
305b1523 | 10 | #include <asm/pgtable.h> |
a01ef308 | 11 | #include <asm/pgalloc.h> |
6b70a920 | 12 | #include <asm/page.h> |
e6c7c630 | 13 | #include <asm/set_memory.h> |
6b70a920 | 14 | |
f7f8d7e5 HC |
15 | static inline unsigned long sske_frame(unsigned long addr, unsigned char skey) |
16 | { | |
17 | asm volatile(".insn rrf,0xb22b0000,%[skey],%[addr],9,0" | |
18 | : [addr] "+a" (addr) : [skey] "d" (skey)); | |
19 | return addr; | |
20 | } | |
21 | ||
127c1fef | 22 | void __storage_key_init_range(unsigned long start, unsigned long end) |
6b70a920 | 23 | { |
f7f8d7e5 | 24 | unsigned long boundary, size; |
6b70a920 | 25 | |
de3fa841 HC |
26 | if (!PAGE_DEFAULT_KEY) |
27 | return; | |
6b70a920 | 28 | while (start < end) { |
6b70a920 HC |
29 | if (MACHINE_HAS_EDAT1) { |
30 | /* set storage keys for a 1MB frame */ | |
6b70a920 HC |
31 | size = 1UL << 20; |
32 | boundary = (start + size) & ~(size - 1); | |
33 | if (boundary <= end) { | |
34 | do { | |
f7f8d7e5 | 35 | start = sske_frame(start, PAGE_DEFAULT_KEY); |
6b70a920 HC |
36 | } while (start < boundary); |
37 | continue; | |
38 | } | |
39 | } | |
40 | page_set_storage_key(start, PAGE_DEFAULT_KEY, 0); | |
41 | start += PAGE_SIZE; | |
42 | } | |
43 | } | |
305b1523 | 44 | |
37cd944c HC |
45 | #ifdef CONFIG_PROC_FS |
46 | atomic_long_t direct_pages_count[PG_DIRECT_MAP_MAX]; | |
47 | ||
48 | void arch_report_meminfo(struct seq_file *m) | |
49 | { | |
50 | seq_printf(m, "DirectMap4k: %8lu kB\n", | |
51 | atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_4K]) << 2); | |
52 | seq_printf(m, "DirectMap1M: %8lu kB\n", | |
53 | atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_1M]) << 10); | |
54 | seq_printf(m, "DirectMap2G: %8lu kB\n", | |
55 | atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_2G]) << 21); | |
56 | } | |
57 | #endif /* CONFIG_PROC_FS */ | |
58 | ||
e8a97e42 HC |
59 | static void pgt_set(unsigned long *old, unsigned long new, unsigned long addr, |
60 | unsigned long dtt) | |
5b1ba9e3 | 61 | { |
e8a97e42 HC |
62 | unsigned long table, mask; |
63 | ||
64 | mask = 0; | |
65 | if (MACHINE_HAS_EDAT2) { | |
66 | switch (dtt) { | |
67 | case CRDTE_DTT_REGION3: | |
68 | mask = ~(PTRS_PER_PUD * sizeof(pud_t) - 1); | |
69 | break; | |
70 | case CRDTE_DTT_SEGMENT: | |
71 | mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1); | |
72 | break; | |
73 | case CRDTE_DTT_PAGE: | |
74 | mask = ~(PTRS_PER_PTE * sizeof(pte_t) - 1); | |
75 | break; | |
76 | } | |
77 | table = (unsigned long)old & mask; | |
78 | crdte(*old, new, table, dtt, addr, S390_lowcore.kernel_asce); | |
79 | } else if (MACHINE_HAS_IDTE) { | |
80 | cspg(old, *old, new); | |
81 | } else { | |
82 | csp((unsigned int *)old + 1, *old, new); | |
83 | } | |
84 | } | |
85 | ||
e8a97e42 | 86 | static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end, |
57d7f939 | 87 | unsigned long flags) |
e8a97e42 HC |
88 | { |
89 | pte_t *ptep, new; | |
90 | ||
91 | ptep = pte_offset(pmdp, addr); | |
92 | do { | |
57d7f939 MS |
93 | new = *ptep; |
94 | if (pte_none(new)) | |
e8a97e42 | 95 | return -EINVAL; |
57d7f939 MS |
96 | if (flags & SET_MEMORY_RO) |
97 | new = pte_wrprotect(new); | |
98 | else if (flags & SET_MEMORY_RW) | |
99 | new = pte_mkwrite(pte_mkdirty(new)); | |
1366def3 | 100 | if (flags & SET_MEMORY_NX) |
57d7f939 MS |
101 | pte_val(new) |= _PAGE_NOEXEC; |
102 | else if (flags & SET_MEMORY_X) | |
103 | pte_val(new) &= ~_PAGE_NOEXEC; | |
e8a97e42 HC |
104 | pgt_set((unsigned long *)ptep, pte_val(new), addr, CRDTE_DTT_PAGE); |
105 | ptep++; | |
106 | addr += PAGE_SIZE; | |
107 | cond_resched(); | |
108 | } while (addr < end); | |
109 | return 0; | |
110 | } | |
111 | ||
112 | static int split_pmd_page(pmd_t *pmdp, unsigned long addr) | |
113 | { | |
114 | unsigned long pte_addr, prot; | |
115 | pte_t *pt_dir, *ptep; | |
116 | pmd_t new; | |
57d7f939 | 117 | int i, ro, nx; |
e8a97e42 HC |
118 | |
119 | pt_dir = vmem_pte_alloc(); | |
120 | if (!pt_dir) | |
121 | return -ENOMEM; | |
122 | pte_addr = pmd_pfn(*pmdp) << PAGE_SHIFT; | |
123 | ro = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT); | |
57d7f939 | 124 | nx = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_NOEXEC); |
e8a97e42 | 125 | prot = pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL); |
57d7f939 MS |
126 | if (!nx) |
127 | prot &= ~_PAGE_NOEXEC; | |
e8a97e42 HC |
128 | ptep = pt_dir; |
129 | for (i = 0; i < PTRS_PER_PTE; i++) { | |
130 | pte_val(*ptep) = pte_addr | prot; | |
131 | pte_addr += PAGE_SIZE; | |
132 | ptep++; | |
133 | } | |
134 | pmd_val(new) = __pa(pt_dir) | _SEGMENT_ENTRY; | |
135 | pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT); | |
37cd944c HC |
136 | update_page_count(PG_DIRECT_MAP_4K, PTRS_PER_PTE); |
137 | update_page_count(PG_DIRECT_MAP_1M, -1); | |
e8a97e42 HC |
138 | return 0; |
139 | } | |
140 | ||
57d7f939 MS |
141 | static void modify_pmd_page(pmd_t *pmdp, unsigned long addr, |
142 | unsigned long flags) | |
e8a97e42 | 143 | { |
57d7f939 MS |
144 | pmd_t new = *pmdp; |
145 | ||
146 | if (flags & SET_MEMORY_RO) | |
147 | new = pmd_wrprotect(new); | |
148 | else if (flags & SET_MEMORY_RW) | |
149 | new = pmd_mkwrite(pmd_mkdirty(new)); | |
1366def3 | 150 | if (flags & SET_MEMORY_NX) |
57d7f939 MS |
151 | pmd_val(new) |= _SEGMENT_ENTRY_NOEXEC; |
152 | else if (flags & SET_MEMORY_X) | |
153 | pmd_val(new) &= ~_SEGMENT_ENTRY_NOEXEC; | |
e8a97e42 HC |
154 | pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT); |
155 | } | |
156 | ||
157 | static int walk_pmd_level(pud_t *pudp, unsigned long addr, unsigned long end, | |
57d7f939 | 158 | unsigned long flags) |
e8a97e42 HC |
159 | { |
160 | unsigned long next; | |
5b1ba9e3 | 161 | pmd_t *pmdp; |
e8a97e42 | 162 | int rc = 0; |
5b1ba9e3 | 163 | |
5b1ba9e3 | 164 | pmdp = pmd_offset(pudp, addr); |
e8a97e42 HC |
165 | do { |
166 | if (pmd_none(*pmdp)) | |
167 | return -EINVAL; | |
168 | next = pmd_addr_end(addr, end); | |
169 | if (pmd_large(*pmdp)) { | |
170 | if (addr & ~PMD_MASK || addr + PMD_SIZE > next) { | |
171 | rc = split_pmd_page(pmdp, addr); | |
172 | if (rc) | |
173 | return rc; | |
174 | continue; | |
175 | } | |
57d7f939 | 176 | modify_pmd_page(pmdp, addr, flags); |
e8a97e42 | 177 | } else { |
57d7f939 | 178 | rc = walk_pte_level(pmdp, addr, next, flags); |
e8a97e42 HC |
179 | if (rc) |
180 | return rc; | |
181 | } | |
182 | pmdp++; | |
183 | addr = next; | |
184 | cond_resched(); | |
185 | } while (addr < end); | |
186 | return rc; | |
5b1ba9e3 HC |
187 | } |
188 | ||
e8a97e42 | 189 | static int split_pud_page(pud_t *pudp, unsigned long addr) |
305b1523 | 190 | { |
e8a97e42 HC |
191 | unsigned long pmd_addr, prot; |
192 | pmd_t *pm_dir, *pmdp; | |
193 | pud_t new; | |
57d7f939 | 194 | int i, ro, nx; |
305b1523 | 195 | |
a01ef308 | 196 | pm_dir = vmem_crst_alloc(_SEGMENT_ENTRY_EMPTY); |
e8a97e42 HC |
197 | if (!pm_dir) |
198 | return -ENOMEM; | |
199 | pmd_addr = pud_pfn(*pudp) << PAGE_SHIFT; | |
200 | ro = !!(pud_val(*pudp) & _REGION_ENTRY_PROTECT); | |
57d7f939 | 201 | nx = !!(pud_val(*pudp) & _REGION_ENTRY_NOEXEC); |
e8a97e42 | 202 | prot = pgprot_val(ro ? SEGMENT_KERNEL_RO : SEGMENT_KERNEL); |
57d7f939 MS |
203 | if (!nx) |
204 | prot &= ~_SEGMENT_ENTRY_NOEXEC; | |
e8a97e42 HC |
205 | pmdp = pm_dir; |
206 | for (i = 0; i < PTRS_PER_PMD; i++) { | |
207 | pmd_val(*pmdp) = pmd_addr | prot; | |
208 | pmd_addr += PMD_SIZE; | |
209 | pmdp++; | |
305b1523 | 210 | } |
e8a97e42 HC |
211 | pud_val(new) = __pa(pm_dir) | _REGION3_ENTRY; |
212 | pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3); | |
37cd944c HC |
213 | update_page_count(PG_DIRECT_MAP_1M, PTRS_PER_PMD); |
214 | update_page_count(PG_DIRECT_MAP_2G, -1); | |
e8a97e42 HC |
215 | return 0; |
216 | } | |
217 | ||
57d7f939 MS |
218 | static void modify_pud_page(pud_t *pudp, unsigned long addr, |
219 | unsigned long flags) | |
e8a97e42 | 220 | { |
57d7f939 MS |
221 | pud_t new = *pudp; |
222 | ||
223 | if (flags & SET_MEMORY_RO) | |
224 | new = pud_wrprotect(new); | |
225 | else if (flags & SET_MEMORY_RW) | |
226 | new = pud_mkwrite(pud_mkdirty(new)); | |
1366def3 | 227 | if (flags & SET_MEMORY_NX) |
57d7f939 MS |
228 | pud_val(new) |= _REGION_ENTRY_NOEXEC; |
229 | else if (flags & SET_MEMORY_X) | |
230 | pud_val(new) &= ~_REGION_ENTRY_NOEXEC; | |
e8a97e42 HC |
231 | pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3); |
232 | } | |
233 | ||
1aea9b3f | 234 | static int walk_pud_level(p4d_t *p4d, unsigned long addr, unsigned long end, |
57d7f939 | 235 | unsigned long flags) |
e8a97e42 HC |
236 | { |
237 | unsigned long next; | |
238 | pud_t *pudp; | |
239 | int rc = 0; | |
240 | ||
1aea9b3f | 241 | pudp = pud_offset(p4d, addr); |
e8a97e42 HC |
242 | do { |
243 | if (pud_none(*pudp)) | |
244 | return -EINVAL; | |
245 | next = pud_addr_end(addr, end); | |
246 | if (pud_large(*pudp)) { | |
247 | if (addr & ~PUD_MASK || addr + PUD_SIZE > next) { | |
248 | rc = split_pud_page(pudp, addr); | |
249 | if (rc) | |
250 | break; | |
251 | continue; | |
252 | } | |
57d7f939 | 253 | modify_pud_page(pudp, addr, flags); |
e8a97e42 | 254 | } else { |
57d7f939 | 255 | rc = walk_pmd_level(pudp, addr, next, flags); |
e8a97e42 HC |
256 | } |
257 | pudp++; | |
258 | addr = next; | |
259 | cond_resched(); | |
260 | } while (addr < end && !rc); | |
261 | return rc; | |
262 | } | |
263 | ||
1aea9b3f MS |
264 | static int walk_p4d_level(pgd_t *pgd, unsigned long addr, unsigned long end, |
265 | unsigned long flags) | |
266 | { | |
267 | unsigned long next; | |
268 | p4d_t *p4dp; | |
269 | int rc = 0; | |
270 | ||
271 | p4dp = p4d_offset(pgd, addr); | |
272 | do { | |
273 | if (p4d_none(*p4dp)) | |
274 | return -EINVAL; | |
275 | next = p4d_addr_end(addr, end); | |
276 | rc = walk_pud_level(p4dp, addr, next, flags); | |
277 | p4dp++; | |
278 | addr = next; | |
279 | cond_resched(); | |
280 | } while (addr < end && !rc); | |
281 | return rc; | |
282 | } | |
283 | ||
e8a97e42 HC |
284 | static DEFINE_MUTEX(cpa_mutex); |
285 | ||
286 | static int change_page_attr(unsigned long addr, unsigned long end, | |
57d7f939 | 287 | unsigned long flags) |
e8a97e42 HC |
288 | { |
289 | unsigned long next; | |
290 | int rc = -EINVAL; | |
291 | pgd_t *pgdp; | |
292 | ||
4d81aaa5 HC |
293 | if (addr == end) |
294 | return 0; | |
e8a97e42 HC |
295 | if (end >= MODULES_END) |
296 | return -EINVAL; | |
297 | mutex_lock(&cpa_mutex); | |
298 | pgdp = pgd_offset_k(addr); | |
299 | do { | |
300 | if (pgd_none(*pgdp)) | |
301 | break; | |
302 | next = pgd_addr_end(addr, end); | |
1aea9b3f | 303 | rc = walk_p4d_level(pgdp, addr, next, flags); |
e8a97e42 HC |
304 | if (rc) |
305 | break; | |
306 | cond_resched(); | |
307 | } while (pgdp++, addr = next, addr < end && !rc); | |
308 | mutex_unlock(&cpa_mutex); | |
309 | return rc; | |
305b1523 JG |
310 | } |
311 | ||
57d7f939 | 312 | int __set_memory(unsigned long addr, int numpages, unsigned long flags) |
305b1523 | 313 | { |
1366def3 HC |
314 | if (!MACHINE_HAS_NX) |
315 | flags &= ~(SET_MEMORY_NX | SET_MEMORY_X); | |
316 | if (!flags) | |
317 | return 0; | |
e8a97e42 | 318 | addr &= PAGE_MASK; |
57d7f939 | 319 | return change_page_attr(addr, addr + numpages * PAGE_SIZE, flags); |
448694a1 | 320 | } |
0a4ccc99 HC |
321 | |
322 | #ifdef CONFIG_DEBUG_PAGEALLOC | |
cfb0b241 HC |
323 | |
324 | static void ipte_range(pte_t *pte, unsigned long address, int nr) | |
325 | { | |
326 | int i; | |
327 | ||
5a79859a | 328 | if (test_facility(13)) { |
34eeaf37 | 329 | __ptep_ipte_range(address, nr - 1, pte, IPTE_GLOBAL); |
cfb0b241 HC |
330 | return; |
331 | } | |
332 | for (i = 0; i < nr; i++) { | |
28c807e5 | 333 | __ptep_ipte(address, pte, 0, 0, IPTE_GLOBAL); |
cfb0b241 HC |
334 | address += PAGE_SIZE; |
335 | pte++; | |
336 | } | |
337 | } | |
338 | ||
031bc574 | 339 | void __kernel_map_pages(struct page *page, int numpages, int enable) |
0a4ccc99 HC |
340 | { |
341 | unsigned long address; | |
cfb0b241 | 342 | int nr, i, j; |
0a4ccc99 | 343 | pgd_t *pgd; |
1aea9b3f | 344 | p4d_t *p4d; |
0a4ccc99 HC |
345 | pud_t *pud; |
346 | pmd_t *pmd; | |
347 | pte_t *pte; | |
0a4ccc99 | 348 | |
cfb0b241 | 349 | for (i = 0; i < numpages;) { |
0a4ccc99 HC |
350 | address = page_to_phys(page + i); |
351 | pgd = pgd_offset_k(address); | |
1aea9b3f MS |
352 | p4d = p4d_offset(pgd, address); |
353 | pud = pud_offset(p4d, address); | |
0a4ccc99 HC |
354 | pmd = pmd_offset(pud, address); |
355 | pte = pte_offset_kernel(pmd, address); | |
cfb0b241 HC |
356 | nr = (unsigned long)pte >> ilog2(sizeof(long)); |
357 | nr = PTRS_PER_PTE - (nr & (PTRS_PER_PTE - 1)); | |
358 | nr = min(numpages - i, nr); | |
359 | if (enable) { | |
360 | for (j = 0; j < nr; j++) { | |
57d7f939 | 361 | pte_val(*pte) &= ~_PAGE_INVALID; |
cfb0b241 HC |
362 | address += PAGE_SIZE; |
363 | pte++; | |
364 | } | |
365 | } else { | |
366 | ipte_range(pte, address, nr); | |
0a4ccc99 | 367 | } |
cfb0b241 | 368 | i += nr; |
0a4ccc99 HC |
369 | } |
370 | } | |
371 | ||
372 | #ifdef CONFIG_HIBERNATION | |
373 | bool kernel_page_present(struct page *page) | |
374 | { | |
375 | unsigned long addr; | |
376 | int cc; | |
377 | ||
378 | addr = page_to_phys(page); | |
379 | asm volatile( | |
380 | " lra %1,0(%1)\n" | |
381 | " ipm %0\n" | |
382 | " srl %0,28" | |
383 | : "=d" (cc), "+a" (addr) : : "cc"); | |
384 | return cc == 0; | |
385 | } | |
386 | #endif /* CONFIG_HIBERNATION */ | |
387 | ||
388 | #endif /* CONFIG_DEBUG_PAGEALLOC */ |