Commit | Line | Data |
---|---|---|
3610cce8 | 1 | /* |
239a6425 | 2 | * Copyright IBM Corp. 2007,2009 |
3610cce8 MS |
3 | * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> |
4 | */ | |
5 | ||
6 | #include <linux/sched.h> | |
7 | #include <linux/kernel.h> | |
8 | #include <linux/errno.h> | |
5a0e3ad6 | 9 | #include <linux/gfp.h> |
3610cce8 MS |
10 | #include <linux/mm.h> |
11 | #include <linux/swap.h> | |
12 | #include <linux/smp.h> | |
13 | #include <linux/highmem.h> | |
3610cce8 MS |
14 | #include <linux/pagemap.h> |
15 | #include <linux/spinlock.h> | |
16 | #include <linux/module.h> | |
17 | #include <linux/quicklist.h> | |
80217147 | 18 | #include <linux/rcupdate.h> |
3610cce8 MS |
19 | |
20 | #include <asm/system.h> | |
21 | #include <asm/pgtable.h> | |
22 | #include <asm/pgalloc.h> | |
23 | #include <asm/tlb.h> | |
24 | #include <asm/tlbflush.h> | |
6252d702 | 25 | #include <asm/mmu_context.h> |
3610cce8 | 26 | |
80217147 MS |
27 | struct rcu_table_freelist { |
28 | struct rcu_head rcu; | |
29 | struct mm_struct *mm; | |
30 | unsigned int pgt_index; | |
31 | unsigned int crst_index; | |
32 | unsigned long *table[0]; | |
33 | }; | |
34 | ||
35 | #define RCU_FREELIST_SIZE \ | |
36 | ((PAGE_SIZE - sizeof(struct rcu_table_freelist)) \ | |
37 | / sizeof(unsigned long)) | |
38 | ||
39 | DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); | |
40 | static DEFINE_PER_CPU(struct rcu_table_freelist *, rcu_table_freelist); | |
41 | ||
42 | static void __page_table_free(struct mm_struct *mm, unsigned long *table); | |
43 | static void __crst_table_free(struct mm_struct *mm, unsigned long *table); | |
44 | ||
45 | static struct rcu_table_freelist *rcu_table_freelist_get(struct mm_struct *mm) | |
46 | { | |
47 | struct rcu_table_freelist **batchp = &__get_cpu_var(rcu_table_freelist); | |
48 | struct rcu_table_freelist *batch = *batchp; | |
49 | ||
50 | if (batch) | |
51 | return batch; | |
52 | batch = (struct rcu_table_freelist *) __get_free_page(GFP_ATOMIC); | |
53 | if (batch) { | |
54 | batch->mm = mm; | |
55 | batch->pgt_index = 0; | |
56 | batch->crst_index = RCU_FREELIST_SIZE; | |
57 | *batchp = batch; | |
58 | } | |
59 | return batch; | |
60 | } | |
61 | ||
62 | static void rcu_table_freelist_callback(struct rcu_head *head) | |
63 | { | |
64 | struct rcu_table_freelist *batch = | |
65 | container_of(head, struct rcu_table_freelist, rcu); | |
66 | ||
67 | while (batch->pgt_index > 0) | |
68 | __page_table_free(batch->mm, batch->table[--batch->pgt_index]); | |
69 | while (batch->crst_index < RCU_FREELIST_SIZE) | |
70 | __crst_table_free(batch->mm, batch->table[batch->crst_index++]); | |
71 | free_page((unsigned long) batch); | |
72 | } | |
73 | ||
74 | void rcu_table_freelist_finish(void) | |
75 | { | |
76 | struct rcu_table_freelist *batch = __get_cpu_var(rcu_table_freelist); | |
77 | ||
78 | if (!batch) | |
79 | return; | |
80 | call_rcu(&batch->rcu, rcu_table_freelist_callback); | |
81 | __get_cpu_var(rcu_table_freelist) = NULL; | |
82 | } | |
83 | ||
84 | static void smp_sync(void *arg) | |
85 | { | |
86 | } | |
87 | ||
3610cce8 MS |
88 | #ifndef CONFIG_64BIT |
89 | #define ALLOC_ORDER 1 | |
146e4b3c MS |
90 | #define TABLES_PER_PAGE 4 |
91 | #define FRAG_MASK 15UL | |
92 | #define SECOND_HALVES 10UL | |
402b0862 CO |
93 | |
94 | void clear_table_pgstes(unsigned long *table) | |
95 | { | |
96 | clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE/4); | |
97 | memset(table + 256, 0, PAGE_SIZE/4); | |
98 | clear_table(table + 512, _PAGE_TYPE_EMPTY, PAGE_SIZE/4); | |
99 | memset(table + 768, 0, PAGE_SIZE/4); | |
100 | } | |
101 | ||
3610cce8 MS |
102 | #else |
103 | #define ALLOC_ORDER 2 | |
146e4b3c MS |
104 | #define TABLES_PER_PAGE 2 |
105 | #define FRAG_MASK 3UL | |
106 | #define SECOND_HALVES 2UL | |
402b0862 CO |
107 | |
108 | void clear_table_pgstes(unsigned long *table) | |
109 | { | |
110 | clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE/2); | |
111 | memset(table + 256, 0, PAGE_SIZE/2); | |
112 | } | |
113 | ||
3610cce8 MS |
114 | #endif |
115 | ||
239a6425 HC |
116 | unsigned long VMALLOC_START = VMALLOC_END - VMALLOC_SIZE; |
117 | EXPORT_SYMBOL(VMALLOC_START); | |
118 | ||
119 | static int __init parse_vmalloc(char *arg) | |
120 | { | |
121 | if (!arg) | |
122 | return -EINVAL; | |
123 | VMALLOC_START = (VMALLOC_END - memparse(arg, &arg)) & PAGE_MASK; | |
124 | return 0; | |
125 | } | |
126 | early_param("vmalloc", parse_vmalloc); | |
127 | ||
3610cce8 MS |
128 | unsigned long *crst_table_alloc(struct mm_struct *mm, int noexec) |
129 | { | |
130 | struct page *page = alloc_pages(GFP_KERNEL, ALLOC_ORDER); | |
131 | ||
132 | if (!page) | |
133 | return NULL; | |
134 | page->index = 0; | |
135 | if (noexec) { | |
136 | struct page *shadow = alloc_pages(GFP_KERNEL, ALLOC_ORDER); | |
137 | if (!shadow) { | |
138 | __free_pages(page, ALLOC_ORDER); | |
139 | return NULL; | |
140 | } | |
141 | page->index = page_to_phys(shadow); | |
142 | } | |
80217147 | 143 | spin_lock_bh(&mm->context.list_lock); |
146e4b3c | 144 | list_add(&page->lru, &mm->context.crst_list); |
80217147 | 145 | spin_unlock_bh(&mm->context.list_lock); |
3610cce8 MS |
146 | return (unsigned long *) page_to_phys(page); |
147 | } | |
148 | ||
80217147 | 149 | static void __crst_table_free(struct mm_struct *mm, unsigned long *table) |
3610cce8 MS |
150 | { |
151 | unsigned long *shadow = get_shadow_table(table); | |
152 | ||
153 | if (shadow) | |
154 | free_pages((unsigned long) shadow, ALLOC_ORDER); | |
155 | free_pages((unsigned long) table, ALLOC_ORDER); | |
156 | } | |
157 | ||
80217147 MS |
158 | void crst_table_free(struct mm_struct *mm, unsigned long *table) |
159 | { | |
160 | struct page *page = virt_to_page(table); | |
161 | ||
162 | spin_lock_bh(&mm->context.list_lock); | |
163 | list_del(&page->lru); | |
164 | spin_unlock_bh(&mm->context.list_lock); | |
165 | __crst_table_free(mm, table); | |
166 | } | |
167 | ||
168 | void crst_table_free_rcu(struct mm_struct *mm, unsigned long *table) | |
169 | { | |
170 | struct rcu_table_freelist *batch; | |
171 | struct page *page = virt_to_page(table); | |
172 | ||
173 | spin_lock_bh(&mm->context.list_lock); | |
174 | list_del(&page->lru); | |
175 | spin_unlock_bh(&mm->context.list_lock); | |
176 | if (atomic_read(&mm->mm_users) < 2 && | |
177 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) { | |
178 | __crst_table_free(mm, table); | |
179 | return; | |
180 | } | |
181 | batch = rcu_table_freelist_get(mm); | |
182 | if (!batch) { | |
183 | smp_call_function(smp_sync, NULL, 1); | |
184 | __crst_table_free(mm, table); | |
185 | return; | |
186 | } | |
187 | batch->table[--batch->crst_index] = table; | |
188 | if (batch->pgt_index >= batch->crst_index) | |
189 | rcu_table_freelist_finish(); | |
190 | } | |
191 | ||
6252d702 MS |
192 | #ifdef CONFIG_64BIT |
193 | int crst_table_upgrade(struct mm_struct *mm, unsigned long limit) | |
194 | { | |
195 | unsigned long *table, *pgd; | |
196 | unsigned long entry; | |
197 | ||
198 | BUG_ON(limit > (1UL << 53)); | |
199 | repeat: | |
200 | table = crst_table_alloc(mm, mm->context.noexec); | |
201 | if (!table) | |
202 | return -ENOMEM; | |
80217147 | 203 | spin_lock_bh(&mm->page_table_lock); |
6252d702 MS |
204 | if (mm->context.asce_limit < limit) { |
205 | pgd = (unsigned long *) mm->pgd; | |
206 | if (mm->context.asce_limit <= (1UL << 31)) { | |
207 | entry = _REGION3_ENTRY_EMPTY; | |
208 | mm->context.asce_limit = 1UL << 42; | |
209 | mm->context.asce_bits = _ASCE_TABLE_LENGTH | | |
210 | _ASCE_USER_BITS | | |
211 | _ASCE_TYPE_REGION3; | |
212 | } else { | |
213 | entry = _REGION2_ENTRY_EMPTY; | |
214 | mm->context.asce_limit = 1UL << 53; | |
215 | mm->context.asce_bits = _ASCE_TABLE_LENGTH | | |
216 | _ASCE_USER_BITS | | |
217 | _ASCE_TYPE_REGION2; | |
218 | } | |
219 | crst_table_init(table, entry); | |
220 | pgd_populate(mm, (pgd_t *) table, (pud_t *) pgd); | |
221 | mm->pgd = (pgd_t *) table; | |
f481bfaf | 222 | mm->task_size = mm->context.asce_limit; |
6252d702 MS |
223 | table = NULL; |
224 | } | |
80217147 | 225 | spin_unlock_bh(&mm->page_table_lock); |
6252d702 MS |
226 | if (table) |
227 | crst_table_free(mm, table); | |
228 | if (mm->context.asce_limit < limit) | |
229 | goto repeat; | |
230 | update_mm(mm, current); | |
231 | return 0; | |
232 | } | |
233 | ||
234 | void crst_table_downgrade(struct mm_struct *mm, unsigned long limit) | |
235 | { | |
236 | pgd_t *pgd; | |
237 | ||
238 | if (mm->context.asce_limit <= limit) | |
239 | return; | |
240 | __tlb_flush_mm(mm); | |
241 | while (mm->context.asce_limit > limit) { | |
242 | pgd = mm->pgd; | |
243 | switch (pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) { | |
244 | case _REGION_ENTRY_TYPE_R2: | |
245 | mm->context.asce_limit = 1UL << 42; | |
246 | mm->context.asce_bits = _ASCE_TABLE_LENGTH | | |
247 | _ASCE_USER_BITS | | |
248 | _ASCE_TYPE_REGION3; | |
249 | break; | |
250 | case _REGION_ENTRY_TYPE_R3: | |
251 | mm->context.asce_limit = 1UL << 31; | |
252 | mm->context.asce_bits = _ASCE_TABLE_LENGTH | | |
253 | _ASCE_USER_BITS | | |
254 | _ASCE_TYPE_SEGMENT; | |
255 | break; | |
256 | default: | |
257 | BUG(); | |
258 | } | |
259 | mm->pgd = (pgd_t *) (pgd_val(*pgd) & _REGION_ENTRY_ORIGIN); | |
f481bfaf | 260 | mm->task_size = mm->context.asce_limit; |
6252d702 MS |
261 | crst_table_free(mm, (unsigned long *) pgd); |
262 | } | |
263 | update_mm(mm, current); | |
264 | } | |
265 | #endif | |
266 | ||
3610cce8 MS |
267 | /* |
268 | * page table entry allocation/free routines. | |
269 | */ | |
146e4b3c | 270 | unsigned long *page_table_alloc(struct mm_struct *mm) |
3610cce8 | 271 | { |
146e4b3c | 272 | struct page *page; |
3610cce8 | 273 | unsigned long *table; |
146e4b3c | 274 | unsigned long bits; |
3610cce8 | 275 | |
250cf776 | 276 | bits = (mm->context.noexec || mm->context.has_pgste) ? 3UL : 1UL; |
80217147 | 277 | spin_lock_bh(&mm->context.list_lock); |
146e4b3c MS |
278 | page = NULL; |
279 | if (!list_empty(&mm->context.pgtable_list)) { | |
280 | page = list_first_entry(&mm->context.pgtable_list, | |
281 | struct page, lru); | |
282 | if ((page->flags & FRAG_MASK) == ((1UL << TABLES_PER_PAGE) - 1)) | |
283 | page = NULL; | |
284 | } | |
285 | if (!page) { | |
80217147 | 286 | spin_unlock_bh(&mm->context.list_lock); |
146e4b3c MS |
287 | page = alloc_page(GFP_KERNEL|__GFP_REPEAT); |
288 | if (!page) | |
3610cce8 | 289 | return NULL; |
146e4b3c MS |
290 | pgtable_page_ctor(page); |
291 | page->flags &= ~FRAG_MASK; | |
292 | table = (unsigned long *) page_to_phys(page); | |
250cf776 | 293 | if (mm->context.has_pgste) |
402b0862 CO |
294 | clear_table_pgstes(table); |
295 | else | |
296 | clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE); | |
80217147 | 297 | spin_lock_bh(&mm->context.list_lock); |
146e4b3c | 298 | list_add(&page->lru, &mm->context.pgtable_list); |
3610cce8 MS |
299 | } |
300 | table = (unsigned long *) page_to_phys(page); | |
146e4b3c MS |
301 | while (page->flags & bits) { |
302 | table += 256; | |
303 | bits <<= 1; | |
304 | } | |
305 | page->flags |= bits; | |
306 | if ((page->flags & FRAG_MASK) == ((1UL << TABLES_PER_PAGE) - 1)) | |
307 | list_move_tail(&page->lru, &mm->context.pgtable_list); | |
80217147 | 308 | spin_unlock_bh(&mm->context.list_lock); |
3610cce8 MS |
309 | return table; |
310 | } | |
311 | ||
80217147 MS |
312 | static void __page_table_free(struct mm_struct *mm, unsigned long *table) |
313 | { | |
314 | struct page *page; | |
315 | unsigned long bits; | |
316 | ||
317 | bits = ((unsigned long) table) & 15; | |
318 | table = (unsigned long *)(((unsigned long) table) ^ bits); | |
319 | page = pfn_to_page(__pa(table) >> PAGE_SHIFT); | |
320 | page->flags ^= bits; | |
321 | if (!(page->flags & FRAG_MASK)) { | |
322 | pgtable_page_dtor(page); | |
323 | __free_page(page); | |
324 | } | |
325 | } | |
326 | ||
146e4b3c | 327 | void page_table_free(struct mm_struct *mm, unsigned long *table) |
3610cce8 | 328 | { |
146e4b3c MS |
329 | struct page *page; |
330 | unsigned long bits; | |
3610cce8 | 331 | |
250cf776 | 332 | bits = (mm->context.noexec || mm->context.has_pgste) ? 3UL : 1UL; |
146e4b3c MS |
333 | bits <<= (__pa(table) & (PAGE_SIZE - 1)) / 256 / sizeof(unsigned long); |
334 | page = pfn_to_page(__pa(table) >> PAGE_SHIFT); | |
80217147 | 335 | spin_lock_bh(&mm->context.list_lock); |
146e4b3c MS |
336 | page->flags ^= bits; |
337 | if (page->flags & FRAG_MASK) { | |
338 | /* Page now has some free pgtable fragments. */ | |
f1be77bb MS |
339 | if (!list_empty(&page->lru)) |
340 | list_move(&page->lru, &mm->context.pgtable_list); | |
146e4b3c MS |
341 | page = NULL; |
342 | } else | |
343 | /* All fragments of the 4K page have been freed. */ | |
344 | list_del(&page->lru); | |
80217147 | 345 | spin_unlock_bh(&mm->context.list_lock); |
146e4b3c MS |
346 | if (page) { |
347 | pgtable_page_dtor(page); | |
348 | __free_page(page); | |
349 | } | |
350 | } | |
3610cce8 | 351 | |
80217147 MS |
352 | void page_table_free_rcu(struct mm_struct *mm, unsigned long *table) |
353 | { | |
354 | struct rcu_table_freelist *batch; | |
355 | struct page *page; | |
356 | unsigned long bits; | |
357 | ||
358 | if (atomic_read(&mm->mm_users) < 2 && | |
359 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) { | |
360 | page_table_free(mm, table); | |
361 | return; | |
362 | } | |
363 | batch = rcu_table_freelist_get(mm); | |
364 | if (!batch) { | |
365 | smp_call_function(smp_sync, NULL, 1); | |
366 | page_table_free(mm, table); | |
367 | return; | |
368 | } | |
369 | bits = (mm->context.noexec || mm->context.has_pgste) ? 3UL : 1UL; | |
370 | bits <<= (__pa(table) & (PAGE_SIZE - 1)) / 256 / sizeof(unsigned long); | |
371 | page = pfn_to_page(__pa(table) >> PAGE_SHIFT); | |
372 | spin_lock_bh(&mm->context.list_lock); | |
373 | /* Delayed freeing with rcu prevents reuse of pgtable fragments */ | |
374 | list_del_init(&page->lru); | |
375 | spin_unlock_bh(&mm->context.list_lock); | |
376 | table = (unsigned long *)(((unsigned long) table) | bits); | |
377 | batch->table[batch->pgt_index++] = table; | |
378 | if (batch->pgt_index >= batch->crst_index) | |
379 | rcu_table_freelist_finish(); | |
380 | } | |
381 | ||
146e4b3c MS |
382 | void disable_noexec(struct mm_struct *mm, struct task_struct *tsk) |
383 | { | |
384 | struct page *page; | |
385 | ||
80217147 | 386 | spin_lock_bh(&mm->context.list_lock); |
146e4b3c MS |
387 | /* Free shadow region and segment tables. */ |
388 | list_for_each_entry(page, &mm->context.crst_list, lru) | |
389 | if (page->index) { | |
390 | free_pages((unsigned long) page->index, ALLOC_ORDER); | |
391 | page->index = 0; | |
392 | } | |
393 | /* "Free" second halves of page tables. */ | |
394 | list_for_each_entry(page, &mm->context.pgtable_list, lru) | |
395 | page->flags &= ~SECOND_HALVES; | |
80217147 | 396 | spin_unlock_bh(&mm->context.list_lock); |
146e4b3c MS |
397 | mm->context.noexec = 0; |
398 | update_mm(mm, tsk); | |
3610cce8 | 399 | } |
402b0862 CO |
400 | |
401 | /* | |
402 | * switch on pgstes for its userspace process (for kvm) | |
403 | */ | |
404 | int s390_enable_sie(void) | |
405 | { | |
406 | struct task_struct *tsk = current; | |
74b6b522 | 407 | struct mm_struct *mm, *old_mm; |
402b0862 | 408 | |
702d9e58 | 409 | /* Do we have switched amode? If no, we cannot do sie */ |
b11b5334 | 410 | if (user_mode == HOME_SPACE_MODE) |
702d9e58 CO |
411 | return -EINVAL; |
412 | ||
74b6b522 | 413 | /* Do we have pgstes? if yes, we are done */ |
250cf776 | 414 | if (tsk->mm->context.has_pgste) |
74b6b522 | 415 | return 0; |
402b0862 | 416 | |
74b6b522 CB |
417 | /* lets check if we are allowed to replace the mm */ |
418 | task_lock(tsk); | |
402b0862 | 419 | if (!tsk->mm || atomic_read(&tsk->mm->mm_users) > 1 || |
52a21f2c MS |
420 | #ifdef CONFIG_AIO |
421 | !hlist_empty(&tsk->mm->ioctx_list) || | |
422 | #endif | |
423 | tsk->mm != tsk->active_mm) { | |
74b6b522 CB |
424 | task_unlock(tsk); |
425 | return -EINVAL; | |
426 | } | |
427 | task_unlock(tsk); | |
402b0862 | 428 | |
250cf776 CB |
429 | /* we copy the mm and let dup_mm create the page tables with_pgstes */ |
430 | tsk->mm->context.alloc_pgste = 1; | |
402b0862 | 431 | mm = dup_mm(tsk); |
250cf776 | 432 | tsk->mm->context.alloc_pgste = 0; |
402b0862 | 433 | if (!mm) |
74b6b522 CB |
434 | return -ENOMEM; |
435 | ||
250cf776 | 436 | /* Now lets check again if something happened */ |
74b6b522 CB |
437 | task_lock(tsk); |
438 | if (!tsk->mm || atomic_read(&tsk->mm->mm_users) > 1 || | |
52a21f2c MS |
439 | #ifdef CONFIG_AIO |
440 | !hlist_empty(&tsk->mm->ioctx_list) || | |
441 | #endif | |
442 | tsk->mm != tsk->active_mm) { | |
74b6b522 CB |
443 | mmput(mm); |
444 | task_unlock(tsk); | |
445 | return -EINVAL; | |
446 | } | |
447 | ||
448 | /* ok, we are alone. No ptrace, no threads, etc. */ | |
449 | old_mm = tsk->mm; | |
402b0862 CO |
450 | tsk->mm = tsk->active_mm = mm; |
451 | preempt_disable(); | |
452 | update_mm(mm, tsk); | |
e05ef9bd CB |
453 | atomic_inc(&mm->context.attach_count); |
454 | atomic_dec(&old_mm->context.attach_count); | |
005f8eee | 455 | cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm)); |
402b0862 | 456 | preempt_enable(); |
402b0862 | 457 | task_unlock(tsk); |
74b6b522 CB |
458 | mmput(old_mm); |
459 | return 0; | |
402b0862 CO |
460 | } |
461 | EXPORT_SYMBOL_GPL(s390_enable_sie); | |
7db11a36 | 462 | |
87458ff4 | 463 | #if defined(CONFIG_DEBUG_PAGEALLOC) && defined(CONFIG_HIBERNATION) |
7db11a36 HJP |
464 | bool kernel_page_present(struct page *page) |
465 | { | |
466 | unsigned long addr; | |
467 | int cc; | |
468 | ||
469 | addr = page_to_phys(page); | |
87458ff4 HC |
470 | asm volatile( |
471 | " lra %1,0(%1)\n" | |
472 | " ipm %0\n" | |
473 | " srl %0,28" | |
474 | : "=d" (cc), "+a" (addr) : : "cc"); | |
7db11a36 HJP |
475 | return cc == 0; |
476 | } | |
87458ff4 | 477 | #endif /* CONFIG_HIBERNATION && CONFIG_DEBUG_PAGEALLOC */ |