projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
powerpc: handle pgtable_page_ctor() fail
[linux-2.6-block.git] / arch / s390 / mm / pgtable.c
CommitLineData
3610cce8 1/*
a53c8fab 2 * Copyright IBM Corp. 2007, 2011
3610cce8
MS		 3 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
4 */
5
6#include <linux/sched.h>
7#include <linux/kernel.h>
8#include <linux/errno.h>
5a0e3ad6 9#include <linux/gfp.h>
3610cce8
MS		 10#include <linux/mm.h>
11#include <linux/swap.h>
12#include <linux/smp.h>
13#include <linux/highmem.h>
3610cce8
MS		 14#include <linux/pagemap.h>
15#include <linux/spinlock.h>
16#include <linux/module.h>
17#include <linux/quicklist.h>
80217147 18#include <linux/rcupdate.h>
e5992f2e 19#include <linux/slab.h>
3610cce8 20
3610cce8
MS		 21#include <asm/pgtable.h>
22#include <asm/pgalloc.h>
23#include <asm/tlb.h>
24#include <asm/tlbflush.h>
6252d702 25#include <asm/mmu_context.h>
3610cce8
MS		 26
27#ifndef CONFIG_64BIT
28#define ALLOC_ORDER 1
36409f63 29#define FRAG_MASK 0x0f
3610cce8
MS		 30#else
31#define ALLOC_ORDER 2
36409f63 32#define FRAG_MASK 0x03
3610cce8
MS		 33#endif
34
239a6425 35
043d0708 36unsigned long *crst_table_alloc(struct mm_struct *mm)
3610cce8
MS		 37{
38 struct page *page = alloc_pages(GFP_KERNEL, ALLOC_ORDER);
39
40 if (!page)
41 return NULL;
3610cce8
MS		 42 return (unsigned long *) page_to_phys(page);
43}
44
80217147
MS		 45void crst_table_free(struct mm_struct *mm, unsigned long *table)
46{
043d0708 47 free_pages((unsigned long) table, ALLOC_ORDER);
80217147
MS		 48}
49
6252d702 50#ifdef CONFIG_64BIT
10607864
MS		 51static void __crst_table_upgrade(void *arg)
52{
53 struct mm_struct *mm = arg;
54
55 if (current->active_mm == mm)
56 update_mm(mm, current);
57 __tlb_flush_local();
58}
59
6252d702
MS		 60int crst_table_upgrade(struct mm_struct *mm, unsigned long limit)
61{
62 unsigned long *table, *pgd;
63 unsigned long entry;
10607864 64 int flush;
6252d702
MS		 65
66 BUG_ON(limit > (1UL << 53));
10607864 67 flush = 0;
6252d702 68repeat:
043d0708 69 table = crst_table_alloc(mm);
6252d702
MS		 70 if (!table)
71 return -ENOMEM;
80217147 72 spin_lock_bh(&mm->page_table_lock);
6252d702
MS		 73 if (mm->context.asce_limit < limit) {
74 pgd = (unsigned long *) mm->pgd;
75 if (mm->context.asce_limit <= (1UL << 31)) {
76 entry = _REGION3_ENTRY_EMPTY;
77 mm->context.asce_limit = 1UL << 42;
78 mm->context.asce_bits = _ASCE_TABLE_LENGTH |
79 _ASCE_USER_BITS |
80 _ASCE_TYPE_REGION3;
81 } else {
82 entry = _REGION2_ENTRY_EMPTY;
83 mm->context.asce_limit = 1UL << 53;
84 mm->context.asce_bits = _ASCE_TABLE_LENGTH |
85 _ASCE_USER_BITS |
86 _ASCE_TYPE_REGION2;
87 }
88 crst_table_init(table, entry);
89 pgd_populate(mm, (pgd_t *) table, (pud_t *) pgd);
90 mm->pgd = (pgd_t *) table;
f481bfaf 91 mm->task_size = mm->context.asce_limit;
6252d702 92 table = NULL;
10607864 93 flush = 1;
6252d702 94 }
80217147 95 spin_unlock_bh(&mm->page_table_lock);
6252d702
MS		 96 if (table)
97 crst_table_free(mm, table);
98 if (mm->context.asce_limit < limit)
99 goto repeat;
10607864
MS		 100 if (flush)
101 on_each_cpu(__crst_table_upgrade, mm, 0);
6252d702
MS		 102 return 0;
103}
104
105void crst_table_downgrade(struct mm_struct *mm, unsigned long limit)
106{
107 pgd_t *pgd;
108
10607864
MS		 109 if (current->active_mm == mm)
110 __tlb_flush_mm(mm);
6252d702
MS		 111 while (mm->context.asce_limit > limit) {
112 pgd = mm->pgd;
113 switch (pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) {
114 case _REGION_ENTRY_TYPE_R2:
115 mm->context.asce_limit = 1UL << 42;
116 mm->context.asce_bits = _ASCE_TABLE_LENGTH |
117 _ASCE_USER_BITS |
118 _ASCE_TYPE_REGION3;
119 break;
120 case _REGION_ENTRY_TYPE_R3:
121 mm->context.asce_limit = 1UL << 31;
122 mm->context.asce_bits = _ASCE_TABLE_LENGTH |
123 _ASCE_USER_BITS |
124 _ASCE_TYPE_SEGMENT;
125 break;
126 default:
127 BUG();
128 }
129 mm->pgd = (pgd_t *) (pgd_val(*pgd) & _REGION_ENTRY_ORIGIN);
f481bfaf 130 mm->task_size = mm->context.asce_limit;
6252d702
MS		 131 crst_table_free(mm, (unsigned long *) pgd);
132 }
10607864
MS		 133 if (current->active_mm == mm)
134 update_mm(mm, current);
6252d702
MS		 135}
136#endif
137
e5992f2e
MS		 138#ifdef CONFIG_PGSTE
139
140/**
141 * gmap_alloc - allocate a guest address space
142 * @mm: pointer to the parent mm_struct
143 *
144 * Returns a guest address space structure.
145 */
146struct gmap *gmap_alloc(struct mm_struct *mm)
36409f63 147{
e5992f2e
MS		 148 struct gmap *gmap;
149 struct page *page;
150 unsigned long *table;
36409f63 151
e5992f2e
MS		 152 gmap = kzalloc(sizeof(struct gmap), GFP_KERNEL);
153 if (!gmap)
154 goto out;
155 INIT_LIST_HEAD(&gmap->crst_list);
156 gmap->mm = mm;
157 page = alloc_pages(GFP_KERNEL, ALLOC_ORDER);
158 if (!page)
159 goto out_free;
160 list_add(&page->lru, &gmap->crst_list);
161 table = (unsigned long *) page_to_phys(page);
162 crst_table_init(table, _REGION1_ENTRY_EMPTY);
163 gmap->table = table;
480e5926
CB		 164 gmap->asce = _ASCE_TYPE_REGION1 | _ASCE_TABLE_LENGTH |
165 _ASCE_USER_BITS | __pa(table);
e5992f2e
MS		 166 list_add(&gmap->list, &mm->context.gmap_list);
167 return gmap;
168
169out_free:
170 kfree(gmap);
171out:
172 return NULL;
36409f63 173}
e5992f2e 174EXPORT_SYMBOL_GPL(gmap_alloc);
36409f63 175
e5992f2e
MS		 176static int gmap_unlink_segment(struct gmap *gmap, unsigned long *table)
177{
178 struct gmap_pgtable *mp;
179 struct gmap_rmap *rmap;
180 struct page *page;
181
e5098611 182 if (*table & _SEGMENT_ENTRY_INVALID)
e5992f2e
MS		 183 return 0;
184 page = pfn_to_page(*table >> PAGE_SHIFT);
185 mp = (struct gmap_pgtable *) page->index;
186 list_for_each_entry(rmap, &mp->mapper, list) {
187 if (rmap->entry != table)
188 continue;
189 list_del(&rmap->list);
190 kfree(rmap);
191 break;
192 }
e5098611 193 *table = mp->vmaddr | _SEGMENT_ENTRY_INVALID | _SEGMENT_ENTRY_PROTECT;
e5992f2e
MS		 194 return 1;
195}
196
197static void gmap_flush_tlb(struct gmap *gmap)
198{
199 if (MACHINE_HAS_IDTE)
200 __tlb_flush_idte((unsigned long) gmap->table |
201 _ASCE_TYPE_REGION1);
202 else
203 __tlb_flush_global();
204}
205
206/**
207 * gmap_free - free a guest address space
208 * @gmap: pointer to the guest address space structure
3610cce8 209 */
e5992f2e
MS		 210void gmap_free(struct gmap *gmap)
211{
212 struct page *page, *next;
213 unsigned long *table;
214 int i;
215
216
217 /* Flush tlb. */
218 if (MACHINE_HAS_IDTE)
219 __tlb_flush_idte((unsigned long) gmap->table |
220 _ASCE_TYPE_REGION1);
221 else
222 __tlb_flush_global();
223
224 /* Free all segment & region tables. */
225 down_read(&gmap->mm->mmap_sem);
cc772456 226 spin_lock(&gmap->mm->page_table_lock);
e5992f2e
MS		 227 list_for_each_entry_safe(page, next, &gmap->crst_list, lru) {
228 table = (unsigned long *) page_to_phys(page);
229 if ((*table & _REGION_ENTRY_TYPE_MASK) == 0)
230 /* Remove gmap rmap structures for segment table. */
231 for (i = 0; i < PTRS_PER_PMD; i++, table++)
232 gmap_unlink_segment(gmap, table);
233 __free_pages(page, ALLOC_ORDER);
234 }
cc772456 235 spin_unlock(&gmap->mm->page_table_lock);
e5992f2e
MS		 236 up_read(&gmap->mm->mmap_sem);
237 list_del(&gmap->list);
238 kfree(gmap);
239}
240EXPORT_SYMBOL_GPL(gmap_free);
241
242/**
243 * gmap_enable - switch primary space to the guest address space
244 * @gmap: pointer to the guest address space structure
245 */
246void gmap_enable(struct gmap *gmap)
247{
e5992f2e
MS		 248 S390_lowcore.gmap = (unsigned long) gmap;
249}
250EXPORT_SYMBOL_GPL(gmap_enable);
251
252/**
253 * gmap_disable - switch back to the standard primary address space
254 * @gmap: pointer to the guest address space structure
255 */
256void gmap_disable(struct gmap *gmap)
257{
e5992f2e
MS		 258 S390_lowcore.gmap = 0UL;
259}
260EXPORT_SYMBOL_GPL(gmap_disable);
261
a9162f23
CO		 262/*
263 * gmap_alloc_table is assumed to be called with mmap_sem held
264 */
e5992f2e 265static int gmap_alloc_table(struct gmap *gmap,
984e2a59
HC		 266 unsigned long *table, unsigned long init)
267 __releases(&gmap->mm->page_table_lock)
268 __acquires(&gmap->mm->page_table_lock)
e5992f2e
MS		 269{
270 struct page *page;
271 unsigned long *new;
272
c86cce2a
CB		 273 /* since we dont free the gmap table until gmap_free we can unlock */
274 spin_unlock(&gmap->mm->page_table_lock);
e5992f2e 275 page = alloc_pages(GFP_KERNEL, ALLOC_ORDER);
c86cce2a 276 spin_lock(&gmap->mm->page_table_lock);
e5992f2e
MS		 277 if (!page)
278 return -ENOMEM;
279 new = (unsigned long *) page_to_phys(page);
280 crst_table_init(new, init);
e5098611 281 if (*table & _REGION_ENTRY_INVALID) {
e5992f2e
MS		 282 list_add(&page->lru, &gmap->crst_list);
283 *table = (unsigned long) new | _REGION_ENTRY_LENGTH |
284 (*table & _REGION_ENTRY_TYPE_MASK);
285 } else
286 __free_pages(page, ALLOC_ORDER);
e5992f2e
MS		 287 return 0;
288}
289
290/**
291 * gmap_unmap_segment - unmap segment from the guest address space
292 * @gmap: pointer to the guest address space structure
293 * @addr: address in the guest address space
294 * @len: length of the memory area to unmap
295 *
296 * Returns 0 if the unmap succeded, -EINVAL if not.
297 */
298int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len)
299{
300 unsigned long *table;
301 unsigned long off;
302 int flush;
303
304 if ((to | len) & (PMD_SIZE - 1))
305 return -EINVAL;
306 if (len == 0 || to + len < to)
307 return -EINVAL;
308
309 flush = 0;
310 down_read(&gmap->mm->mmap_sem);
cc772456 311 spin_lock(&gmap->mm->page_table_lock);
e5992f2e
MS		 312 for (off = 0; off < len; off += PMD_SIZE) {
313 /* Walk the guest addr space page table */
314 table = gmap->table + (((to + off) >> 53) & 0x7ff);
e5098611 315 if (*table & _REGION_ENTRY_INVALID)
05873df9 316 goto out;
e5992f2e
MS		 317 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
318 table = table + (((to + off) >> 42) & 0x7ff);
e5098611 319 if (*table & _REGION_ENTRY_INVALID)
05873df9 320 goto out;
e5992f2e
MS		 321 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
322 table = table + (((to + off) >> 31) & 0x7ff);
e5098611 323 if (*table & _REGION_ENTRY_INVALID)
05873df9 324 goto out;
e5992f2e
MS		 325 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
326 table = table + (((to + off) >> 20) & 0x7ff);
327
328 /* Clear segment table entry in guest address space. */
329 flush |= gmap_unlink_segment(gmap, table);
e5098611 330 *table = _SEGMENT_ENTRY_INVALID;
e5992f2e 331 }
05873df9 332out:
cc772456 333 spin_unlock(&gmap->mm->page_table_lock);
e5992f2e
MS		 334 up_read(&gmap->mm->mmap_sem);
335 if (flush)
336 gmap_flush_tlb(gmap);
337 return 0;
338}
339EXPORT_SYMBOL_GPL(gmap_unmap_segment);
340
341/**
342 * gmap_mmap_segment - map a segment to the guest address space
343 * @gmap: pointer to the guest address space structure
344 * @from: source address in the parent address space
345 * @to: target address in the guest address space
346 *
347 * Returns 0 if the mmap succeded, -EINVAL or -ENOMEM if not.
348 */
349int gmap_map_segment(struct gmap *gmap, unsigned long from,
350 unsigned long to, unsigned long len)
351{
352 unsigned long *table;
353 unsigned long off;
354 int flush;
355
356 if ((from | to | len) & (PMD_SIZE - 1))
357 return -EINVAL;
ee6ee55b 358 if (len == 0 || from + len > TASK_MAX_SIZE ||
e5992f2e
MS		 359 from + len < from || to + len < to)
360 return -EINVAL;
361
362 flush = 0;
363 down_read(&gmap->mm->mmap_sem);
cc772456 364 spin_lock(&gmap->mm->page_table_lock);
e5992f2e
MS		 365 for (off = 0; off < len; off += PMD_SIZE) {
366 /* Walk the gmap address space page table */
367 table = gmap->table + (((to + off) >> 53) & 0x7ff);
e5098611 368 if ((*table & _REGION_ENTRY_INVALID) &&
e5992f2e
MS		 369 gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY))
370 goto out_unmap;
371 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
372 table = table + (((to + off) >> 42) & 0x7ff);
e5098611 373 if ((*table & _REGION_ENTRY_INVALID) &&
e5992f2e
MS		 374 gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY))
375 goto out_unmap;
376 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
377 table = table + (((to + off) >> 31) & 0x7ff);
e5098611 378 if ((*table & _REGION_ENTRY_INVALID) &&
e5992f2e
MS		 379 gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY))
380 goto out_unmap;
381 table = (unsigned long *) (*table & _REGION_ENTRY_ORIGIN);
382 table = table + (((to + off) >> 20) & 0x7ff);
383
384 /* Store 'from' address in an invalid segment table entry. */
385 flush |= gmap_unlink_segment(gmap, table);
e5098611
MS		 386 *table = (from + off) | (_SEGMENT_ENTRY_INVALID |
387 _SEGMENT_ENTRY_PROTECT);
e5992f2e 388 }
cc772456 389 spin_unlock(&gmap->mm->page_table_lock);
e5992f2e
MS		 390 up_read(&gmap->mm->mmap_sem);
391 if (flush)
392 gmap_flush_tlb(gmap);
393 return 0;
394
395out_unmap:
cc772456 396 spin_unlock(&gmap->mm->page_table_lock);
e5992f2e
MS		 397 up_read(&gmap->mm->mmap_sem);
398 gmap_unmap_segment(gmap, to, len);
399 return -ENOMEM;
400}
401EXPORT_SYMBOL_GPL(gmap_map_segment);
402
c5034945
HC		 403static unsigned long *gmap_table_walk(unsigned long address, struct gmap *gmap)
404{
405 unsigned long *table;
406
407 table = gmap->table + ((address >> 53) & 0x7ff);
e5098611 408 if (unlikely(*table & _REGION_ENTRY_INVALID))
c5034945
HC		 409 return ERR_PTR(-EFAULT);
410 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
411 table = table + ((address >> 42) & 0x7ff);
e5098611 412 if (unlikely(*table & _REGION_ENTRY_INVALID))
c5034945
HC		 413 return ERR_PTR(-EFAULT);
414 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
415 table = table + ((address >> 31) & 0x7ff);
e5098611 416 if (unlikely(*table & _REGION_ENTRY_INVALID))
c5034945
HC		 417 return ERR_PTR(-EFAULT);
418 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
419 table = table + ((address >> 20) & 0x7ff);
420 return table;
421}
422
423/**
424 * __gmap_translate - translate a guest address to a user space address
425 * @address: guest address
426 * @gmap: pointer to guest mapping meta data structure
427 *
428 * Returns user space address which corresponds to the guest address or
429 * -EFAULT if no such mapping exists.
430 * This function does not establish potentially missing page table entries.
431 * The mmap_sem of the mm that belongs to the address space must be held
432 * when this function gets called.
433 */
434unsigned long __gmap_translate(unsigned long address, struct gmap *gmap)
435{
436 unsigned long *segment_ptr, vmaddr, segment;
437 struct gmap_pgtable *mp;
438 struct page *page;
439
440 current->thread.gmap_addr = address;
441 segment_ptr = gmap_table_walk(address, gmap);
442 if (IS_ERR(segment_ptr))
443 return PTR_ERR(segment_ptr);
444 /* Convert the gmap address to an mm address. */
445 segment = *segment_ptr;
e5098611 446 if (!(segment & _SEGMENT_ENTRY_INVALID)) {
c5034945
HC		 447 page = pfn_to_page(segment >> PAGE_SHIFT);
448 mp = (struct gmap_pgtable *) page->index;
449 return mp->vmaddr | (address & ~PMD_MASK);
e5098611 450 } else if (segment & _SEGMENT_ENTRY_PROTECT) {
c5034945
HC		 451 vmaddr = segment & _SEGMENT_ENTRY_ORIGIN;
452 return vmaddr | (address & ~PMD_MASK);
453 }
454 return -EFAULT;
455}
456EXPORT_SYMBOL_GPL(__gmap_translate);
457
458/**
459 * gmap_translate - translate a guest address to a user space address
460 * @address: guest address
461 * @gmap: pointer to guest mapping meta data structure
462 *
463 * Returns user space address which corresponds to the guest address or
464 * -EFAULT if no such mapping exists.
465 * This function does not establish potentially missing page table entries.
466 */
467unsigned long gmap_translate(unsigned long address, struct gmap *gmap)
468{
469 unsigned long rc;
470
471 down_read(&gmap->mm->mmap_sem);
472 rc = __gmap_translate(address, gmap);
473 up_read(&gmap->mm->mmap_sem);
474 return rc;
475}
476EXPORT_SYMBOL_GPL(gmap_translate);
477
d3383632
MS		 478static int gmap_connect_pgtable(unsigned long address, unsigned long segment,
479 unsigned long *segment_ptr, struct gmap *gmap)
e5992f2e 480{
ab8e5235 481 unsigned long vmaddr;
c5034945 482 struct vm_area_struct *vma;
e5992f2e
MS		 483 struct gmap_pgtable *mp;
484 struct gmap_rmap *rmap;
c5034945 485 struct mm_struct *mm;
e5992f2e
MS		 486 struct page *page;
487 pgd_t *pgd;
488 pud_t *pud;
489 pmd_t *pmd;
490
ab8e5235
MS		 491 mm = gmap->mm;
492 vmaddr = segment & _SEGMENT_ENTRY_ORIGIN;
493 vma = find_vma(mm, vmaddr);
494 if (!vma || vma->vm_start > vmaddr)
495 return -EFAULT;
496 /* Walk the parent mm page table */
497 pgd = pgd_offset(mm, vmaddr);
498 pud = pud_alloc(mm, pgd, vmaddr);
499 if (!pud)
500 return -ENOMEM;
501 pmd = pmd_alloc(mm, pud, vmaddr);
502 if (!pmd)
503 return -ENOMEM;
504 if (!pmd_present(*pmd) &&
505 __pte_alloc(mm, vma, pmd, vmaddr))
506 return -ENOMEM;
507 /* pmd now points to a valid segment table entry. */
508 rmap = kmalloc(sizeof(*rmap), GFP_KERNEL|__GFP_REPEAT);
509 if (!rmap)
510 return -ENOMEM;
511 /* Link gmap segment table entry location to page table. */
512 page = pmd_page(*pmd);
513 mp = (struct gmap_pgtable *) page->index;
d3383632 514 rmap->gmap = gmap;
ab8e5235 515 rmap->entry = segment_ptr;
e86cbd87 516 rmap->vmaddr = address & PMD_MASK;
ab8e5235
MS		 517 spin_lock(&mm->page_table_lock);
518 if (*segment_ptr == segment) {
519 list_add(&rmap->list, &mp->mapper);
520 /* Set gmap segment table entry to page table. */
521 *segment_ptr = pmd_val(*pmd) & PAGE_MASK;
522 rmap = NULL;
523 }
524 spin_unlock(&mm->page_table_lock);
525 kfree(rmap);
526 return 0;
527}
528
529static void gmap_disconnect_pgtable(struct mm_struct *mm, unsigned long *table)
530{
531 struct gmap_rmap *rmap, *next;
532 struct gmap_pgtable *mp;
533 struct page *page;
534 int flush;
535
536 flush = 0;
537 spin_lock(&mm->page_table_lock);
538 page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
539 mp = (struct gmap_pgtable *) page->index;
540 list_for_each_entry_safe(rmap, next, &mp->mapper, list) {
e5098611
MS		 541 *rmap->entry = mp->vmaddr | (_SEGMENT_ENTRY_INVALID |
542 _SEGMENT_ENTRY_PROTECT);
ab8e5235
MS		 543 list_del(&rmap->list);
544 kfree(rmap);
545 flush = 1;
546 }
547 spin_unlock(&mm->page_table_lock);
548 if (flush)
549 __tlb_flush_global();
550}
551
552/*
553 * this function is assumed to be called with mmap_sem held
554 */
555unsigned long __gmap_fault(unsigned long address, struct gmap *gmap)
556{
557 unsigned long *segment_ptr, segment;
558 struct gmap_pgtable *mp;
559 struct page *page;
560 int rc;
561
e5992f2e 562 current->thread.gmap_addr = address;
c5034945
HC		 563 segment_ptr = gmap_table_walk(address, gmap);
564 if (IS_ERR(segment_ptr))
e5992f2e 565 return -EFAULT;
e5992f2e 566 /* Convert the gmap address to an mm address. */
ab8e5235
MS		 567 while (1) {
568 segment = *segment_ptr;
e5098611 569 if (!(segment & _SEGMENT_ENTRY_INVALID)) {
ab8e5235
MS		 570 /* Page table is present */
571 page = pfn_to_page(segment >> PAGE_SHIFT);
572 mp = (struct gmap_pgtable *) page->index;
573 return mp->vmaddr | (address & ~PMD_MASK);
574 }
e5098611 575 if (!(segment & _SEGMENT_ENTRY_PROTECT))
ab8e5235
MS		 576 /* Nothing mapped in the gmap address space. */
577 break;
d3383632 578 rc = gmap_connect_pgtable(address, segment, segment_ptr, gmap);
ab8e5235
MS		 579 if (rc)
580 return rc;
e5992f2e
MS		 581 }
582 return -EFAULT;
499069e1
CO		 583}
584
585unsigned long gmap_fault(unsigned long address, struct gmap *gmap)
586{
587 unsigned long rc;
588
589 down_read(&gmap->mm->mmap_sem);
590 rc = __gmap_fault(address, gmap);
591 up_read(&gmap->mm->mmap_sem);
e5992f2e 592
499069e1 593 return rc;
e5992f2e
MS		 594}
595EXPORT_SYMBOL_GPL(gmap_fault);
596
388186bc
CB		 597void gmap_discard(unsigned long from, unsigned long to, struct gmap *gmap)
598{
599
600 unsigned long *table, address, size;
601 struct vm_area_struct *vma;
602 struct gmap_pgtable *mp;
603 struct page *page;
604
605 down_read(&gmap->mm->mmap_sem);
606 address = from;
607 while (address < to) {
608 /* Walk the gmap address space page table */
609 table = gmap->table + ((address >> 53) & 0x7ff);
e5098611 610 if (unlikely(*table & _REGION_ENTRY_INVALID)) {
388186bc
CB		 611 address = (address + PMD_SIZE) & PMD_MASK;
612 continue;
613 }
614 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
615 table = table + ((address >> 42) & 0x7ff);
e5098611 616 if (unlikely(*table & _REGION_ENTRY_INVALID)) {
388186bc
CB		 617 address = (address + PMD_SIZE) & PMD_MASK;
618 continue;
619 }
620 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
621 table = table + ((address >> 31) & 0x7ff);
e5098611 622 if (unlikely(*table & _REGION_ENTRY_INVALID)) {
388186bc
CB		 623 address = (address + PMD_SIZE) & PMD_MASK;
624 continue;
625 }
626 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
627 table = table + ((address >> 20) & 0x7ff);
e5098611 628 if (unlikely(*table & _SEGMENT_ENTRY_INVALID)) {
388186bc
CB		 629 address = (address + PMD_SIZE) & PMD_MASK;
630 continue;
631 }
632 page = pfn_to_page(*table >> PAGE_SHIFT);
633 mp = (struct gmap_pgtable *) page->index;
634 vma = find_vma(gmap->mm, mp->vmaddr);
635 size = min(to - address, PMD_SIZE - (address & ~PMD_MASK));
636 zap_page_range(vma, mp->vmaddr | (address & ~PMD_MASK),
637 size, NULL);
638 address = (address + PMD_SIZE) & PMD_MASK;
639 }
640 up_read(&gmap->mm->mmap_sem);
641}
642EXPORT_SYMBOL_GPL(gmap_discard);
643
d3383632
MS		 644static LIST_HEAD(gmap_notifier_list);
645static DEFINE_SPINLOCK(gmap_notifier_lock);
646
647/**
648 * gmap_register_ipte_notifier - register a pte invalidation callback
649 * @nb: pointer to the gmap notifier block
650 */
651void gmap_register_ipte_notifier(struct gmap_notifier *nb)
652{
653 spin_lock(&gmap_notifier_lock);
654 list_add(&nb->list, &gmap_notifier_list);
655 spin_unlock(&gmap_notifier_lock);
656}
657EXPORT_SYMBOL_GPL(gmap_register_ipte_notifier);
658
659/**
660 * gmap_unregister_ipte_notifier - remove a pte invalidation callback
661 * @nb: pointer to the gmap notifier block
662 */
663void gmap_unregister_ipte_notifier(struct gmap_notifier *nb)
664{
665 spin_lock(&gmap_notifier_lock);
666 list_del_init(&nb->list);
667 spin_unlock(&gmap_notifier_lock);
668}
669EXPORT_SYMBOL_GPL(gmap_unregister_ipte_notifier);
670
671/**
672 * gmap_ipte_notify - mark a range of ptes for invalidation notification
673 * @gmap: pointer to guest mapping meta data structure
674 * @address: virtual address in the guest address space
675 * @len: size of area
676 *
677 * Returns 0 if for each page in the given range a gmap mapping exists and
678 * the invalidation notification could be set. If the gmap mapping is missing
679 * for one or more pages -EFAULT is returned. If no memory could be allocated
680 * -ENOMEM is returned. This function establishes missing page table entries.
681 */
682int gmap_ipte_notify(struct gmap *gmap, unsigned long start, unsigned long len)
683{
684 unsigned long addr;
685 spinlock_t *ptl;
686 pte_t *ptep, entry;
687 pgste_t pgste;
688 int rc = 0;
689
690 if ((start & ~PAGE_MASK) || (len & ~PAGE_MASK))
691 return -EINVAL;
692 down_read(&gmap->mm->mmap_sem);
693 while (len) {
694 /* Convert gmap address and connect the page tables */
695 addr = __gmap_fault(start, gmap);
696 if (IS_ERR_VALUE(addr)) {
697 rc = addr;
698 break;
699 }
700 /* Get the page mapped */
bb4b42ce 701 if (fixup_user_fault(current, gmap->mm, addr, FAULT_FLAG_WRITE)) {
d3383632
MS		 702 rc = -EFAULT;
703 break;
704 }
705 /* Walk the process page table, lock and get pte pointer */
706 ptep = get_locked_pte(gmap->mm, addr, &ptl);
707 if (unlikely(!ptep))
708 continue;
709 /* Set notification bit in the pgste of the pte */
710 entry = *ptep;
e5098611 711 if ((pte_val(entry) & (_PAGE_INVALID | _PAGE_PROTECT)) == 0) {
d3383632 712 pgste = pgste_get_lock(ptep);
0d0dafc1 713 pgste_val(pgste) |= PGSTE_IN_BIT;
d3383632
MS		 714 pgste_set_unlock(ptep, pgste);
715 start += PAGE_SIZE;
716 len -= PAGE_SIZE;
717 }
718 spin_unlock(ptl);
719 }
720 up_read(&gmap->mm->mmap_sem);
721 return rc;
722}
723EXPORT_SYMBOL_GPL(gmap_ipte_notify);
724
725/**
726 * gmap_do_ipte_notify - call all invalidation callbacks for a specific pte.
727 * @mm: pointer to the process mm_struct
728 * @addr: virtual address in the process address space
729 * @pte: pointer to the page table entry
730 *
731 * This function is assumed to be called with the page table lock held
732 * for the pte to notify.
733 */
734void gmap_do_ipte_notify(struct mm_struct *mm, unsigned long addr, pte_t *pte)
735{
736 unsigned long segment_offset;
737 struct gmap_notifier *nb;
738 struct gmap_pgtable *mp;
739 struct gmap_rmap *rmap;
740 struct page *page;
741
742 segment_offset = ((unsigned long) pte) & (255 * sizeof(pte_t));
743 segment_offset = segment_offset * (4096 / sizeof(pte_t));
744 page = pfn_to_page(__pa(pte) >> PAGE_SHIFT);
745 mp = (struct gmap_pgtable *) page->index;
746 spin_lock(&gmap_notifier_lock);
747 list_for_each_entry(rmap, &mp->mapper, list) {
748 list_for_each_entry(nb, &gmap_notifier_list, list)
749 nb->notifier_call(rmap->gmap,
750 rmap->vmaddr + segment_offset);
751 }
752 spin_unlock(&gmap_notifier_lock);
753}
754
3eabaee9
MS		 755static inline int page_table_with_pgste(struct page *page)
756{
757 return atomic_read(&page->_mapcount) == 0;
758}
759
e5992f2e
MS		 760static inline unsigned long *page_table_alloc_pgste(struct mm_struct *mm,
761 unsigned long vmaddr)
36409f63
MS		 762{
763 struct page *page;
764 unsigned long *table;
e5992f2e 765 struct gmap_pgtable *mp;
36409f63
MS		 766
767 page = alloc_page(GFP_KERNEL|__GFP_REPEAT);
768 if (!page)
769 return NULL;
e5992f2e
MS		 770 mp = kmalloc(sizeof(*mp), GFP_KERNEL|__GFP_REPEAT);
771 if (!mp) {
772 __free_page(page);
773 return NULL;
774 }
36409f63 775 pgtable_page_ctor(page);
e5992f2e
MS		 776 mp->vmaddr = vmaddr & PMD_MASK;
777 INIT_LIST_HEAD(&mp->mapper);
778 page->index = (unsigned long) mp;
3eabaee9 779 atomic_set(&page->_mapcount, 0);
36409f63 780 table = (unsigned long *) page_to_phys(page);
e5098611 781 clear_table(table, _PAGE_INVALID, PAGE_SIZE/2);
0944fe3f
MS		 782 clear_table(table + PTRS_PER_PTE, PGSTE_HR_BIT | PGSTE_HC_BIT,
783 PAGE_SIZE/2);
36409f63
MS		 784 return table;
785}
786
787static inline void page_table_free_pgste(unsigned long *table)
788{
789 struct page *page;
e5992f2e 790 struct gmap_pgtable *mp;
36409f63
MS		 791
792 page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
e5992f2e
MS		 793 mp = (struct gmap_pgtable *) page->index;
794 BUG_ON(!list_empty(&mp->mapper));
2320c579 795 pgtable_page_dtor(page);
36409f63 796 atomic_set(&page->_mapcount, -1);
e5992f2e 797 kfree(mp);
36409f63
MS		 798 __free_page(page);
799}
36409f63 800
24d5dd02
CB		 801int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
802 unsigned long key, bool nq)
803{
804 spinlock_t *ptl;
805 pgste_t old, new;
806 pte_t *ptep;
807
808 down_read(&mm->mmap_sem);
809 ptep = get_locked_pte(current->mm, addr, &ptl);
810 if (unlikely(!ptep)) {
811 up_read(&mm->mmap_sem);
812 return -EFAULT;
813 }
814
815 new = old = pgste_get_lock(ptep);
816 pgste_val(new) &= ~(PGSTE_GR_BIT | PGSTE_GC_BIT |
817 PGSTE_ACC_BITS | PGSTE_FP_BIT);
818 pgste_val(new) |= (key & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48;
819 pgste_val(new) |= (key & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
820 if (!(pte_val(*ptep) & _PAGE_INVALID)) {
0944fe3f 821 unsigned long address, bits, skey;
24d5dd02
CB		 822
823 address = pte_val(*ptep) & PAGE_MASK;
0944fe3f 824 skey = (unsigned long) page_get_storage_key(address);
24d5dd02 825 bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
0944fe3f 826 skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT);
24d5dd02 827 /* Set storage key ACC and FP */
0944fe3f 828 page_set_storage_key(address, skey, !nq);
24d5dd02
CB		 829 /* Merge host changed & referenced into pgste */
830 pgste_val(new) |= bits << 52;
24d5dd02
CB		 831 }
832 /* changing the guest storage key is considered a change of the page */
833 if ((pgste_val(new) ^ pgste_val(old)) &
834 (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT))
0944fe3f 835 pgste_val(new) |= PGSTE_HC_BIT;
24d5dd02
CB		 836
837 pgste_set_unlock(ptep, new);
838 pte_unmap_unlock(*ptep, ptl);
839 up_read(&mm->mmap_sem);
840 return 0;
841}
842EXPORT_SYMBOL(set_guest_storage_key);
843
e5992f2e
MS		 844#else /* CONFIG_PGSTE */
845
3eabaee9
MS		 846static inline int page_table_with_pgste(struct page *page)
847{
848 return 0;
849}
850
e5992f2e
MS		 851static inline unsigned long *page_table_alloc_pgste(struct mm_struct *mm,
852 unsigned long vmaddr)
853{
944291de 854 return NULL;
e5992f2e
MS		 855}
856
857static inline void page_table_free_pgste(unsigned long *table)
858{
859}
860
ab8e5235
MS		 861static inline void gmap_disconnect_pgtable(struct mm_struct *mm,
862 unsigned long *table)
e5992f2e
MS		 863{
864}
865
866#endif /* CONFIG_PGSTE */
867
868static inline unsigned int atomic_xor_bits(atomic_t *v, unsigned int bits)
869{
870 unsigned int old, new;
871
872 do {
873 old = atomic_read(v);
874 new = old ^ bits;
875 } while (atomic_cmpxchg(v, old, new) != old);
876 return new;
877}
878
879/*
880 * page table entry allocation/free routines.
881 */
882unsigned long *page_table_alloc(struct mm_struct *mm, unsigned long vmaddr)
3610cce8 883{
41459d36
HC		 884 unsigned long *uninitialized_var(table);
885 struct page *uninitialized_var(page);
36409f63 886 unsigned int mask, bit;
3610cce8 887
36409f63 888 if (mm_has_pgste(mm))
e5992f2e 889 return page_table_alloc_pgste(mm, vmaddr);
36409f63 890 /* Allocate fragments of a 4K page as 1K/2K page table */
80217147 891 spin_lock_bh(&mm->context.list_lock);
36409f63 892 mask = FRAG_MASK;
146e4b3c
MS		 893 if (!list_empty(&mm->context.pgtable_list)) {
894 page = list_first_entry(&mm->context.pgtable_list,
895 struct page, lru);
36409f63
MS		 896 table = (unsigned long *) page_to_phys(page);
897 mask = atomic_read(&page->_mapcount);
898 mask = mask | (mask >> 4);
146e4b3c 899 }
36409f63 900 if ((mask & FRAG_MASK) == FRAG_MASK) {
80217147 901 spin_unlock_bh(&mm->context.list_lock);
146e4b3c
MS		 902 page = alloc_page(GFP_KERNEL|__GFP_REPEAT);
903 if (!page)
3610cce8 904 return NULL;
146e4b3c 905 pgtable_page_ctor(page);
36409f63 906 atomic_set(&page->_mapcount, 1);
146e4b3c 907 table = (unsigned long *) page_to_phys(page);
e5098611 908 clear_table(table, _PAGE_INVALID, PAGE_SIZE);
80217147 909 spin_lock_bh(&mm->context.list_lock);
146e4b3c 910 list_add(&page->lru, &mm->context.pgtable_list);
36409f63
MS		 911 } else {
912 for (bit = 1; mask & bit; bit <<= 1)
913 table += PTRS_PER_PTE;
914 mask = atomic_xor_bits(&page->_mapcount, bit);
915 if ((mask & FRAG_MASK) == FRAG_MASK)
916 list_del(&page->lru);
3610cce8 917 }
80217147 918 spin_unlock_bh(&mm->context.list_lock);
3610cce8
MS		 919 return table;
920}
921
36409f63 922void page_table_free(struct mm_struct *mm, unsigned long *table)
80217147
MS		 923{
924 struct page *page;
36409f63 925 unsigned int bit, mask;
80217147 926
3eabaee9
MS		 927 page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
928 if (page_table_with_pgste(page)) {
ab8e5235 929 gmap_disconnect_pgtable(mm, table);
36409f63 930 return page_table_free_pgste(table);
e5992f2e 931 }
36409f63 932 /* Free 1K/2K page table fragment of a 4K page */
36409f63
MS		 933 bit = 1 << ((__pa(table) & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t)));
934 spin_lock_bh(&mm->context.list_lock);
935 if ((atomic_read(&page->_mapcount) & FRAG_MASK) != FRAG_MASK)
936 list_del(&page->lru);
937 mask = atomic_xor_bits(&page->_mapcount, bit);
938 if (mask & FRAG_MASK)
939 list_add(&page->lru, &mm->context.pgtable_list);
940 spin_unlock_bh(&mm->context.list_lock);
941 if (mask == 0) {
80217147 942 pgtable_page_dtor(page);
36409f63 943 atomic_set(&page->_mapcount, -1);
80217147
MS		 944 __free_page(page);
945 }
946}
947
36409f63 948static void __page_table_free_rcu(void *table, unsigned bit)
3610cce8 949{
146e4b3c 950 struct page *page;
3610cce8 951
36409f63
MS		 952 if (bit == FRAG_MASK)
953 return page_table_free_pgste(table);
36409f63 954 /* Free 1K/2K page table fragment of a 4K page */
146e4b3c 955 page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
36409f63 956 if (atomic_xor_bits(&page->_mapcount, bit) == 0) {
146e4b3c 957 pgtable_page_dtor(page);
36409f63 958 atomic_set(&page->_mapcount, -1);
146e4b3c
MS		 959 __free_page(page);
960 }
961}
3610cce8 962
36409f63 963void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table)
80217147 964{
36409f63 965 struct mm_struct *mm;
80217147 966 struct page *page;
36409f63 967 unsigned int bit, mask;
80217147 968
36409f63 969 mm = tlb->mm;
3eabaee9
MS		 970 page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
971 if (page_table_with_pgste(page)) {
ab8e5235 972 gmap_disconnect_pgtable(mm, table);
36409f63
MS		 973 table = (unsigned long *) (__pa(table) | FRAG_MASK);
974 tlb_remove_table(tlb, table);
975 return;
80217147 976 }
36409f63 977 bit = 1 << ((__pa(table) & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t)));
80217147 978 spin_lock_bh(&mm->context.list_lock);
36409f63
MS		 979 if ((atomic_read(&page->_mapcount) & FRAG_MASK) != FRAG_MASK)
980 list_del(&page->lru);
981 mask = atomic_xor_bits(&page->_mapcount, bit | (bit << 4));
982 if (mask & FRAG_MASK)
983 list_add_tail(&page->lru, &mm->context.pgtable_list);
80217147 984 spin_unlock_bh(&mm->context.list_lock);
36409f63
MS		 985 table = (unsigned long *) (__pa(table) | (bit << 4));
986 tlb_remove_table(tlb, table);
987}
988
63df41d6 989static void __tlb_remove_table(void *_table)
36409f63 990{
e73b7fff
MS		 991 const unsigned long mask = (FRAG_MASK << 4) | FRAG_MASK;
992 void *table = (void *)((unsigned long) _table & ~mask);
993 unsigned type = (unsigned long) _table & mask;
36409f63
MS		 994
995 if (type)
996 __page_table_free_rcu(table, type);
997 else
998 free_pages((unsigned long) table, ALLOC_ORDER);
80217147
MS		 999}
1000
cd94154c
MS		 1001static void tlb_remove_table_smp_sync(void *arg)
1002{
1003 /* Simply deliver the interrupt */
1004}
1005
1006static void tlb_remove_table_one(void *table)
1007{
1008 /*
1009 * This isn't an RCU grace period and hence the page-tables cannot be
1010 * assumed to be actually RCU-freed.
1011 *
1012 * It is however sufficient for software page-table walkers that rely
1013 * on IRQ disabling. See the comment near struct mmu_table_batch.
1014 */
1015 smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
1016 __tlb_remove_table(table);
1017}
1018
1019static void tlb_remove_table_rcu(struct rcu_head *head)
1020{
1021 struct mmu_table_batch *batch;
1022 int i;
1023
1024 batch = container_of(head, struct mmu_table_batch, rcu);
1025
1026 for (i = 0; i < batch->nr; i++)
1027 __tlb_remove_table(batch->tables[i]);
1028
1029 free_page((unsigned long)batch);
1030}
1031
1032void tlb_table_flush(struct mmu_gather *tlb)
1033{
1034 struct mmu_table_batch **batch = &tlb->batch;
1035
1036 if (*batch) {
cd94154c
MS		 1037 call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu);
1038 *batch = NULL;
1039 }
1040}
1041
1042void tlb_remove_table(struct mmu_gather *tlb, void *table)
1043{
1044 struct mmu_table_batch **batch = &tlb->batch;
1045
5c474a1e 1046 tlb->mm->context.flush_mm = 1;
cd94154c
MS		 1047 if (*batch == NULL) {
1048 *batch = (struct mmu_table_batch *)
1049 __get_free_page(GFP_NOWAIT | __GFP_NOWARN);
1050 if (*batch == NULL) {
5c474a1e 1051 __tlb_flush_mm_lazy(tlb->mm);
cd94154c
MS		 1052 tlb_remove_table_one(table);
1053 return;
1054 }
1055 (*batch)->nr = 0;
1056 }
1057 (*batch)->tables[(*batch)->nr++] = table;
1058 if ((*batch)->nr == MAX_TABLE_BATCH)
5c474a1e 1059 tlb_flush_mmu(tlb);
cd94154c 1060}
36409f63 1061
274023da 1062#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3eabaee9 1063static inline void thp_split_vma(struct vm_area_struct *vma)
274023da
GS		 1064{
1065 unsigned long addr;
274023da 1066
3eabaee9
MS		 1067 for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE)
1068 follow_page(vma, addr, FOLL_SPLIT);
274023da
GS		 1069}
1070
3eabaee9 1071static inline void thp_split_mm(struct mm_struct *mm)
274023da 1072{
3eabaee9 1073 struct vm_area_struct *vma;
274023da 1074
3eabaee9 1075 for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) {
274023da
GS		 1076 thp_split_vma(vma);
1077 vma->vm_flags &= ~VM_HUGEPAGE;
1078 vma->vm_flags |= VM_NOHUGEPAGE;
274023da 1079 }
3eabaee9
MS		 1080 mm->def_flags |= VM_NOHUGEPAGE;
1081}
1082#else
1083static inline void thp_split_mm(struct mm_struct *mm)
1084{
274023da
GS		 1085}
1086#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1087
3eabaee9
MS		 1088static unsigned long page_table_realloc_pmd(struct mmu_gather *tlb,
1089 struct mm_struct *mm, pud_t *pud,
1090 unsigned long addr, unsigned long end)
1091{
1092 unsigned long next, *table, *new;
1093 struct page *page;
1094 pmd_t *pmd;
1095
1096 pmd = pmd_offset(pud, addr);
1097 do {
1098 next = pmd_addr_end(addr, end);
1099again:
1100 if (pmd_none_or_clear_bad(pmd))
1101 continue;
1102 table = (unsigned long *) pmd_deref(*pmd);
1103 page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
1104 if (page_table_with_pgste(page))
1105 continue;
1106 /* Allocate new page table with pgstes */
1107 new = page_table_alloc_pgste(mm, addr);
be39f196
DD		 1108 if (!new)
1109 return -ENOMEM;
1110
3eabaee9
MS		 1111 spin_lock(&mm->page_table_lock);
1112 if (likely((unsigned long *) pmd_deref(*pmd) == table)) {
1113 /* Nuke pmd entry pointing to the "short" page table */
1114 pmdp_flush_lazy(mm, addr, pmd);
1115 pmd_clear(pmd);
1116 /* Copy ptes from old table to new table */
1117 memcpy(new, table, PAGE_SIZE/2);
1118 clear_table(table, _PAGE_INVALID, PAGE_SIZE/2);
1119 /* Establish new table */
1120 pmd_populate(mm, pmd, (pte_t *) new);
1121 /* Free old table with rcu, there might be a walker! */
1122 page_table_free_rcu(tlb, table);
1123 new = NULL;
1124 }
1125 spin_unlock(&mm->page_table_lock);
1126 if (new) {
1127 page_table_free_pgste(new);
1128 goto again;
1129 }
1130 } while (pmd++, addr = next, addr != end);
1131
1132 return addr;
1133}
1134
1135static unsigned long page_table_realloc_pud(struct mmu_gather *tlb,
1136 struct mm_struct *mm, pgd_t *pgd,
1137 unsigned long addr, unsigned long end)
1138{
1139 unsigned long next;
1140 pud_t *pud;
1141
1142 pud = pud_offset(pgd, addr);
1143 do {
1144 next = pud_addr_end(addr, end);
1145 if (pud_none_or_clear_bad(pud))
1146 continue;
1147 next = page_table_realloc_pmd(tlb, mm, pud, addr, next);
be39f196
DD		 1148 if (unlikely(IS_ERR_VALUE(next)))
1149 return next;
3eabaee9
MS		 1150 } while (pud++, addr = next, addr != end);
1151
1152 return addr;
1153}
1154
be39f196
DD		 1155static unsigned long page_table_realloc(struct mmu_gather *tlb, struct mm_struct *mm,
1156 unsigned long addr, unsigned long end)
3eabaee9
MS		 1157{
1158 unsigned long next;
1159 pgd_t *pgd;
1160
1161 pgd = pgd_offset(mm, addr);
1162 do {
1163 next = pgd_addr_end(addr, end);
1164 if (pgd_none_or_clear_bad(pgd))
1165 continue;
1166 next = page_table_realloc_pud(tlb, mm, pgd, addr, next);
be39f196
DD		 1167 if (unlikely(IS_ERR_VALUE(next)))
1168 return next;
3eabaee9 1169 } while (pgd++, addr = next, addr != end);
be39f196
DD		 1170
1171 return 0;
3eabaee9
MS		 1172}
1173
402b0862
CO		 1174/*
1175 * switch on pgstes for its userspace process (for kvm)
1176 */
1177int s390_enable_sie(void)
1178{
1179 struct task_struct *tsk = current;
3eabaee9
MS		 1180 struct mm_struct *mm = tsk->mm;
1181 struct mmu_gather tlb;
402b0862 1182
74b6b522 1183 /* Do we have pgstes? if yes, we are done */
36409f63 1184 if (mm_has_pgste(tsk->mm))
74b6b522 1185 return 0;
402b0862 1186
3eabaee9 1187 down_write(&mm->mmap_sem);
274023da
GS		 1188 /* split thp mappings and disable thp for future mappings */
1189 thp_split_mm(mm);
3eabaee9 1190 /* Reallocate the page tables with pgstes */
ae7a835c 1191 tlb_gather_mmu(&tlb, mm, 0, TASK_SIZE);
be39f196
DD		 1192 if (!page_table_realloc(&tlb, mm, 0, TASK_SIZE))
1193 mm->context.has_pgste = 1;
ae7a835c 1194 tlb_finish_mmu(&tlb, 0, TASK_SIZE);
3eabaee9
MS		 1195 up_write(&mm->mmap_sem);
1196 return mm->context.has_pgste ? 0 : -ENOMEM;
402b0862
CO		 1197}
1198EXPORT_SYMBOL_GPL(s390_enable_sie);
7db11a36 1199
75077afb 1200#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1ae1c1d0
GS		 1201int pmdp_clear_flush_young(struct vm_area_struct *vma, unsigned long address,
1202 pmd_t *pmdp)
1203{
1204 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
1205 /* No need to flush TLB
1206 * On s390 reference bits are in storage key and never in TLB */
1207 return pmdp_test_and_clear_young(vma, address, pmdp);
1208}
1209
1210int pmdp_set_access_flags(struct vm_area_struct *vma,
1211 unsigned long address, pmd_t *pmdp,
1212 pmd_t entry, int dirty)
1213{
1214 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
1215
1216 if (pmd_same(*pmdp, entry))
1217 return 0;
1218 pmdp_invalidate(vma, address, pmdp);
1219 set_pmd_at(vma->vm_mm, address, pmdp, entry);
1220 return 1;
1221}
1222
75077afb
GS		 1223static void pmdp_splitting_flush_sync(void *arg)
1224{
1225 /* Simply deliver the interrupt */
1226}
1227
1228void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,
1229 pmd_t *pmdp)
1230{
1231 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
1232 if (!test_and_set_bit(_SEGMENT_ENTRY_SPLIT_BIT,
1233 (unsigned long *) pmdp)) {
1234 /* need to serialize against gup-fast (IRQ disabled) */
1235 smp_call_function(pmdp_splitting_flush_sync, NULL, 1);
1236 }
1237}
9501d09f 1238
6b0b50b0
AK		 1239void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
1240 pgtable_t pgtable)
9501d09f
GS		 1241{
1242 struct list_head *lh = (struct list_head *) pgtable;
1243
1244 assert_spin_locked(&mm->page_table_lock);
1245
1246 /* FIFO */
c389a250 1247 if (!pmd_huge_pte(mm, pmdp))
9501d09f
GS		 1248 INIT_LIST_HEAD(lh);
1249 else
c389a250
KS		 1250 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
1251 pmd_huge_pte(mm, pmdp) = pgtable;
9501d09f
GS		 1252}
1253
6b0b50b0 1254pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
9501d09f
GS		 1255{
1256 struct list_head *lh;
1257 pgtable_t pgtable;
1258 pte_t *ptep;
1259
1260 assert_spin_locked(&mm->page_table_lock);
1261
1262 /* FIFO */
c389a250 1263 pgtable = pmd_huge_pte(mm, pmdp);
9501d09f
GS		 1264 lh = (struct list_head *) pgtable;
1265 if (list_empty(lh))
c389a250 1266 pmd_huge_pte(mm, pmdp) = NULL;
9501d09f 1267 else {
c389a250 1268 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
9501d09f
GS		 1269 list_del(lh);
1270 }
1271 ptep = (pte_t *) pgtable;
e5098611 1272 pte_val(*ptep) = _PAGE_INVALID;
9501d09f 1273 ptep++;
e5098611 1274 pte_val(*ptep) = _PAGE_INVALID;
9501d09f
GS		 1275 return pgtable;
1276}
75077afb 1277#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
Linux 6.x block layer and io_uring tree(s)