arch/s390/mm/vmem.c

   1 /*
   2  *    Copyright IBM Corp. 2006
   3  *    Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
   4  */
   5
   6 #include <linux/bootmem.h>
   7 #include <linux/pfn.h>
   8 #include <linux/mm.h>
   9 #include <linux/module.h>
  10 #include <linux/list.h>
  11 #include <linux/hugetlb.h>
  12 #include <linux/slab.h>
  13 #include <linux/memblock.h>
  14 #include <asm/cacheflush.h>
  15 #include <asm/pgalloc.h>
  16 #include <asm/pgtable.h>
  17 #include <asm/setup.h>
  18 #include <asm/tlbflush.h>
  19 #include <asm/sections.h>
  20
  21 static DEFINE_MUTEX(vmem_mutex);
  22
  23 struct memory_segment {
  24         struct list_head list;
  25         unsigned long start;
  26         unsigned long size;
  27 };
  28
  29 static LIST_HEAD(mem_segs);
  30
  31 static void __ref *vmem_alloc_pages(unsigned int order)
  32 {
  33         unsigned long size = PAGE_SIZE << order;
  34
  35         if (slab_is_available())
  36                 return (void *)__get_free_pages(GFP_KERNEL, order);
  37         return (void *) memblock_alloc(size, size);
  38 }
  39
  40 static inline pud_t *vmem_pud_alloc(void)
  41 {
  42         pud_t *pud = NULL;
  43
  44         pud = vmem_alloc_pages(2);
  45         if (!pud)
  46                 return NULL;
  47         clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4);
  48         return pud;
  49 }
  50
  51 pmd_t *vmem_pmd_alloc(void)
  52 {
  53         pmd_t *pmd = NULL;
  54
  55         pmd = vmem_alloc_pages(2);
  56         if (!pmd)
  57                 return NULL;
  58         clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4);
  59         return pmd;
  60 }
  61
  62 pte_t __ref *vmem_pte_alloc(void)
  63 {
  64         unsigned long size = PTRS_PER_PTE * sizeof(pte_t);
  65         pte_t *pte;
  66
  67         if (slab_is_available())
  68                 pte = (pte_t *) page_table_alloc(&init_mm);
  69         else
  70                 pte = (pte_t *) memblock_alloc(size, size);
  71         if (!pte)
  72                 return NULL;
  73         clear_table((unsigned long *) pte, _PAGE_INVALID, size);
  74         return pte;
  75 }
  76
  77 /*
  78  * Add a physical memory range to the 1:1 mapping.
  79  */
  80 static int vmem_add_mem(unsigned long start, unsigned long size)
  81 {
  82         unsigned long pages4k, pages1m, pages2g;
  83         unsigned long end = start + size;
  84         unsigned long address = start;
  85         pgd_t *pg_dir;
  86         pud_t *pu_dir;
  87         pmd_t *pm_dir;
  88         pte_t *pt_dir;
  89         int ret = -ENOMEM;
  90
  91         pages4k = pages1m = pages2g = 0;
  92         while (address < end) {
  93                 pg_dir = pgd_offset_k(address);
  94                 if (pgd_none(*pg_dir)) {
  95                         pu_dir = vmem_pud_alloc();
  96                         if (!pu_dir)
  97                                 goto out;
  98                         pgd_populate(&init_mm, pg_dir, pu_dir);
  99                 }
 100                 pu_dir = pud_offset(pg_dir, address);
 101                 if (MACHINE_HAS_EDAT2 && pud_none(*pu_dir) && address &&
 102                     !(address & ~PUD_MASK) && (address + PUD_SIZE <= end) &&
 103                      !debug_pagealloc_enabled()) {
 104                         pud_val(*pu_dir) = address | pgprot_val(REGION3_KERNEL);
 105                         address += PUD_SIZE;
 106                         pages2g++;
 107                         continue;
 108                 }
 109                 if (pud_none(*pu_dir)) {
 110                         pm_dir = vmem_pmd_alloc();
 111                         if (!pm_dir)
 112                                 goto out;
 113                         pud_populate(&init_mm, pu_dir, pm_dir);
 114                 }
 115                 pm_dir = pmd_offset(pu_dir, address);
 116                 if (MACHINE_HAS_EDAT1 && pmd_none(*pm_dir) && address &&
 117                     !(address & ~PMD_MASK) && (address + PMD_SIZE <= end) &&
 118                     !debug_pagealloc_enabled()) {
 119                         pmd_val(*pm_dir) = address | pgprot_val(SEGMENT_KERNEL);
 120                         address += PMD_SIZE;
 121                         pages1m++;
 122                         continue;
 123                 }
 124                 if (pmd_none(*pm_dir)) {
 125                         pt_dir = vmem_pte_alloc();
 126                         if (!pt_dir)
 127                                 goto out;
 128                         pmd_populate(&init_mm, pm_dir, pt_dir);
 129                 }
 130
 131                 pt_dir = pte_offset_kernel(pm_dir, address);
 132                 pte_val(*pt_dir) = address |  pgprot_val(PAGE_KERNEL);
 133                 address += PAGE_SIZE;
 134                 pages4k++;
 135         }
 136         ret = 0;
 137 out:
 138         update_page_count(PG_DIRECT_MAP_4K, pages4k);
 139         update_page_count(PG_DIRECT_MAP_1M, pages1m);
 140         update_page_count(PG_DIRECT_MAP_2G, pages2g);
 141         return ret;
 142 }
 143
 144 /*
 145  * Remove a physical memory range from the 1:1 mapping.
 146  * Currently only invalidates page table entries.
 147  */
 148 static void vmem_remove_range(unsigned long start, unsigned long size)
 149 {
 150         unsigned long pages4k, pages1m, pages2g;
 151         unsigned long end = start + size;
 152         unsigned long address = start;
 153         pgd_t *pg_dir;
 154         pud_t *pu_dir;
 155         pmd_t *pm_dir;
 156         pte_t *pt_dir;
 157
 158         pages4k = pages1m = pages2g = 0;
 159         while (address < end) {
 160                 pg_dir = pgd_offset_k(address);
 161                 if (pgd_none(*pg_dir)) {
 162                         address += PGDIR_SIZE;
 163                         continue;
 164                 }
 165                 pu_dir = pud_offset(pg_dir, address);
 166                 if (pud_none(*pu_dir)) {
 167                         address += PUD_SIZE;
 168                         continue;
 169                 }
 170                 if (pud_large(*pu_dir)) {
 171                         pud_clear(pu_dir);
 172                         address += PUD_SIZE;
 173                         pages2g++;
 174                         continue;
 175                 }
 176                 pm_dir = pmd_offset(pu_dir, address);
 177                 if (pmd_none(*pm_dir)) {
 178                         address += PMD_SIZE;
 179                         continue;
 180                 }
 181                 if (pmd_large(*pm_dir)) {
 182                         pmd_clear(pm_dir);
 183                         address += PMD_SIZE;
 184                         pages1m++;
 185                         continue;
 186                 }
 187                 pt_dir = pte_offset_kernel(pm_dir, address);
 188                 pte_clear(&init_mm, address, pt_dir);
 189                 address += PAGE_SIZE;
 190                 pages4k++;
 191         }
 192         flush_tlb_kernel_range(start, end);
 193         update_page_count(PG_DIRECT_MAP_4K, -pages4k);
 194         update_page_count(PG_DIRECT_MAP_1M, -pages1m);
 195         update_page_count(PG_DIRECT_MAP_2G, -pages2g);
 196 }
 197
 198 /*
 199  * Add a backed mem_map array to the virtual mem_map array.
 200  */
 201 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
 202 {
 203         unsigned long address = start;
 204         pgd_t *pg_dir;
 205         pud_t *pu_dir;
 206         pmd_t *pm_dir;
 207         pte_t *pt_dir;
 208         int ret = -ENOMEM;
 209
 210         for (address = start; address < end;) {
 211                 pg_dir = pgd_offset_k(address);
 212                 if (pgd_none(*pg_dir)) {
 213                         pu_dir = vmem_pud_alloc();
 214                         if (!pu_dir)
 215                                 goto out;
 216                         pgd_populate(&init_mm, pg_dir, pu_dir);
 217                 }
 218
 219                 pu_dir = pud_offset(pg_dir, address);
 220                 if (pud_none(*pu_dir)) {
 221                         pm_dir = vmem_pmd_alloc();
 222                         if (!pm_dir)
 223                                 goto out;
 224                         pud_populate(&init_mm, pu_dir, pm_dir);
 225                 }
 226
 227                 pm_dir = pmd_offset(pu_dir, address);
 228                 if (pmd_none(*pm_dir)) {
 229                         /* Use 1MB frames for vmemmap if available. We always
 230                          * use large frames even if they are only partially
 231                          * used.
 232                          * Otherwise we would have also page tables since
 233                          * vmemmap_populate gets called for each section
 234                          * separately. */
 235                         if (MACHINE_HAS_EDAT1) {
 236                                 void *new_page;
 237
 238                                 new_page = vmemmap_alloc_block(PMD_SIZE, node);
 239                                 if (!new_page)
 240                                         goto out;
 241                                 pmd_val(*pm_dir) = __pa(new_page) |
 242                                         _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE;
 243                                 address = (address + PMD_SIZE) & PMD_MASK;
 244                                 continue;
 245                         }
 246                         pt_dir = vmem_pte_alloc();
 247                         if (!pt_dir)
 248                                 goto out;
 249                         pmd_populate(&init_mm, pm_dir, pt_dir);
 250                 } else if (pmd_large(*pm_dir)) {
 251                         address = (address + PMD_SIZE) & PMD_MASK;
 252                         continue;
 253                 }
 254
 255                 pt_dir = pte_offset_kernel(pm_dir, address);
 256                 if (pte_none(*pt_dir)) {
 257                         void *new_page;
 258
 259                         new_page = vmemmap_alloc_block(PAGE_SIZE, node);
 260                         if (!new_page)
 261                                 goto out;
 262                         pte_val(*pt_dir) =
 263                                 __pa(new_page) | pgprot_val(PAGE_KERNEL);
 264                 }
 265                 address += PAGE_SIZE;
 266         }
 267         ret = 0;
 268 out:
 269         return ret;
 270 }
 271
 272 void vmemmap_free(unsigned long start, unsigned long end)
 273 {
 274 }
 275
 276 /*
 277  * Add memory segment to the segment list if it doesn't overlap with
 278  * an already present segment.
 279  */
 280 static int insert_memory_segment(struct memory_segment *seg)
 281 {
 282         struct memory_segment *tmp;
 283
 284         if (seg->start + seg->size > VMEM_MAX_PHYS ||
 285             seg->start + seg->size < seg->start)
 286                 return -ERANGE;
 287
 288         list_for_each_entry(tmp, &mem_segs, list) {
 289                 if (seg->start >= tmp->start + tmp->size)
 290                         continue;
 291                 if (seg->start + seg->size <= tmp->start)
 292                         continue;
 293                 return -ENOSPC;
 294         }
 295         list_add(&seg->list, &mem_segs);
 296         return 0;
 297 }
 298
 299 /*
 300  * Remove memory segment from the segment list.
 301  */
 302 static void remove_memory_segment(struct memory_segment *seg)
 303 {
 304         list_del(&seg->list);
 305 }
 306
 307 static void __remove_shared_memory(struct memory_segment *seg)
 308 {
 309         remove_memory_segment(seg);
 310         vmem_remove_range(seg->start, seg->size);
 311 }
 312
 313 int vmem_remove_mapping(unsigned long start, unsigned long size)
 314 {
 315         struct memory_segment *seg;
 316         int ret;
 317
 318         mutex_lock(&vmem_mutex);
 319
 320         ret = -ENOENT;
 321         list_for_each_entry(seg, &mem_segs, list) {
 322                 if (seg->start == start && seg->size == size)
 323                         break;
 324         }
 325
 326         if (seg->start != start || seg->size != size)
 327                 goto out;
 328
 329         ret = 0;
 330         __remove_shared_memory(seg);
 331         kfree(seg);
 332 out:
 333         mutex_unlock(&vmem_mutex);
 334         return ret;
 335 }
 336
 337 int vmem_add_mapping(unsigned long start, unsigned long size)
 338 {
 339         struct memory_segment *seg;
 340         int ret;
 341
 342         mutex_lock(&vmem_mutex);
 343         ret = -ENOMEM;
 344         seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 345         if (!seg)
 346                 goto out;
 347         seg->start = start;
 348         seg->size = size;
 349
 350         ret = insert_memory_segment(seg);
 351         if (ret)
 352                 goto out_free;
 353
 354         ret = vmem_add_mem(start, size);
 355         if (ret)
 356                 goto out_remove;
 357         goto out;
 358
 359 out_remove:
 360         __remove_shared_memory(seg);
 361 out_free:
 362         kfree(seg);
 363 out:
 364         mutex_unlock(&vmem_mutex);
 365         return ret;
 366 }
 367
 368 /*
 369  * map whole physical memory to virtual memory (identity mapping)
 370  * we reserve enough space in the vmalloc area for vmemmap to hotplug
 371  * additional memory segments.
 372  */
 373 void __init vmem_map_init(void)
 374 {
 375         unsigned long size = _eshared - _stext;
 376         struct memblock_region *reg;
 377
 378         for_each_memblock(memory, reg)
 379                 vmem_add_mem(reg->base, reg->size);
 380         set_memory_ro((unsigned long)_stext, size >> PAGE_SHIFT);
 381         pr_info("Write protected kernel read-only data: %luk\n", size >> 10);
 382 }
 383
 384 /*
 385  * Convert memblock.memory  to a memory segment list so there is a single
 386  * list that contains all memory segments.
 387  */
 388 static int __init vmem_convert_memory_chunk(void)
 389 {
 390         struct memblock_region *reg;
 391         struct memory_segment *seg;
 392
 393         mutex_lock(&vmem_mutex);
 394         for_each_memblock(memory, reg) {
 395                 seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 396                 if (!seg)
 397                         panic("Out of memory...\n");
 398                 seg->start = reg->base;
 399                 seg->size = reg->size;
 400                 insert_memory_segment(seg);
 401         }
 402         mutex_unlock(&vmem_mutex);
 403         return 0;
 404 }
 405
 406 core_initcall(vmem_convert_memory_chunk);