fs/hugetlbfs/inode.c

   1 /*
   2  * hugetlbpage-backed filesystem.  Based on ramfs.
   3  *
   4  * Nadia Yvette Chambers, 2002
   5  *
   6  * Copyright (C) 2002 Linus Torvalds.
   7  */
   8
   9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  10
  11 #include <linux/module.h>
  12 #include <linux/thread_info.h>
  13 #include <asm/current.h>
  14 #include <linux/sched.h>                /* remove ASAP */
  15 #include <linux/fs.h>
  16 #include <linux/mount.h>
  17 #include <linux/file.h>
  18 #include <linux/kernel.h>
  19 #include <linux/writeback.h>
  20 #include <linux/pagemap.h>
  21 #include <linux/highmem.h>
  22 #include <linux/init.h>
  23 #include <linux/string.h>
  24 #include <linux/capability.h>
  25 #include <linux/ctype.h>
  26 #include <linux/backing-dev.h>
  27 #include <linux/hugetlb.h>
  28 #include <linux/pagevec.h>
  29 #include <linux/parser.h>
  30 #include <linux/mman.h>
  31 #include <linux/slab.h>
  32 #include <linux/dnotify.h>
  33 #include <linux/statfs.h>
  34 #include <linux/security.h>
  35 #include <linux/magic.h>
  36 #include <linux/migrate.h>
  37 #include <linux/uio.h>
  38
  39 #include <asm/uaccess.h>
  40
  41 static const struct super_operations hugetlbfs_ops;
  42 static const struct address_space_operations hugetlbfs_aops;
  43 const struct file_operations hugetlbfs_file_operations;
  44 static const struct inode_operations hugetlbfs_dir_inode_operations;
  45 static const struct inode_operations hugetlbfs_inode_operations;
  46
  47 struct hugetlbfs_config {
  48         kuid_t   uid;
  49         kgid_t   gid;
  50         umode_t mode;
  51         long    max_hpages;
  52         long    nr_inodes;
  53         struct hstate *hstate;
  54         long    min_hpages;
  55 };
  56
  57 struct hugetlbfs_inode_info {
  58         struct shared_policy policy;
  59         struct inode vfs_inode;
  60 };
  61
  62 static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode)
  63 {
  64         return container_of(inode, struct hugetlbfs_inode_info, vfs_inode);
  65 }
  66
  67 int sysctl_hugetlb_shm_group;
  68
  69 enum {
  70         Opt_size, Opt_nr_inodes,
  71         Opt_mode, Opt_uid, Opt_gid,
  72         Opt_pagesize, Opt_min_size,
  73         Opt_err,
  74 };
  75
  76 static const match_table_t tokens = {
  77         {Opt_size,      "size=%s"},
  78         {Opt_nr_inodes, "nr_inodes=%s"},
  79         {Opt_mode,      "mode=%o"},
  80         {Opt_uid,       "uid=%u"},
  81         {Opt_gid,       "gid=%u"},
  82         {Opt_pagesize,  "pagesize=%s"},
  83         {Opt_min_size,  "min_size=%s"},
  84         {Opt_err,       NULL},
  85 };
  86
  87 static void huge_pagevec_release(struct pagevec *pvec)
  88 {
  89         int i;
  90
  91         for (i = 0; i < pagevec_count(pvec); ++i)
  92                 put_page(pvec->pages[i]);
  93
  94         pagevec_reinit(pvec);
  95 }
  96
  97 static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma)
  98 {
  99         struct inode *inode = file_inode(file);
 100         loff_t len, vma_len;
 101         int ret;
 102         struct hstate *h = hstate_file(file);
 103
 104         /*
 105          * vma address alignment (but not the pgoff alignment) has
 106          * already been checked by prepare_hugepage_range.  If you add
 107          * any error returns here, do so after setting VM_HUGETLB, so
 108          * is_vm_hugetlb_page tests below unmap_region go the right
 109          * way when do_mmap_pgoff unwinds (may be important on powerpc
 110          * and ia64).
 111          */
 112         vma->vm_flags |= VM_HUGETLB | VM_DONTEXPAND;
 113         vma->vm_ops = &hugetlb_vm_ops;
 114
 115         if (vma->vm_pgoff & (~huge_page_mask(h) >> PAGE_SHIFT))
 116                 return -EINVAL;
 117
 118         vma_len = (loff_t)(vma->vm_end - vma->vm_start);
 119
 120         mutex_lock(&inode->i_mutex);
 121         file_accessed(file);
 122
 123         ret = -ENOMEM;
 124         len = vma_len + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
 125
 126         if (hugetlb_reserve_pages(inode,
 127                                 vma->vm_pgoff >> huge_page_order(h),
 128                                 len >> huge_page_shift(h), vma,
 129                                 vma->vm_flags))
 130                 goto out;
 131
 132         ret = 0;
 133         if (vma->vm_flags & VM_WRITE && inode->i_size < len)
 134                 inode->i_size = len;
 135 out:
 136         mutex_unlock(&inode->i_mutex);
 137
 138         return ret;
 139 }
 140
 141 /*
 142  * Called under down_write(mmap_sem).
 143  */
 144
 145 #ifndef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
 146 static unsigned long
 147 hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
 148                 unsigned long len, unsigned long pgoff, unsigned long flags)
 149 {
 150         struct mm_struct *mm = current->mm;
 151         struct vm_area_struct *vma;
 152         struct hstate *h = hstate_file(file);
 153         struct vm_unmapped_area_info info;
 154
 155         if (len & ~huge_page_mask(h))
 156                 return -EINVAL;
 157         if (len > TASK_SIZE)
 158                 return -ENOMEM;
 159
 160         if (flags & MAP_FIXED) {
 161                 if (prepare_hugepage_range(file, addr, len))
 162                         return -EINVAL;
 163                 return addr;
 164         }
 165
 166         if (addr) {
 167                 addr = ALIGN(addr, huge_page_size(h));
 168                 vma = find_vma(mm, addr);
 169                 if (TASK_SIZE - len >= addr &&
 170                     (!vma || addr + len <= vma->vm_start))
 171                         return addr;
 172         }
 173
 174         info.flags = 0;
 175         info.length = len;
 176         info.low_limit = TASK_UNMAPPED_BASE;
 177         info.high_limit = TASK_SIZE;
 178         info.align_mask = PAGE_MASK & ~huge_page_mask(h);
 179         info.align_offset = 0;
 180         return vm_unmapped_area(&info);
 181 }
 182 #endif
 183
 184 static size_t
 185 hugetlbfs_read_actor(struct page *page, unsigned long offset,
 186                         struct iov_iter *to, unsigned long size)
 187 {
 188         size_t copied = 0;
 189         int i, chunksize;
 190
 191         /* Find which 4k chunk and offset with in that chunk */
 192         i = offset >> PAGE_CACHE_SHIFT;
 193         offset = offset & ~PAGE_CACHE_MASK;
 194
 195         while (size) {
 196                 size_t n;
 197                 chunksize = PAGE_CACHE_SIZE;
 198                 if (offset)
 199                         chunksize -= offset;
 200                 if (chunksize > size)
 201                         chunksize = size;
 202                 n = copy_page_to_iter(&page[i], offset, chunksize, to);
 203                 copied += n;
 204                 if (n != chunksize)
 205                         return copied;
 206                 offset = 0;
 207                 size -= chunksize;
 208                 i++;
 209         }
 210         return copied;
 211 }
 212
 213 /*
 214  * Support for read() - Find the page attached to f_mapping and copy out the
 215  * data. Its *very* similar to do_generic_mapping_read(), we can't use that
 216  * since it has PAGE_CACHE_SIZE assumptions.
 217  */
 218 static ssize_t hugetlbfs_read_iter(struct kiocb *iocb, struct iov_iter *to)
 219 {
 220         struct file *file = iocb->ki_filp;
 221         struct hstate *h = hstate_file(file);
 222         struct address_space *mapping = file->f_mapping;
 223         struct inode *inode = mapping->host;
 224         unsigned long index = iocb->ki_pos >> huge_page_shift(h);
 225         unsigned long offset = iocb->ki_pos & ~huge_page_mask(h);
 226         unsigned long end_index;
 227         loff_t isize;
 228         ssize_t retval = 0;
 229
 230         while (iov_iter_count(to)) {
 231                 struct page *page;
 232                 size_t nr, copied;
 233
 234                 /* nr is the maximum number of bytes to copy from this page */
 235                 nr = huge_page_size(h);
 236                 isize = i_size_read(inode);
 237                 if (!isize)
 238                         break;
 239                 end_index = (isize - 1) >> huge_page_shift(h);
 240                 if (index > end_index)
 241                         break;
 242                 if (index == end_index) {
 243                         nr = ((isize - 1) & ~huge_page_mask(h)) + 1;
 244                         if (nr <= offset)
 245                                 break;
 246                 }
 247                 nr = nr - offset;
 248
 249                 /* Find the page */
 250                 page = find_lock_page(mapping, index);
 251                 if (unlikely(page == NULL)) {
 252                         /*
 253                          * We have a HOLE, zero out the user-buffer for the
 254                          * length of the hole or request.
 255                          */
 256                         copied = iov_iter_zero(nr, to);
 257                 } else {
 258                         unlock_page(page);
 259
 260                         /*
 261                          * We have the page, copy it to user space buffer.
 262                          */
 263                         copied = hugetlbfs_read_actor(page, offset, to, nr);
 264                         page_cache_release(page);
 265                 }
 266                 offset += copied;
 267                 retval += copied;
 268                 if (copied != nr && iov_iter_count(to)) {
 269                         if (!retval)
 270                                 retval = -EFAULT;
 271                         break;
 272                 }
 273                 index += offset >> huge_page_shift(h);
 274                 offset &= ~huge_page_mask(h);
 275         }
 276         iocb->ki_pos = ((loff_t)index << huge_page_shift(h)) + offset;
 277         return retval;
 278 }
 279
 280 static int hugetlbfs_write_begin(struct file *file,
 281                         struct address_space *mapping,
 282                         loff_t pos, unsigned len, unsigned flags,
 283                         struct page **pagep, void **fsdata)
 284 {
 285         return -EINVAL;
 286 }
 287
 288 static int hugetlbfs_write_end(struct file *file, struct address_space *mapping,
 289                         loff_t pos, unsigned len, unsigned copied,
 290                         struct page *page, void *fsdata)
 291 {
 292         BUG();
 293         return -EINVAL;
 294 }
 295
 296 static void remove_huge_page(struct page *page)
 297 {
 298         ClearPageDirty(page);
 299         ClearPageUptodate(page);
 300         delete_from_page_cache(page);
 301 }
 302
 303
 304 /*
 305  * remove_inode_hugepages handles two distinct cases: truncation and hole
 306  * punch.  There are subtle differences in operation for each case.
 307
 308  * truncation is indicated by end of range being LLONG_MAX
 309  *      In this case, we first scan the range and release found pages.
 310  *      After releasing pages, hugetlb_unreserve_pages cleans up region/reserv
 311  *      maps and global counts.
 312  * hole punch is indicated if end is not LLONG_MAX
 313  *      In the hole punch case we scan the range and release found pages.
 314  *      Only when releasing a page is the associated region/reserv map
 315  *      deleted.  The region/reserv map for ranges without associated
 316  *      pages are not modified.
 317  * Note: If the passed end of range value is beyond the end of file, but
 318  * not LLONG_MAX this routine still performs a hole punch operation.
 319  */
 320 static void remove_inode_hugepages(struct inode *inode, loff_t lstart,
 321                                    loff_t lend)
 322 {
 323         struct hstate *h = hstate_inode(inode);
 324         struct address_space *mapping = &inode->i_data;
 325         const pgoff_t start = lstart >> huge_page_shift(h);
 326         const pgoff_t end = lend >> huge_page_shift(h);
 327         struct vm_area_struct pseudo_vma;
 328         struct pagevec pvec;
 329         pgoff_t next;
 330         int i, freed = 0;
 331         long lookup_nr = PAGEVEC_SIZE;
 332         bool truncate_op = (lend == LLONG_MAX);
 333
 334         memset(&pseudo_vma, 0, sizeof(struct vm_area_struct));
 335         pseudo_vma.vm_flags = (VM_HUGETLB | VM_MAYSHARE | VM_SHARED);
 336         pagevec_init(&pvec, 0);
 337         next = start;
 338         while (next < end) {
 339                 /*
 340                  * Make sure to never grab more pages that we
 341                  * might possibly need.
 342                  */
 343                 if (end - next < lookup_nr)
 344                         lookup_nr = end - next;
 345
 346                 /*
 347                  * This pagevec_lookup() may return pages past 'end',
 348                  * so we must check for page->index > end.
 349                  */
 350                 if (!pagevec_lookup(&pvec, mapping, next, lookup_nr)) {
 351                         if (next == start)
 352                                 break;
 353                         next = start;
 354                         continue;
 355                 }
 356
 357                 for (i = 0; i < pagevec_count(&pvec); ++i) {
 358                         struct page *page = pvec.pages[i];
 359                         u32 hash;
 360
 361                         hash = hugetlb_fault_mutex_hash(h, current->mm,
 362                                                         &pseudo_vma,
 363                                                         mapping, next, 0);
 364                         mutex_lock(&hugetlb_fault_mutex_table[hash]);
 365
 366                         lock_page(page);
 367                         if (page->index >= end) {
 368                                 unlock_page(page);
 369                                 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
 370                                 next = end;     /* we are done */
 371                                 break;
 372                         }
 373
 374                         /*
 375                          * If page is mapped, it was faulted in after being
 376                          * unmapped.  Do nothing in this race case.  In the
 377                          * normal case page is not mapped.
 378                          */
 379                         if (!page_mapped(page)) {
 380                                 bool rsv_on_error = !PagePrivate(page);
 381                                 /*
 382                                  * We must free the huge page and remove
 383                                  * from page cache (remove_huge_page) BEFORE
 384                                  * removing the region/reserve map
 385                                  * (hugetlb_unreserve_pages).  In rare out
 386                                  * of memory conditions, removal of the
 387                                  * region/reserve map could fail.  Before
 388                                  * free'ing the page, note PagePrivate which
 389                                  * is used in case of error.
 390                                  */
 391                                 remove_huge_page(page);
 392                                 freed++;
 393                                 if (!truncate_op) {
 394                                         if (unlikely(hugetlb_unreserve_pages(
 395                                                         inode, next,
 396                                                         next + 1, 1)))
 397                                                 hugetlb_fix_reserve_counts(
 398                                                         inode, rsv_on_error);
 399                                 }
 400                         }
 401
 402                         if (page->index > next)
 403                                 next = page->index;
 404
 405                         ++next;
 406                         unlock_page(page);
 407
 408                         mutex_unlock(&hugetlb_fault_mutex_table[hash]);
 409                 }
 410                 huge_pagevec_release(&pvec);
 411         }
 412
 413         if (truncate_op)
 414                 (void)hugetlb_unreserve_pages(inode, start, LONG_MAX, freed);
 415 }
 416
 417 static void hugetlbfs_evict_inode(struct inode *inode)
 418 {
 419         struct resv_map *resv_map;
 420
 421         remove_inode_hugepages(inode, 0, LLONG_MAX);
 422         resv_map = (struct resv_map *)inode->i_mapping->private_data;
 423         /* root inode doesn't have the resv_map, so we should check it */
 424         if (resv_map)
 425                 resv_map_release(&resv_map->refs);
 426         clear_inode(inode);
 427 }
 428
 429 static inline void
 430 hugetlb_vmdelete_list(struct rb_root *root, pgoff_t start, pgoff_t end)
 431 {
 432         struct vm_area_struct *vma;
 433
 434         /*
 435          * end == 0 indicates that the entire range after
 436          * start should be unmapped.
 437          */
 438         vma_interval_tree_foreach(vma, root, start, end ? end : ULONG_MAX) {
 439                 unsigned long v_offset;
 440
 441                 /*
 442                  * Can the expression below overflow on 32-bit arches?
 443                  * No, because the interval tree returns us only those vmas
 444                  * which overlap the truncated area starting at pgoff,
 445                  * and no vma on a 32-bit arch can span beyond the 4GB.
 446                  */
 447                 if (vma->vm_pgoff < start)
 448                         v_offset = (start - vma->vm_pgoff) << PAGE_SHIFT;
 449                 else
 450                         v_offset = 0;
 451
 452                 if (end) {
 453                         end = ((end - start) << PAGE_SHIFT) +
 454                                vma->vm_start + v_offset;
 455                         if (end > vma->vm_end)
 456                                 end = vma->vm_end;
 457                 } else
 458                         end = vma->vm_end;
 459
 460                 unmap_hugepage_range(vma, vma->vm_start + v_offset, end, NULL);
 461         }
 462 }
 463
 464 static int hugetlb_vmtruncate(struct inode *inode, loff_t offset)
 465 {
 466         pgoff_t pgoff;
 467         struct address_space *mapping = inode->i_mapping;
 468         struct hstate *h = hstate_inode(inode);
 469
 470         BUG_ON(offset & ~huge_page_mask(h));
 471         pgoff = offset >> PAGE_SHIFT;
 472
 473         i_size_write(inode, offset);
 474         i_mmap_lock_write(mapping);
 475         if (!RB_EMPTY_ROOT(&mapping->i_mmap))
 476                 hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0);
 477         i_mmap_unlock_write(mapping);
 478         remove_inode_hugepages(inode, offset, LLONG_MAX);
 479         return 0;
 480 }
 481
 482 static int hugetlbfs_setattr(struct dentry *dentry, struct iattr *attr)
 483 {
 484         struct inode *inode = d_inode(dentry);
 485         struct hstate *h = hstate_inode(inode);
 486         int error;
 487         unsigned int ia_valid = attr->ia_valid;
 488
 489         BUG_ON(!inode);
 490
 491         error = inode_change_ok(inode, attr);
 492         if (error)
 493                 return error;
 494
 495         if (ia_valid & ATTR_SIZE) {
 496                 error = -EINVAL;
 497                 if (attr->ia_size & ~huge_page_mask(h))
 498                         return -EINVAL;
 499                 error = hugetlb_vmtruncate(inode, attr->ia_size);
 500                 if (error)
 501                         return error;
 502         }
 503
 504         setattr_copy(inode, attr);
 505         mark_inode_dirty(inode);
 506         return 0;
 507 }
 508
 509 static struct inode *hugetlbfs_get_root(struct super_block *sb,
 510                                         struct hugetlbfs_config *config)
 511 {
 512         struct inode *inode;
 513
 514         inode = new_inode(sb);
 515         if (inode) {
 516                 struct hugetlbfs_inode_info *info;
 517                 inode->i_ino = get_next_ino();
 518                 inode->i_mode = S_IFDIR | config->mode;
 519                 inode->i_uid = config->uid;
 520                 inode->i_gid = config->gid;
 521                 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
 522                 info = HUGETLBFS_I(inode);
 523                 mpol_shared_policy_init(&info->policy, NULL);
 524                 inode->i_op = &hugetlbfs_dir_inode_operations;
 525                 inode->i_fop = &simple_dir_operations;
 526                 /* directory inodes start off with i_nlink == 2 (for "." entry) */
 527                 inc_nlink(inode);
 528                 lockdep_annotate_inode_mutex_key(inode);
 529         }
 530         return inode;
 531 }
 532
 533 /*
 534  * Hugetlbfs is not reclaimable; therefore its i_mmap_rwsem will never
 535  * be taken from reclaim -- unlike regular filesystems. This needs an
 536  * annotation because huge_pmd_share() does an allocation under
 537  * i_mmap_rwsem.
 538  */
 539 static struct lock_class_key hugetlbfs_i_mmap_rwsem_key;
 540
 541 static struct inode *hugetlbfs_get_inode(struct super_block *sb,
 542                                         struct inode *dir,
 543                                         umode_t mode, dev_t dev)
 544 {
 545         struct inode *inode;
 546         struct resv_map *resv_map;
 547
 548         resv_map = resv_map_alloc();
 549         if (!resv_map)
 550                 return NULL;
 551
 552         inode = new_inode(sb);
 553         if (inode) {
 554                 struct hugetlbfs_inode_info *info;
 555                 inode->i_ino = get_next_ino();
 556                 inode_init_owner(inode, dir, mode);
 557                 lockdep_set_class(&inode->i_mapping->i_mmap_rwsem,
 558                                 &hugetlbfs_i_mmap_rwsem_key);
 559                 inode->i_mapping->a_ops = &hugetlbfs_aops;
 560                 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
 561                 inode->i_mapping->private_data = resv_map;
 562                 info = HUGETLBFS_I(inode);
 563                 /*
 564                  * The policy is initialized here even if we are creating a
 565                  * private inode because initialization simply creates an
 566                  * an empty rb tree and calls spin_lock_init(), later when we
 567                  * call mpol_free_shared_policy() it will just return because
 568                  * the rb tree will still be empty.
 569                  */
 570                 mpol_shared_policy_init(&info->policy, NULL);
 571                 switch (mode & S_IFMT) {
 572                 default:
 573                         init_special_inode(inode, mode, dev);
 574                         break;
 575                 case S_IFREG:
 576                         inode->i_op = &hugetlbfs_inode_operations;
 577                         inode->i_fop = &hugetlbfs_file_operations;
 578                         break;
 579                 case S_IFDIR:
 580                         inode->i_op = &hugetlbfs_dir_inode_operations;
 581                         inode->i_fop = &simple_dir_operations;
 582
 583                         /* directory inodes start off with i_nlink == 2 (for "." entry) */
 584                         inc_nlink(inode);
 585                         break;
 586                 case S_IFLNK:
 587                         inode->i_op = &page_symlink_inode_operations;
 588                         break;
 589                 }
 590                 lockdep_annotate_inode_mutex_key(inode);
 591         } else
 592                 kref_put(&resv_map->refs, resv_map_release);
 593
 594         return inode;
 595 }
 596
 597 /*
 598  * File creation. Allocate an inode, and we're done..
 599  */
 600 static int hugetlbfs_mknod(struct inode *dir,
 601                         struct dentry *dentry, umode_t mode, dev_t dev)
 602 {
 603         struct inode *inode;
 604         int error = -ENOSPC;
 605
 606         inode = hugetlbfs_get_inode(dir->i_sb, dir, mode, dev);
 607         if (inode) {
 608                 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
 609                 d_instantiate(dentry, inode);
 610                 dget(dentry);   /* Extra count - pin the dentry in core */
 611                 error = 0;
 612         }
 613         return error;
 614 }
 615
 616 static int hugetlbfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
 617 {
 618         int retval = hugetlbfs_mknod(dir, dentry, mode | S_IFDIR, 0);
 619         if (!retval)
 620                 inc_nlink(dir);
 621         return retval;
 622 }
 623
 624 static int hugetlbfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool excl)
 625 {
 626         return hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0);
 627 }
 628
 629 static int hugetlbfs_symlink(struct inode *dir,
 630                         struct dentry *dentry, const char *symname)
 631 {
 632         struct inode *inode;
 633         int error = -ENOSPC;
 634
 635         inode = hugetlbfs_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0);
 636         if (inode) {
 637                 int l = strlen(symname)+1;
 638                 error = page_symlink(inode, symname, l);
 639                 if (!error) {
 640                         d_instantiate(dentry, inode);
 641                         dget(dentry);
 642                 } else
 643                         iput(inode);
 644         }
 645         dir->i_ctime = dir->i_mtime = CURRENT_TIME;
 646
 647         return error;
 648 }
 649
 650 /*
 651  * mark the head page dirty
 652  */
 653 static int hugetlbfs_set_page_dirty(struct page *page)
 654 {
 655         struct page *head = compound_head(page);
 656
 657         SetPageDirty(head);
 658         return 0;
 659 }
 660
 661 static int hugetlbfs_migrate_page(struct address_space *mapping,
 662                                 struct page *newpage, struct page *page,
 663                                 enum migrate_mode mode)
 664 {
 665         int rc;
 666
 667         rc = migrate_huge_page_move_mapping(mapping, newpage, page);
 668         if (rc != MIGRATEPAGE_SUCCESS)
 669                 return rc;
 670         migrate_page_copy(newpage, page);
 671
 672         return MIGRATEPAGE_SUCCESS;
 673 }
 674
 675 static int hugetlbfs_statfs(struct dentry *dentry, struct kstatfs *buf)
 676 {
 677         struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(dentry->d_sb);
 678         struct hstate *h = hstate_inode(d_inode(dentry));
 679
 680         buf->f_type = HUGETLBFS_MAGIC;
 681         buf->f_bsize = huge_page_size(h);
 682         if (sbinfo) {
 683                 spin_lock(&sbinfo->stat_lock);
 684                 /* If no limits set, just report 0 for max/free/used
 685                  * blocks, like simple_statfs() */
 686                 if (sbinfo->spool) {
 687                         long free_pages;
 688
 689                         spin_lock(&sbinfo->spool->lock);
 690                         buf->f_blocks = sbinfo->spool->max_hpages;
 691                         free_pages = sbinfo->spool->max_hpages
 692                                 - sbinfo->spool->used_hpages;
 693                         buf->f_bavail = buf->f_bfree = free_pages;
 694                         spin_unlock(&sbinfo->spool->lock);
 695                         buf->f_files = sbinfo->max_inodes;
 696                         buf->f_ffree = sbinfo->free_inodes;
 697                 }
 698                 spin_unlock(&sbinfo->stat_lock);
 699         }
 700         buf->f_namelen = NAME_MAX;
 701         return 0;
 702 }
 703
 704 static void hugetlbfs_put_super(struct super_block *sb)
 705 {
 706         struct hugetlbfs_sb_info *sbi = HUGETLBFS_SB(sb);
 707
 708         if (sbi) {
 709                 sb->s_fs_info = NULL;
 710
 711                 if (sbi->spool)
 712                         hugepage_put_subpool(sbi->spool);
 713
 714                 kfree(sbi);
 715         }
 716 }
 717
 718 static inline int hugetlbfs_dec_free_inodes(struct hugetlbfs_sb_info *sbinfo)
 719 {
 720         if (sbinfo->free_inodes >= 0) {
 721                 spin_lock(&sbinfo->stat_lock);
 722                 if (unlikely(!sbinfo->free_inodes)) {
 723                         spin_unlock(&sbinfo->stat_lock);
 724                         return 0;
 725                 }
 726                 sbinfo->free_inodes--;
 727                 spin_unlock(&sbinfo->stat_lock);
 728         }
 729
 730         return 1;
 731 }
 732
 733 static void hugetlbfs_inc_free_inodes(struct hugetlbfs_sb_info *sbinfo)
 734 {
 735         if (sbinfo->free_inodes >= 0) {
 736                 spin_lock(&sbinfo->stat_lock);
 737                 sbinfo->free_inodes++;
 738                 spin_unlock(&sbinfo->stat_lock);
 739         }
 740 }
 741
 742
 743 static struct kmem_cache *hugetlbfs_inode_cachep;
 744
 745 static struct inode *hugetlbfs_alloc_inode(struct super_block *sb)
 746 {
 747         struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(sb);
 748         struct hugetlbfs_inode_info *p;
 749
 750         if (unlikely(!hugetlbfs_dec_free_inodes(sbinfo)))
 751                 return NULL;
 752         p = kmem_cache_alloc(hugetlbfs_inode_cachep, GFP_KERNEL);
 753         if (unlikely(!p)) {
 754                 hugetlbfs_inc_free_inodes(sbinfo);
 755                 return NULL;
 756         }
 757         return &p->vfs_inode;
 758 }
 759
 760 static void hugetlbfs_i_callback(struct rcu_head *head)
 761 {
 762         struct inode *inode = container_of(head, struct inode, i_rcu);
 763         kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));
 764 }
 765
 766 static void hugetlbfs_destroy_inode(struct inode *inode)
 767 {
 768         hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));
 769         mpol_free_shared_policy(&HUGETLBFS_I(inode)->policy);
 770         call_rcu(&inode->i_rcu, hugetlbfs_i_callback);
 771 }
 772
 773 static const struct address_space_operations hugetlbfs_aops = {
 774         .write_begin    = hugetlbfs_write_begin,
 775         .write_end      = hugetlbfs_write_end,
 776         .set_page_dirty = hugetlbfs_set_page_dirty,
 777         .migratepage    = hugetlbfs_migrate_page,
 778 };
 779
 780
 781 static void init_once(void *foo)
 782 {
 783         struct hugetlbfs_inode_info *ei = (struct hugetlbfs_inode_info *)foo;
 784
 785         inode_init_once(&ei->vfs_inode);
 786 }
 787
 788 const struct file_operations hugetlbfs_file_operations = {
 789         .read_iter              = hugetlbfs_read_iter,
 790         .mmap                   = hugetlbfs_file_mmap,
 791         .fsync                  = noop_fsync,
 792         .get_unmapped_area      = hugetlb_get_unmapped_area,
 793         .llseek         = default_llseek,
 794 };
 795
 796 static const struct inode_operations hugetlbfs_dir_inode_operations = {
 797         .create         = hugetlbfs_create,
 798         .lookup         = simple_lookup,
 799         .link           = simple_link,
 800         .unlink         = simple_unlink,
 801         .symlink        = hugetlbfs_symlink,
 802         .mkdir          = hugetlbfs_mkdir,
 803         .rmdir          = simple_rmdir,
 804         .mknod          = hugetlbfs_mknod,
 805         .rename         = simple_rename,
 806         .setattr        = hugetlbfs_setattr,
 807 };
 808
 809 static const struct inode_operations hugetlbfs_inode_operations = {
 810         .setattr        = hugetlbfs_setattr,
 811 };
 812
 813 static const struct super_operations hugetlbfs_ops = {
 814         .alloc_inode    = hugetlbfs_alloc_inode,
 815         .destroy_inode  = hugetlbfs_destroy_inode,
 816         .evict_inode    = hugetlbfs_evict_inode,
 817         .statfs         = hugetlbfs_statfs,
 818         .put_super      = hugetlbfs_put_super,
 819         .show_options   = generic_show_options,
 820 };
 821
 822 enum { NO_SIZE, SIZE_STD, SIZE_PERCENT };
 823
 824 /*
 825  * Convert size option passed from command line to number of huge pages
 826  * in the pool specified by hstate.  Size option could be in bytes
 827  * (val_type == SIZE_STD) or percentage of the pool (val_type == SIZE_PERCENT).
 828  */
 829 static long long
 830 hugetlbfs_size_to_hpages(struct hstate *h, unsigned long long size_opt,
 831                                                                 int val_type)
 832 {
 833         if (val_type == NO_SIZE)
 834                 return -1;
 835
 836         if (val_type == SIZE_PERCENT) {
 837                 size_opt <<= huge_page_shift(h);
 838                 size_opt *= h->max_huge_pages;
 839                 do_div(size_opt, 100);
 840         }
 841
 842         size_opt >>= huge_page_shift(h);
 843         return size_opt;
 844 }
 845
 846 static int
 847 hugetlbfs_parse_options(char *options, struct hugetlbfs_config *pconfig)
 848 {
 849         char *p, *rest;
 850         substring_t args[MAX_OPT_ARGS];
 851         int option;
 852         unsigned long long max_size_opt = 0, min_size_opt = 0;
 853         int max_val_type = NO_SIZE, min_val_type = NO_SIZE;
 854
 855         if (!options)
 856                 return 0;
 857
 858         while ((p = strsep(&options, ",")) != NULL) {
 859                 int token;
 860                 if (!*p)
 861                         continue;
 862
 863                 token = match_token(p, tokens, args);
 864                 switch (token) {
 865                 case Opt_uid:
 866                         if (match_int(&args[0], &option))
 867                                 goto bad_val;
 868                         pconfig->uid = make_kuid(current_user_ns(), option);
 869                         if (!uid_valid(pconfig->uid))
 870                                 goto bad_val;
 871                         break;
 872
 873                 case Opt_gid:
 874                         if (match_int(&args[0], &option))
 875                                 goto bad_val;
 876                         pconfig->gid = make_kgid(current_user_ns(), option);
 877                         if (!gid_valid(pconfig->gid))
 878                                 goto bad_val;
 879                         break;
 880
 881                 case Opt_mode:
 882                         if (match_octal(&args[0], &option))
 883                                 goto bad_val;
 884                         pconfig->mode = option & 01777U;
 885                         break;
 886
 887                 case Opt_size: {
 888                         /* memparse() will accept a K/M/G without a digit */
 889                         if (!isdigit(*args[0].from))
 890                                 goto bad_val;
 891                         max_size_opt = memparse(args[0].from, &rest);
 892                         max_val_type = SIZE_STD;
 893                         if (*rest == '%')
 894                                 max_val_type = SIZE_PERCENT;
 895                         break;
 896                 }
 897
 898                 case Opt_nr_inodes:
 899                         /* memparse() will accept a K/M/G without a digit */
 900                         if (!isdigit(*args[0].from))
 901                                 goto bad_val;
 902                         pconfig->nr_inodes = memparse(args[0].from, &rest);
 903                         break;
 904
 905                 case Opt_pagesize: {
 906                         unsigned long ps;
 907                         ps = memparse(args[0].from, &rest);
 908                         pconfig->hstate = size_to_hstate(ps);
 909                         if (!pconfig->hstate) {
 910                                 pr_err("Unsupported page size %lu MB\n",
 911                                         ps >> 20);
 912                                 return -EINVAL;
 913                         }
 914                         break;
 915                 }
 916
 917                 case Opt_min_size: {
 918                         /* memparse() will accept a K/M/G without a digit */
 919                         if (!isdigit(*args[0].from))
 920                                 goto bad_val;
 921                         min_size_opt = memparse(args[0].from, &rest);
 922                         min_val_type = SIZE_STD;
 923                         if (*rest == '%')
 924                                 min_val_type = SIZE_PERCENT;
 925                         break;
 926                 }
 927
 928                 default:
 929                         pr_err("Bad mount option: \"%s\"\n", p);
 930                         return -EINVAL;
 931                         break;
 932                 }
 933         }
 934
 935         /*
 936          * Use huge page pool size (in hstate) to convert the size
 937          * options to number of huge pages.  If NO_SIZE, -1 is returned.
 938          */
 939         pconfig->max_hpages = hugetlbfs_size_to_hpages(pconfig->hstate,
 940                                                 max_size_opt, max_val_type);
 941         pconfig->min_hpages = hugetlbfs_size_to_hpages(pconfig->hstate,
 942                                                 min_size_opt, min_val_type);
 943
 944         /*
 945          * If max_size was specified, then min_size must be smaller
 946          */
 947         if (max_val_type > NO_SIZE &&
 948             pconfig->min_hpages > pconfig->max_hpages) {
 949                 pr_err("minimum size can not be greater than maximum size\n");
 950                 return -EINVAL;
 951         }
 952
 953         return 0;
 954
 955 bad_val:
 956         pr_err("Bad value '%s' for mount option '%s'\n", args[0].from, p);
 957         return -EINVAL;
 958 }
 959
 960 static int
 961 hugetlbfs_fill_super(struct super_block *sb, void *data, int silent)
 962 {
 963         int ret;
 964         struct hugetlbfs_config config;
 965         struct hugetlbfs_sb_info *sbinfo;
 966
 967         save_mount_options(sb, data);
 968
 969         config.max_hpages = -1; /* No limit on size by default */
 970         config.nr_inodes = -1; /* No limit on number of inodes by default */
 971         config.uid = current_fsuid();
 972         config.gid = current_fsgid();
 973         config.mode = 0755;
 974         config.hstate = &default_hstate;
 975         config.min_hpages = -1; /* No default minimum size */
 976         ret = hugetlbfs_parse_options(data, &config);
 977         if (ret)
 978                 return ret;
 979
 980         sbinfo = kmalloc(sizeof(struct hugetlbfs_sb_info), GFP_KERNEL);
 981         if (!sbinfo)
 982                 return -ENOMEM;
 983         sb->s_fs_info = sbinfo;
 984         sbinfo->hstate = config.hstate;
 985         spin_lock_init(&sbinfo->stat_lock);
 986         sbinfo->max_inodes = config.nr_inodes;
 987         sbinfo->free_inodes = config.nr_inodes;
 988         sbinfo->spool = NULL;
 989         /*
 990          * Allocate and initialize subpool if maximum or minimum size is
 991          * specified.  Any needed reservations (for minimim size) are taken
 992          * taken when the subpool is created.
 993          */
 994         if (config.max_hpages != -1 || config.min_hpages != -1) {
 995                 sbinfo->spool = hugepage_new_subpool(config.hstate,
 996                                                         config.max_hpages,
 997                                                         config.min_hpages);
 998                 if (!sbinfo->spool)
 999                         goto out_free;
1000         }
1001         sb->s_maxbytes = MAX_LFS_FILESIZE;
1002         sb->s_blocksize = huge_page_size(config.hstate);
1003         sb->s_blocksize_bits = huge_page_shift(config.hstate);
1004         sb->s_magic = HUGETLBFS_MAGIC;
1005         sb->s_op = &hugetlbfs_ops;
1006         sb->s_time_gran = 1;
1007         sb->s_root = d_make_root(hugetlbfs_get_root(sb, &config));
1008         if (!sb->s_root)
1009                 goto out_free;
1010         return 0;
1011 out_free:
1012         kfree(sbinfo->spool);
1013         kfree(sbinfo);
1014         return -ENOMEM;
1015 }
1016
1017 static struct dentry *hugetlbfs_mount(struct file_system_type *fs_type,
1018         int flags, const char *dev_name, void *data)
1019 {
1020         return mount_nodev(fs_type, flags, data, hugetlbfs_fill_super);
1021 }
1022
1023 static struct file_system_type hugetlbfs_fs_type = {
1024         .name           = "hugetlbfs",
1025         .mount          = hugetlbfs_mount,
1026         .kill_sb        = kill_litter_super,
1027 };
1028 MODULE_ALIAS_FS("hugetlbfs");
1029
1030 static struct vfsmount *hugetlbfs_vfsmount[HUGE_MAX_HSTATE];
1031
1032 static int can_do_hugetlb_shm(void)
1033 {
1034         kgid_t shm_group;
1035         shm_group = make_kgid(&init_user_ns, sysctl_hugetlb_shm_group);
1036         return capable(CAP_IPC_LOCK) || in_group_p(shm_group);
1037 }
1038
1039 static int get_hstate_idx(int page_size_log)
1040 {
1041         struct hstate *h = hstate_sizelog(page_size_log);
1042
1043         if (!h)
1044                 return -1;
1045         return h - hstates;
1046 }
1047
1048 static const struct dentry_operations anon_ops = {
1049         .d_dname = simple_dname
1050 };
1051
1052 /*
1053  * Note that size should be aligned to proper hugepage size in caller side,
1054  * otherwise hugetlb_reserve_pages reserves one less hugepages than intended.
1055  */
1056 struct file *hugetlb_file_setup(const char *name, size_t size,
1057                                 vm_flags_t acctflag, struct user_struct **user,
1058                                 int creat_flags, int page_size_log)
1059 {
1060         struct file *file = ERR_PTR(-ENOMEM);
1061         struct inode *inode;
1062         struct path path;
1063         struct super_block *sb;
1064         struct qstr quick_string;
1065         int hstate_idx;
1066
1067         hstate_idx = get_hstate_idx(page_size_log);
1068         if (hstate_idx < 0)
1069                 return ERR_PTR(-ENODEV);
1070
1071         *user = NULL;
1072         if (!hugetlbfs_vfsmount[hstate_idx])
1073                 return ERR_PTR(-ENOENT);
1074
1075         if (creat_flags == HUGETLB_SHMFS_INODE && !can_do_hugetlb_shm()) {
1076                 *user = current_user();
1077                 if (user_shm_lock(size, *user)) {
1078                         task_lock(current);
1079                         pr_warn_once("%s (%d): Using mlock ulimits for SHM_HUGETLB is deprecated\n",
1080                                 current->comm, current->pid);
1081                         task_unlock(current);
1082                 } else {
1083                         *user = NULL;
1084                         return ERR_PTR(-EPERM);
1085                 }
1086         }
1087
1088         sb = hugetlbfs_vfsmount[hstate_idx]->mnt_sb;
1089         quick_string.name = name;
1090         quick_string.len = strlen(quick_string.name);
1091         quick_string.hash = 0;
1092         path.dentry = d_alloc_pseudo(sb, &quick_string);
1093         if (!path.dentry)
1094                 goto out_shm_unlock;
1095
1096         d_set_d_op(path.dentry, &anon_ops);
1097         path.mnt = mntget(hugetlbfs_vfsmount[hstate_idx]);
1098         file = ERR_PTR(-ENOSPC);
1099         inode = hugetlbfs_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0);
1100         if (!inode)
1101                 goto out_dentry;
1102         if (creat_flags == HUGETLB_SHMFS_INODE)
1103                 inode->i_flags |= S_PRIVATE;
1104
1105         file = ERR_PTR(-ENOMEM);
1106         if (hugetlb_reserve_pages(inode, 0,
1107                         size >> huge_page_shift(hstate_inode(inode)), NULL,
1108                         acctflag))
1109                 goto out_inode;
1110
1111         d_instantiate(path.dentry, inode);
1112         inode->i_size = size;
1113         clear_nlink(inode);
1114
1115         file = alloc_file(&path, FMODE_WRITE | FMODE_READ,
1116                         &hugetlbfs_file_operations);
1117         if (IS_ERR(file))
1118                 goto out_dentry; /* inode is already attached */
1119
1120         return file;
1121
1122 out_inode:
1123         iput(inode);
1124 out_dentry:
1125         path_put(&path);
1126 out_shm_unlock:
1127         if (*user) {
1128                 user_shm_unlock(size, *user);
1129                 *user = NULL;
1130         }
1131         return file;
1132 }
1133
1134 static int __init init_hugetlbfs_fs(void)
1135 {
1136         struct hstate *h;
1137         int error;
1138         int i;
1139
1140         if (!hugepages_supported()) {
1141                 pr_info("disabling because there are no supported hugepage sizes\n");
1142                 return -ENOTSUPP;
1143         }
1144
1145         error = -ENOMEM;
1146         hugetlbfs_inode_cachep = kmem_cache_create("hugetlbfs_inode_cache",
1147                                         sizeof(struct hugetlbfs_inode_info),
1148                                         0, 0, init_once);
1149         if (hugetlbfs_inode_cachep == NULL)
1150                 goto out2;
1151
1152         error = register_filesystem(&hugetlbfs_fs_type);
1153         if (error)
1154                 goto out;
1155
1156         i = 0;
1157         for_each_hstate(h) {
1158                 char buf[50];
1159                 unsigned ps_kb = 1U << (h->order + PAGE_SHIFT - 10);
1160
1161                 snprintf(buf, sizeof(buf), "pagesize=%uK", ps_kb);
1162                 hugetlbfs_vfsmount[i] = kern_mount_data(&hugetlbfs_fs_type,
1163                                                         buf);
1164
1165                 if (IS_ERR(hugetlbfs_vfsmount[i])) {
1166                         pr_err("Cannot mount internal hugetlbfs for "
1167                                 "page size %uK", ps_kb);
1168                         error = PTR_ERR(hugetlbfs_vfsmount[i]);
1169                         hugetlbfs_vfsmount[i] = NULL;
1170                 }
1171                 i++;
1172         }
1173         /* Non default hstates are optional */
1174         if (!IS_ERR_OR_NULL(hugetlbfs_vfsmount[default_hstate_idx]))
1175                 return 0;
1176
1177  out:
1178         kmem_cache_destroy(hugetlbfs_inode_cachep);
1179  out2:
1180         return error;
1181 }
1182
1183 static void __exit exit_hugetlbfs_fs(void)
1184 {
1185         struct hstate *h;
1186         int i;
1187
1188
1189         /*
1190          * Make sure all delayed rcu free inodes are flushed before we
1191          * destroy cache.
1192          */
1193         rcu_barrier();
1194         kmem_cache_destroy(hugetlbfs_inode_cachep);
1195         i = 0;
1196         for_each_hstate(h)
1197                 kern_unmount(hugetlbfs_vfsmount[i++]);
1198         unregister_filesystem(&hugetlbfs_fs_type);
1199 }
1200
1201 module_init(init_hugetlbfs_fs)
1202 module_exit(exit_hugetlbfs_fs)
1203
1204 MODULE_LICENSE("GPL");