mm: migrate: support non-lru movable page migration
[linux-block.git] / include / linux / mm.h
CommitLineData
1da177e4
LT
1#ifndef _LINUX_MM_H
2#define _LINUX_MM_H
3
1da177e4
LT
4#include <linux/errno.h>
5
6#ifdef __KERNEL__
7
309381fe 8#include <linux/mmdebug.h>
1da177e4 9#include <linux/gfp.h>
187f1882 10#include <linux/bug.h>
1da177e4
LT
11#include <linux/list.h>
12#include <linux/mmzone.h>
13#include <linux/rbtree.h>
83aeeada 14#include <linux/atomic.h>
9a11b49a 15#include <linux/debug_locks.h>
5b99cd0e 16#include <linux/mm_types.h>
08677214 17#include <linux/range.h>
c6f6b596 18#include <linux/pfn.h>
3565fce3 19#include <linux/percpu-refcount.h>
e9da73d6 20#include <linux/bit_spinlock.h>
b0d40c92 21#include <linux/shrinker.h>
9c599024 22#include <linux/resource.h>
e30825f1 23#include <linux/page_ext.h>
8025e5dd 24#include <linux/err.h>
fe896d18 25#include <linux/page_ref.h>
1da177e4
LT
26
27struct mempolicy;
28struct anon_vma;
bf181b9f 29struct anon_vma_chain;
4e950f6f 30struct file_ra_state;
e8edc6e0 31struct user_struct;
4e950f6f 32struct writeback_control;
682aa8e1 33struct bdi_writeback;
1da177e4 34
fccc9987 35#ifndef CONFIG_NEED_MULTIPLE_NODES /* Don't use mapnrs, do it properly */
1da177e4 36extern unsigned long max_mapnr;
fccc9987
JL
37
38static inline void set_max_mapnr(unsigned long limit)
39{
40 max_mapnr = limit;
41}
42#else
43static inline void set_max_mapnr(unsigned long limit) { }
1da177e4
LT
44#endif
45
4481374c 46extern unsigned long totalram_pages;
1da177e4 47extern void * high_memory;
1da177e4
LT
48extern int page_cluster;
49
50#ifdef CONFIG_SYSCTL
51extern int sysctl_legacy_va_layout;
52#else
53#define sysctl_legacy_va_layout 0
54#endif
55
d07e2259
DC
56#ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
57extern const int mmap_rnd_bits_min;
58extern const int mmap_rnd_bits_max;
59extern int mmap_rnd_bits __read_mostly;
60#endif
61#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
62extern const int mmap_rnd_compat_bits_min;
63extern const int mmap_rnd_compat_bits_max;
64extern int mmap_rnd_compat_bits __read_mostly;
65#endif
66
1da177e4
LT
67#include <asm/page.h>
68#include <asm/pgtable.h>
69#include <asm/processor.h>
1da177e4 70
79442ed1
TC
71#ifndef __pa_symbol
72#define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0))
73#endif
74
1dff8083
AB
75#ifndef page_to_virt
76#define page_to_virt(x) __va(PFN_PHYS(page_to_pfn(x)))
77#endif
78
593befa6
DD
79/*
80 * To prevent common memory management code establishing
81 * a zero page mapping on a read fault.
82 * This macro should be defined within <asm/pgtable.h>.
83 * s390 does this to prevent multiplexing of hardware bits
84 * related to the physical page in case of virtualization.
85 */
86#ifndef mm_forbids_zeropage
87#define mm_forbids_zeropage(X) (0)
88#endif
89
ea606cf5
AR
90/*
91 * Default maximum number of active map areas, this limits the number of vmas
92 * per mm struct. Users can overwrite this number by sysctl but there is a
93 * problem.
94 *
95 * When a program's coredump is generated as ELF format, a section is created
96 * per a vma. In ELF, the number of sections is represented in unsigned short.
97 * This means the number of sections should be smaller than 65535 at coredump.
98 * Because the kernel adds some informative sections to a image of program at
99 * generating coredump, we need some margin. The number of extra sections is
100 * 1-3 now and depends on arch. We use "5" as safe margin, here.
101 *
102 * ELF extended numbering allows more than 65535 sections, so 16-bit bound is
103 * not a hard limit any more. Although some userspace tools can be surprised by
104 * that.
105 */
106#define MAPCOUNT_ELF_CORE_MARGIN (5)
107#define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
108
109extern int sysctl_max_map_count;
110
c9b1d098 111extern unsigned long sysctl_user_reserve_kbytes;
4eeab4f5 112extern unsigned long sysctl_admin_reserve_kbytes;
c9b1d098 113
49f0ce5f
JM
114extern int sysctl_overcommit_memory;
115extern int sysctl_overcommit_ratio;
116extern unsigned long sysctl_overcommit_kbytes;
117
118extern int overcommit_ratio_handler(struct ctl_table *, int, void __user *,
119 size_t *, loff_t *);
120extern int overcommit_kbytes_handler(struct ctl_table *, int, void __user *,
121 size_t *, loff_t *);
122
1da177e4
LT
123#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
124
27ac792c
AR
125/* to align the pointer to the (next) page boundary */
126#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
127
0fa73b86
AM
128/* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */
129#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)addr, PAGE_SIZE)
130
1da177e4
LT
131/*
132 * Linux kernel virtual memory manager primitives.
133 * The idea being to have a "virtual" mm in the same way
134 * we have a virtual fs - giving a cleaner interface to the
135 * mm details, and allowing different kinds of memory mappings
136 * (from shared memory to executable loading to arbitrary
137 * mmap() functions).
138 */
139
c43692e8
CL
140extern struct kmem_cache *vm_area_cachep;
141
1da177e4 142#ifndef CONFIG_MMU
8feae131
DH
143extern struct rb_root nommu_region_tree;
144extern struct rw_semaphore nommu_region_sem;
1da177e4
LT
145
146extern unsigned int kobjsize(const void *objp);
147#endif
148
149/*
605d9288 150 * vm_flags in vm_area_struct, see mm_types.h.
bcf66917 151 * When changing, update also include/trace/events/mmflags.h
1da177e4 152 */
cc2383ec
KK
153#define VM_NONE 0x00000000
154
1da177e4
LT
155#define VM_READ 0x00000001 /* currently active flags */
156#define VM_WRITE 0x00000002
157#define VM_EXEC 0x00000004
158#define VM_SHARED 0x00000008
159
7e2cff42 160/* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */
1da177e4
LT
161#define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */
162#define VM_MAYWRITE 0x00000020
163#define VM_MAYEXEC 0x00000040
164#define VM_MAYSHARE 0x00000080
165
166#define VM_GROWSDOWN 0x00000100 /* general info on the segment */
16ba6f81 167#define VM_UFFD_MISSING 0x00000200 /* missing pages tracking */
6aab341e 168#define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */
1da177e4 169#define VM_DENYWRITE 0x00000800 /* ETXTBSY on write attempts.. */
16ba6f81 170#define VM_UFFD_WP 0x00001000 /* wrprotect pages tracking */
1da177e4 171
1da177e4
LT
172#define VM_LOCKED 0x00002000
173#define VM_IO 0x00004000 /* Memory mapped I/O or similar */
174
175 /* Used by sys_madvise() */
176#define VM_SEQ_READ 0x00008000 /* App will access data sequentially */
177#define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */
178
179#define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */
180#define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */
de60f5f1 181#define VM_LOCKONFAULT 0x00080000 /* Lock the pages covered when they are faulted in */
1da177e4 182#define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */
cdfd4325 183#define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */
1da177e4 184#define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */
cc2383ec 185#define VM_ARCH_1 0x01000000 /* Architecture-specific flag */
4aae7e43 186#define VM_ARCH_2 0x02000000
0103bd16 187#define VM_DONTDUMP 0x04000000 /* Do not include in the core dump */
d00806b1 188
d9104d1c
CG
189#ifdef CONFIG_MEM_SOFT_DIRTY
190# define VM_SOFTDIRTY 0x08000000 /* Not soft dirty clean area */
191#else
192# define VM_SOFTDIRTY 0
193#endif
194
b379d790 195#define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */
cc2383ec
KK
196#define VM_HUGEPAGE 0x20000000 /* MADV_HUGEPAGE marked this vma */
197#define VM_NOHUGEPAGE 0x40000000 /* MADV_NOHUGEPAGE marked this vma */
f8af4da3 198#define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */
1da177e4 199
63c17fb8
DH
200#ifdef CONFIG_ARCH_USES_HIGH_VMA_FLAGS
201#define VM_HIGH_ARCH_BIT_0 32 /* bit only usable on 64-bit architectures */
202#define VM_HIGH_ARCH_BIT_1 33 /* bit only usable on 64-bit architectures */
203#define VM_HIGH_ARCH_BIT_2 34 /* bit only usable on 64-bit architectures */
204#define VM_HIGH_ARCH_BIT_3 35 /* bit only usable on 64-bit architectures */
205#define VM_HIGH_ARCH_0 BIT(VM_HIGH_ARCH_BIT_0)
206#define VM_HIGH_ARCH_1 BIT(VM_HIGH_ARCH_BIT_1)
207#define VM_HIGH_ARCH_2 BIT(VM_HIGH_ARCH_BIT_2)
208#define VM_HIGH_ARCH_3 BIT(VM_HIGH_ARCH_BIT_3)
209#endif /* CONFIG_ARCH_USES_HIGH_VMA_FLAGS */
210
cc2383ec
KK
211#if defined(CONFIG_X86)
212# define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */
8f62c883
DH
213#if defined (CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS)
214# define VM_PKEY_SHIFT VM_HIGH_ARCH_BIT_0
215# define VM_PKEY_BIT0 VM_HIGH_ARCH_0 /* A protection key is a 4-bit value */
216# define VM_PKEY_BIT1 VM_HIGH_ARCH_1
217# define VM_PKEY_BIT2 VM_HIGH_ARCH_2
218# define VM_PKEY_BIT3 VM_HIGH_ARCH_3
219#endif
cc2383ec
KK
220#elif defined(CONFIG_PPC)
221# define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */
222#elif defined(CONFIG_PARISC)
223# define VM_GROWSUP VM_ARCH_1
9ca52ed9
JH
224#elif defined(CONFIG_METAG)
225# define VM_GROWSUP VM_ARCH_1
cc2383ec
KK
226#elif defined(CONFIG_IA64)
227# define VM_GROWSUP VM_ARCH_1
228#elif !defined(CONFIG_MMU)
229# define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */
230#endif
231
4aae7e43
QR
232#if defined(CONFIG_X86)
233/* MPX specific bounds table or bounds directory */
234# define VM_MPX VM_ARCH_2
235#endif
236
cc2383ec
KK
237#ifndef VM_GROWSUP
238# define VM_GROWSUP VM_NONE
239#endif
240
a8bef8ff
MG
241/* Bits set in the VMA until the stack is in its final location */
242#define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ)
243
1da177e4
LT
244#ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */
245#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
246#endif
247
248#ifdef CONFIG_STACK_GROWSUP
30bdbb78 249#define VM_STACK VM_GROWSUP
1da177e4 250#else
30bdbb78 251#define VM_STACK VM_GROWSDOWN
1da177e4
LT
252#endif
253
30bdbb78
KK
254#define VM_STACK_FLAGS (VM_STACK | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
255
b291f000 256/*
78f11a25
AA
257 * Special vmas that are non-mergable, non-mlock()able.
258 * Note: mm/huge_memory.c VM_NO_THP depends on this definition.
b291f000 259 */
9050d7eb 260#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP)
b291f000 261
a0715cc2
AT
262/* This mask defines which mm->def_flags a process can inherit its parent */
263#define VM_INIT_DEF_MASK VM_NOHUGEPAGE
264
de60f5f1
EM
265/* This mask is used to clear all the VMA flags used by mlock */
266#define VM_LOCKED_CLEAR_MASK (~(VM_LOCKED | VM_LOCKONFAULT))
267
1da177e4
LT
268/*
269 * mapping from the currently active vm_flags protection bits (the
270 * low four bits) to a page protection mask..
271 */
272extern pgprot_t protection_map[16];
273
d0217ac0 274#define FAULT_FLAG_WRITE 0x01 /* Fault was a write access */
9b4bdd2f
KS
275#define FAULT_FLAG_MKWRITE 0x02 /* Fault was mkwrite of existing pte */
276#define FAULT_FLAG_ALLOW_RETRY 0x04 /* Retry fault if blocking */
277#define FAULT_FLAG_RETRY_NOWAIT 0x08 /* Don't drop mmap_sem and wait when retrying */
278#define FAULT_FLAG_KILLABLE 0x10 /* The fault task is in SIGKILL killable region */
279#define FAULT_FLAG_TRIED 0x20 /* Second try */
280#define FAULT_FLAG_USER 0x40 /* The fault originated in userspace */
1b2ee126 281#define FAULT_FLAG_REMOTE 0x80 /* faulting for non current tsk/mm */
d61172b4 282#define FAULT_FLAG_INSTRUCTION 0x100 /* The fault was during an instruction fetch */
d0217ac0 283
54cb8821 284/*
d0217ac0 285 * vm_fault is filled by the the pagefault handler and passed to the vma's
83c54070
NP
286 * ->fault function. The vma's ->fault is responsible for returning a bitmask
287 * of VM_FAULT_xxx flags that give details about how the fault was handled.
54cb8821 288 *
c20cd45e
MH
289 * MM layer fills up gfp_mask for page allocations but fault handler might
290 * alter it if its implementation requires a different allocation context.
291 *
9b4bdd2f 292 * pgoff should be used in favour of virtual_address, if possible.
54cb8821 293 */
d0217ac0
NP
294struct vm_fault {
295 unsigned int flags; /* FAULT_FLAG_xxx flags */
c20cd45e 296 gfp_t gfp_mask; /* gfp mask to be used for allocations */
d0217ac0
NP
297 pgoff_t pgoff; /* Logical page offset based on vma */
298 void __user *virtual_address; /* Faulting virtual address */
299
2e4cdab0 300 struct page *cow_page; /* Handler may choose to COW */
d0217ac0 301 struct page *page; /* ->fault handlers should return a
83c54070 302 * page here, unless VM_FAULT_NOPAGE
d0217ac0 303 * is set (which is also implied by
83c54070 304 * VM_FAULT_ERROR).
d0217ac0 305 */
bc2466e4
JK
306 void *entry; /* ->fault handler can alternatively
307 * return locked DAX entry. In that
308 * case handler should return
309 * VM_FAULT_DAX_LOCKED and fill in
310 * entry here.
311 */
8c6e50b0
KS
312 /* for ->map_pages() only */
313 pgoff_t max_pgoff; /* map pages for offset from pgoff till
314 * max_pgoff inclusive */
315 pte_t *pte; /* pte entry associated with ->pgoff */
54cb8821 316};
1da177e4
LT
317
318/*
319 * These are the virtual MM functions - opening of an area, closing and
320 * unmapping it (needed to keep files on disk up-to-date etc), pointer
321 * to the functions called when a no-page or a wp-page exception occurs.
322 */
323struct vm_operations_struct {
324 void (*open)(struct vm_area_struct * area);
325 void (*close)(struct vm_area_struct * area);
5477e70a 326 int (*mremap)(struct vm_area_struct * area);
d0217ac0 327 int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
b96375f7
MW
328 int (*pmd_fault)(struct vm_area_struct *, unsigned long address,
329 pmd_t *, unsigned int flags);
8c6e50b0 330 void (*map_pages)(struct vm_area_struct *vma, struct vm_fault *vmf);
9637a5ef
DH
331
332 /* notification that a previously read-only page is about to become
333 * writable, if an error is returned it will cause a SIGBUS */
c2ec175c 334 int (*page_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
28b2ee20 335
dd906184
BH
336 /* same as page_mkwrite when using VM_PFNMAP|VM_MIXEDMAP */
337 int (*pfn_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
338
28b2ee20
RR
339 /* called by access_process_vm when get_user_pages() fails, typically
340 * for use by special VMAs that can switch between memory and hardware
341 */
342 int (*access)(struct vm_area_struct *vma, unsigned long addr,
343 void *buf, int len, int write);
78d683e8
AL
344
345 /* Called by the /proc/PID/maps code to ask the vma whether it
346 * has a special name. Returning non-NULL will also cause this
347 * vma to be dumped unconditionally. */
348 const char *(*name)(struct vm_area_struct *vma);
349
1da177e4 350#ifdef CONFIG_NUMA
a6020ed7
LS
351 /*
352 * set_policy() op must add a reference to any non-NULL @new mempolicy
353 * to hold the policy upon return. Caller should pass NULL @new to
354 * remove a policy and fall back to surrounding context--i.e. do not
355 * install a MPOL_DEFAULT policy, nor the task or system default
356 * mempolicy.
357 */
1da177e4 358 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
a6020ed7
LS
359
360 /*
361 * get_policy() op must add reference [mpol_get()] to any policy at
362 * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure
363 * in mm/mempolicy.c will do this automatically.
364 * get_policy() must NOT add a ref if the policy at (vma,addr) is not
365 * marked as MPOL_SHARED. vma policies are protected by the mmap_sem.
366 * If no [shared/vma] mempolicy exists at the addr, get_policy() op
367 * must return NULL--i.e., do not "fallback" to task or system default
368 * policy.
369 */
1da177e4
LT
370 struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
371 unsigned long addr);
372#endif
667a0a06
DV
373 /*
374 * Called by vm_normal_page() for special PTEs to find the
375 * page for @addr. This is useful if the default behavior
376 * (using pte_page()) would not find the correct page.
377 */
378 struct page *(*find_special_page)(struct vm_area_struct *vma,
379 unsigned long addr);
1da177e4
LT
380};
381
382struct mmu_gather;
383struct inode;
384
349aef0b
AM
385#define page_private(page) ((page)->private)
386#define set_page_private(page, v) ((page)->private = (v))
4c21e2f2 387
5c7fb56e
DW
388#if !defined(__HAVE_ARCH_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE)
389static inline int pmd_devmap(pmd_t pmd)
390{
391 return 0;
392}
393#endif
394
1da177e4
LT
395/*
396 * FIXME: take this include out, include page-flags.h in
397 * files which need it (119 of them)
398 */
399#include <linux/page-flags.h>
71e3aac0 400#include <linux/huge_mm.h>
1da177e4
LT
401
402/*
403 * Methods to modify the page usage count.
404 *
405 * What counts for a page usage:
406 * - cache mapping (page->mapping)
407 * - private data (page->private)
408 * - page mapped in a task's page tables, each mapping
409 * is counted separately
410 *
411 * Also, many kernel routines increase the page count before a critical
412 * routine so they can be sure the page doesn't go away from under them.
1da177e4
LT
413 */
414
415/*
da6052f7 416 * Drop a ref, return true if the refcount fell to zero (the page has no users)
1da177e4 417 */
7c8ee9a8
NP
418static inline int put_page_testzero(struct page *page)
419{
fe896d18
JK
420 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
421 return page_ref_dec_and_test(page);
7c8ee9a8 422}
1da177e4
LT
423
424/*
7c8ee9a8
NP
425 * Try to grab a ref unless the page has a refcount of zero, return false if
426 * that is the case.
8e0861fa
AK
427 * This can be called when MMU is off so it must not access
428 * any of the virtual mappings.
1da177e4 429 */
7c8ee9a8
NP
430static inline int get_page_unless_zero(struct page *page)
431{
fe896d18 432 return page_ref_add_unless(page, 1, 0);
7c8ee9a8 433}
1da177e4 434
53df8fdc 435extern int page_is_ram(unsigned long pfn);
124fe20d
DW
436
437enum {
438 REGION_INTERSECTS,
439 REGION_DISJOINT,
440 REGION_MIXED,
441};
442
1c29f25b
TK
443int region_intersects(resource_size_t offset, size_t size, unsigned long flags,
444 unsigned long desc);
53df8fdc 445
48667e7a 446/* Support for virtually mapped pages */
b3bdda02
CL
447struct page *vmalloc_to_page(const void *addr);
448unsigned long vmalloc_to_pfn(const void *addr);
48667e7a 449
0738c4bb
PM
450/*
451 * Determine if an address is within the vmalloc range
452 *
453 * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there
454 * is no special casing required.
455 */
bb00a789 456static inline bool is_vmalloc_addr(const void *x)
9e2779fa 457{
0738c4bb 458#ifdef CONFIG_MMU
9e2779fa
CL
459 unsigned long addr = (unsigned long)x;
460
461 return addr >= VMALLOC_START && addr < VMALLOC_END;
0738c4bb 462#else
bb00a789 463 return false;
8ca3ed87 464#endif
0738c4bb 465}
81ac3ad9
KH
466#ifdef CONFIG_MMU
467extern int is_vmalloc_or_module_addr(const void *x);
468#else
934831d0 469static inline int is_vmalloc_or_module_addr(const void *x)
81ac3ad9
KH
470{
471 return 0;
472}
473#endif
9e2779fa 474
39f1f78d
AV
475extern void kvfree(const void *addr);
476
53f9263b
KS
477static inline atomic_t *compound_mapcount_ptr(struct page *page)
478{
479 return &page[1].compound_mapcount;
480}
481
482static inline int compound_mapcount(struct page *page)
483{
5f527c2b 484 VM_BUG_ON_PAGE(!PageCompound(page), page);
53f9263b
KS
485 page = compound_head(page);
486 return atomic_read(compound_mapcount_ptr(page)) + 1;
487}
488
70b50f94
AA
489/*
490 * The atomic page->_mapcount, starts from -1: so that transitions
491 * both from it and to it can be tracked, using atomic_inc_and_test
492 * and atomic_add_negative(-1).
493 */
22b751c3 494static inline void page_mapcount_reset(struct page *page)
70b50f94
AA
495{
496 atomic_set(&(page)->_mapcount, -1);
497}
498
b20ce5e0
KS
499int __page_mapcount(struct page *page);
500
70b50f94
AA
501static inline int page_mapcount(struct page *page)
502{
1d148e21 503 VM_BUG_ON_PAGE(PageSlab(page), page);
53f9263b 504
b20ce5e0
KS
505 if (unlikely(PageCompound(page)))
506 return __page_mapcount(page);
507 return atomic_read(&page->_mapcount) + 1;
508}
509
510#ifdef CONFIG_TRANSPARENT_HUGEPAGE
511int total_mapcount(struct page *page);
6d0a07ed 512int page_trans_huge_mapcount(struct page *page, int *total_mapcount);
b20ce5e0
KS
513#else
514static inline int total_mapcount(struct page *page)
515{
516 return page_mapcount(page);
70b50f94 517}
6d0a07ed
AA
518static inline int page_trans_huge_mapcount(struct page *page,
519 int *total_mapcount)
520{
521 int mapcount = page_mapcount(page);
522 if (total_mapcount)
523 *total_mapcount = mapcount;
524 return mapcount;
525}
b20ce5e0 526#endif
70b50f94 527
b49af68f
CL
528static inline struct page *virt_to_head_page(const void *x)
529{
530 struct page *page = virt_to_page(x);
ccaafd7f 531
1d798ca3 532 return compound_head(page);
b49af68f
CL
533}
534
ddc58f27
KS
535void __put_page(struct page *page);
536
1d7ea732 537void put_pages_list(struct list_head *pages);
1da177e4 538
8dfcc9ba 539void split_page(struct page *page, unsigned int order);
748446bb 540int split_free_page(struct page *page);
8dfcc9ba 541
33f2ef89
AW
542/*
543 * Compound pages have a destructor function. Provide a
544 * prototype for that function and accessor functions.
f1e61557 545 * These are _only_ valid on the head of a compound page.
33f2ef89 546 */
f1e61557
KS
547typedef void compound_page_dtor(struct page *);
548
549/* Keep the enum in sync with compound_page_dtors array in mm/page_alloc.c */
550enum compound_dtor_id {
551 NULL_COMPOUND_DTOR,
552 COMPOUND_PAGE_DTOR,
553#ifdef CONFIG_HUGETLB_PAGE
554 HUGETLB_PAGE_DTOR,
9a982250
KS
555#endif
556#ifdef CONFIG_TRANSPARENT_HUGEPAGE
557 TRANSHUGE_PAGE_DTOR,
f1e61557
KS
558#endif
559 NR_COMPOUND_DTORS,
560};
561extern compound_page_dtor * const compound_page_dtors[];
33f2ef89
AW
562
563static inline void set_compound_page_dtor(struct page *page,
f1e61557 564 enum compound_dtor_id compound_dtor)
33f2ef89 565{
f1e61557
KS
566 VM_BUG_ON_PAGE(compound_dtor >= NR_COMPOUND_DTORS, page);
567 page[1].compound_dtor = compound_dtor;
33f2ef89
AW
568}
569
570static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
571{
f1e61557
KS
572 VM_BUG_ON_PAGE(page[1].compound_dtor >= NR_COMPOUND_DTORS, page);
573 return compound_page_dtors[page[1].compound_dtor];
33f2ef89
AW
574}
575
d00181b9 576static inline unsigned int compound_order(struct page *page)
d85f3385 577{
6d777953 578 if (!PageHead(page))
d85f3385 579 return 0;
e4b294c2 580 return page[1].compound_order;
d85f3385
CL
581}
582
f1e61557 583static inline void set_compound_order(struct page *page, unsigned int order)
d85f3385 584{
e4b294c2 585 page[1].compound_order = order;
d85f3385
CL
586}
587
9a982250
KS
588void free_compound_page(struct page *page);
589
3dece370 590#ifdef CONFIG_MMU
14fd403f
AA
591/*
592 * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when
593 * servicing faults for write access. In the normal case, do always want
594 * pte_mkwrite. But get_user_pages can cause write faults for mappings
595 * that do not have writing enabled, when used by access_process_vm.
596 */
597static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
598{
599 if (likely(vma->vm_flags & VM_WRITE))
600 pte = pte_mkwrite(pte);
601 return pte;
602}
8c6e50b0
KS
603
604void do_set_pte(struct vm_area_struct *vma, unsigned long address,
315d09bf 605 struct page *page, pte_t *pte, bool write, bool anon);
3dece370 606#endif
14fd403f 607
1da177e4
LT
608/*
609 * Multiple processes may "see" the same page. E.g. for untouched
610 * mappings of /dev/null, all processes see the same page full of
611 * zeroes, and text pages of executables and shared libraries have
612 * only one copy in memory, at most, normally.
613 *
614 * For the non-reserved pages, page_count(page) denotes a reference count.
7e871b6c
PBG
615 * page_count() == 0 means the page is free. page->lru is then used for
616 * freelist management in the buddy allocator.
da6052f7 617 * page_count() > 0 means the page has been allocated.
1da177e4 618 *
da6052f7
NP
619 * Pages are allocated by the slab allocator in order to provide memory
620 * to kmalloc and kmem_cache_alloc. In this case, the management of the
621 * page, and the fields in 'struct page' are the responsibility of mm/slab.c
622 * unless a particular usage is carefully commented. (the responsibility of
623 * freeing the kmalloc memory is the caller's, of course).
1da177e4 624 *
da6052f7
NP
625 * A page may be used by anyone else who does a __get_free_page().
626 * In this case, page_count still tracks the references, and should only
627 * be used through the normal accessor functions. The top bits of page->flags
628 * and page->virtual store page management information, but all other fields
629 * are unused and could be used privately, carefully. The management of this
630 * page is the responsibility of the one who allocated it, and those who have
631 * subsequently been given references to it.
632 *
633 * The other pages (we may call them "pagecache pages") are completely
1da177e4
LT
634 * managed by the Linux memory manager: I/O, buffers, swapping etc.
635 * The following discussion applies only to them.
636 *
da6052f7
NP
637 * A pagecache page contains an opaque `private' member, which belongs to the
638 * page's address_space. Usually, this is the address of a circular list of
639 * the page's disk buffers. PG_private must be set to tell the VM to call
640 * into the filesystem to release these pages.
1da177e4 641 *
da6052f7
NP
642 * A page may belong to an inode's memory mapping. In this case, page->mapping
643 * is the pointer to the inode, and page->index is the file offset of the page,
ea1754a0 644 * in units of PAGE_SIZE.
1da177e4 645 *
da6052f7
NP
646 * If pagecache pages are not associated with an inode, they are said to be
647 * anonymous pages. These may become associated with the swapcache, and in that
648 * case PG_swapcache is set, and page->private is an offset into the swapcache.
1da177e4 649 *
da6052f7
NP
650 * In either case (swapcache or inode backed), the pagecache itself holds one
651 * reference to the page. Setting PG_private should also increment the
652 * refcount. The each user mapping also has a reference to the page.
1da177e4 653 *
da6052f7
NP
654 * The pagecache pages are stored in a per-mapping radix tree, which is
655 * rooted at mapping->page_tree, and indexed by offset.
656 * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space
657 * lists, we instead now tag pages as dirty/writeback in the radix tree.
1da177e4 658 *
da6052f7 659 * All pagecache pages may be subject to I/O:
1da177e4
LT
660 * - inode pages may need to be read from disk,
661 * - inode pages which have been modified and are MAP_SHARED may need
da6052f7
NP
662 * to be written back to the inode on disk,
663 * - anonymous pages (including MAP_PRIVATE file mappings) which have been
664 * modified may need to be swapped out to swap space and (later) to be read
665 * back into memory.
1da177e4
LT
666 */
667
668/*
669 * The zone field is never updated after free_area_init_core()
670 * sets it, so none of the operations on it need to be atomic.
1da177e4 671 */
348f8b6c 672
90572890 673/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */
07808b74 674#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
d41dee36
AW
675#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
676#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
90572890 677#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH)
d41dee36 678
348f8b6c 679/*
25985edc 680 * Define the bit shifts to access each section. For non-existent
348f8b6c
DH
681 * sections we define the shift as 0; that plus a 0 mask ensures
682 * the compiler will optimise away reference to them.
683 */
d41dee36
AW
684#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
685#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
686#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
90572890 687#define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))
348f8b6c 688
bce54bbf
WD
689/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */
690#ifdef NODE_NOT_IN_PAGE_FLAGS
89689ae7 691#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)
bd8029b6
AW
692#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \
693 SECTIONS_PGOFF : ZONES_PGOFF)
d41dee36 694#else
89689ae7 695#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)
bd8029b6
AW
696#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \
697 NODES_PGOFF : ZONES_PGOFF)
89689ae7
CL
698#endif
699
bd8029b6 700#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
348f8b6c 701
9223b419
CL
702#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
703#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
348f8b6c
DH
704#endif
705
d41dee36
AW
706#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
707#define NODES_MASK ((1UL << NODES_WIDTH) - 1)
708#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
834a964a 709#define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_SHIFT) - 1)
89689ae7 710#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
348f8b6c 711
33dd4e0e 712static inline enum zone_type page_zonenum(const struct page *page)
1da177e4 713{
348f8b6c 714 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
1da177e4 715}
1da177e4 716
260ae3f7 717#ifdef CONFIG_ZONE_DEVICE
3565fce3
DW
718void get_zone_device_page(struct page *page);
719void put_zone_device_page(struct page *page);
260ae3f7
DW
720static inline bool is_zone_device_page(const struct page *page)
721{
722 return page_zonenum(page) == ZONE_DEVICE;
723}
724#else
3565fce3
DW
725static inline void get_zone_device_page(struct page *page)
726{
727}
728static inline void put_zone_device_page(struct page *page)
729{
730}
260ae3f7
DW
731static inline bool is_zone_device_page(const struct page *page)
732{
733 return false;
734}
735#endif
736
3565fce3
DW
737static inline void get_page(struct page *page)
738{
739 page = compound_head(page);
740 /*
741 * Getting a normal page or the head of a compound page
0139aa7b 742 * requires to already have an elevated page->_refcount.
3565fce3 743 */
fe896d18
JK
744 VM_BUG_ON_PAGE(page_ref_count(page) <= 0, page);
745 page_ref_inc(page);
3565fce3
DW
746
747 if (unlikely(is_zone_device_page(page)))
748 get_zone_device_page(page);
749}
750
751static inline void put_page(struct page *page)
752{
753 page = compound_head(page);
754
755 if (put_page_testzero(page))
756 __put_page(page);
757
758 if (unlikely(is_zone_device_page(page)))
759 put_zone_device_page(page);
760}
761
9127ab4f
CS
762#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
763#define SECTION_IN_PAGE_FLAGS
764#endif
765
89689ae7 766/*
7a8010cd
VB
767 * The identification function is mainly used by the buddy allocator for
768 * determining if two pages could be buddies. We are not really identifying
769 * the zone since we could be using the section number id if we do not have
770 * node id available in page flags.
771 * We only guarantee that it will return the same value for two combinable
772 * pages in a zone.
89689ae7 773 */
cb2b95e1
AW
774static inline int page_zone_id(struct page *page)
775{
89689ae7 776 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
348f8b6c
DH
777}
778
25ba77c1 779static inline int zone_to_nid(struct zone *zone)
89fa3024 780{
d5f541ed
CL
781#ifdef CONFIG_NUMA
782 return zone->node;
783#else
784 return 0;
785#endif
89fa3024
CL
786}
787
89689ae7 788#ifdef NODE_NOT_IN_PAGE_FLAGS
33dd4e0e 789extern int page_to_nid(const struct page *page);
89689ae7 790#else
33dd4e0e 791static inline int page_to_nid(const struct page *page)
d41dee36 792{
89689ae7 793 return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
d41dee36 794}
89689ae7
CL
795#endif
796
57e0a030 797#ifdef CONFIG_NUMA_BALANCING
90572890 798static inline int cpu_pid_to_cpupid(int cpu, int pid)
57e0a030 799{
90572890 800 return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
57e0a030
MG
801}
802
90572890 803static inline int cpupid_to_pid(int cpupid)
57e0a030 804{
90572890 805 return cpupid & LAST__PID_MASK;
57e0a030 806}
b795854b 807
90572890 808static inline int cpupid_to_cpu(int cpupid)
b795854b 809{
90572890 810 return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;
b795854b
MG
811}
812
90572890 813static inline int cpupid_to_nid(int cpupid)
b795854b 814{
90572890 815 return cpu_to_node(cpupid_to_cpu(cpupid));
b795854b
MG
816}
817
90572890 818static inline bool cpupid_pid_unset(int cpupid)
57e0a030 819{
90572890 820 return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);
b795854b
MG
821}
822
90572890 823static inline bool cpupid_cpu_unset(int cpupid)
b795854b 824{
90572890 825 return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);
b795854b
MG
826}
827
8c8a743c
PZ
828static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid)
829{
830 return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);
831}
832
833#define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid)
90572890
PZ
834#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
835static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
b795854b 836{
1ae71d03 837 return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK);
b795854b 838}
90572890
PZ
839
840static inline int page_cpupid_last(struct page *page)
841{
842 return page->_last_cpupid;
843}
844static inline void page_cpupid_reset_last(struct page *page)
b795854b 845{
1ae71d03 846 page->_last_cpupid = -1 & LAST_CPUPID_MASK;
57e0a030
MG
847}
848#else
90572890 849static inline int page_cpupid_last(struct page *page)
75980e97 850{
90572890 851 return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
75980e97
PZ
852}
853
90572890 854extern int page_cpupid_xchg_last(struct page *page, int cpupid);
75980e97 855
90572890 856static inline void page_cpupid_reset_last(struct page *page)
75980e97 857{
09940a4f 858 page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;
75980e97 859}
90572890
PZ
860#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
861#else /* !CONFIG_NUMA_BALANCING */
862static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
57e0a030 863{
90572890 864 return page_to_nid(page); /* XXX */
57e0a030
MG
865}
866
90572890 867static inline int page_cpupid_last(struct page *page)
57e0a030 868{
90572890 869 return page_to_nid(page); /* XXX */
57e0a030
MG
870}
871
90572890 872static inline int cpupid_to_nid(int cpupid)
b795854b
MG
873{
874 return -1;
875}
876
90572890 877static inline int cpupid_to_pid(int cpupid)
b795854b
MG
878{
879 return -1;
880}
881
90572890 882static inline int cpupid_to_cpu(int cpupid)
b795854b
MG
883{
884 return -1;
885}
886
90572890
PZ
887static inline int cpu_pid_to_cpupid(int nid, int pid)
888{
889 return -1;
890}
891
892static inline bool cpupid_pid_unset(int cpupid)
b795854b
MG
893{
894 return 1;
895}
896
90572890 897static inline void page_cpupid_reset_last(struct page *page)
57e0a030
MG
898{
899}
8c8a743c
PZ
900
901static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
902{
903 return false;
904}
90572890 905#endif /* CONFIG_NUMA_BALANCING */
57e0a030 906
33dd4e0e 907static inline struct zone *page_zone(const struct page *page)
89689ae7
CL
908{
909 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
910}
911
9127ab4f 912#ifdef SECTION_IN_PAGE_FLAGS
bf4e8902
DK
913static inline void set_page_section(struct page *page, unsigned long section)
914{
915 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
916 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
917}
918
aa462abe 919static inline unsigned long page_to_section(const struct page *page)
d41dee36
AW
920{
921 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
922}
308c05e3 923#endif
d41dee36 924
2f1b6248 925static inline void set_page_zone(struct page *page, enum zone_type zone)
348f8b6c
DH
926{
927 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
928 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
929}
2f1b6248 930
348f8b6c
DH
931static inline void set_page_node(struct page *page, unsigned long node)
932{
933 page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
934 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
1da177e4 935}
89689ae7 936
2f1b6248 937static inline void set_page_links(struct page *page, enum zone_type zone,
d41dee36 938 unsigned long node, unsigned long pfn)
1da177e4 939{
348f8b6c
DH
940 set_page_zone(page, zone);
941 set_page_node(page, node);
9127ab4f 942#ifdef SECTION_IN_PAGE_FLAGS
d41dee36 943 set_page_section(page, pfn_to_section_nr(pfn));
bf4e8902 944#endif
1da177e4
LT
945}
946
0610c25d
GT
947#ifdef CONFIG_MEMCG
948static inline struct mem_cgroup *page_memcg(struct page *page)
949{
950 return page->mem_cgroup;
951}
0610c25d
GT
952#else
953static inline struct mem_cgroup *page_memcg(struct page *page)
954{
955 return NULL;
956}
0610c25d
GT
957#endif
958
f6ac2354
CL
959/*
960 * Some inline functions in vmstat.h depend on page_zone()
961 */
962#include <linux/vmstat.h>
963
33dd4e0e 964static __always_inline void *lowmem_page_address(const struct page *page)
1da177e4 965{
1dff8083 966 return page_to_virt(page);
1da177e4
LT
967}
968
969#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
970#define HASHED_PAGE_VIRTUAL
971#endif
972
973#if defined(WANT_PAGE_VIRTUAL)
f92f455f
GU
974static inline void *page_address(const struct page *page)
975{
976 return page->virtual;
977}
978static inline void set_page_address(struct page *page, void *address)
979{
980 page->virtual = address;
981}
1da177e4
LT
982#define page_address_init() do { } while(0)
983#endif
984
985#if defined(HASHED_PAGE_VIRTUAL)
f9918794 986void *page_address(const struct page *page);
1da177e4
LT
987void set_page_address(struct page *page, void *virtual);
988void page_address_init(void);
989#endif
990
991#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
992#define page_address(page) lowmem_page_address(page)
993#define set_page_address(page, address) do { } while(0)
994#define page_address_init() do { } while(0)
995#endif
996
e39155ea
KS
997extern void *page_rmapping(struct page *page);
998extern struct anon_vma *page_anon_vma(struct page *page);
9800339b 999extern struct address_space *page_mapping(struct page *page);
1da177e4 1000
f981c595
MG
1001extern struct address_space *__page_file_mapping(struct page *);
1002
1003static inline
1004struct address_space *page_file_mapping(struct page *page)
1005{
1006 if (unlikely(PageSwapCache(page)))
1007 return __page_file_mapping(page);
1008
1009 return page->mapping;
1010}
1011
1da177e4
LT
1012/*
1013 * Return the pagecache index of the passed page. Regular pagecache pages
1014 * use ->index whereas swapcache pages use ->private
1015 */
1016static inline pgoff_t page_index(struct page *page)
1017{
1018 if (unlikely(PageSwapCache(page)))
4c21e2f2 1019 return page_private(page);
1da177e4
LT
1020 return page->index;
1021}
1022
f981c595
MG
1023extern pgoff_t __page_file_index(struct page *page);
1024
1025/*
1026 * Return the file index of the page. Regular pagecache pages use ->index
1027 * whereas swapcache pages use swp_offset(->private)
1028 */
1029static inline pgoff_t page_file_index(struct page *page)
1030{
1031 if (unlikely(PageSwapCache(page)))
1032 return __page_file_index(page);
1033
1034 return page->index;
1035}
1036
1aa8aea5 1037bool page_mapped(struct page *page);
bda807d4 1038struct address_space *page_mapping(struct page *page);
1da177e4 1039
2f064f34
MH
1040/*
1041 * Return true only if the page has been allocated with
1042 * ALLOC_NO_WATERMARKS and the low watermark was not
1043 * met implying that the system is under some pressure.
1044 */
1045static inline bool page_is_pfmemalloc(struct page *page)
1046{
1047 /*
1048 * Page index cannot be this large so this must be
1049 * a pfmemalloc page.
1050 */
1051 return page->index == -1UL;
1052}
1053
1054/*
1055 * Only to be called by the page allocator on a freshly allocated
1056 * page.
1057 */
1058static inline void set_page_pfmemalloc(struct page *page)
1059{
1060 page->index = -1UL;
1061}
1062
1063static inline void clear_page_pfmemalloc(struct page *page)
1064{
1065 page->index = 0;
1066}
1067
1da177e4
LT
1068/*
1069 * Different kinds of faults, as returned by handle_mm_fault().
1070 * Used to decide whether a process gets delivered SIGBUS or
1071 * just gets major/minor fault counters bumped up.
1072 */
d0217ac0 1073
83c54070
NP
1074#define VM_FAULT_OOM 0x0001
1075#define VM_FAULT_SIGBUS 0x0002
1076#define VM_FAULT_MAJOR 0x0004
1077#define VM_FAULT_WRITE 0x0008 /* Special case for get_user_pages */
aa50d3a7
AK
1078#define VM_FAULT_HWPOISON 0x0010 /* Hit poisoned small page */
1079#define VM_FAULT_HWPOISON_LARGE 0x0020 /* Hit poisoned large page. Index encoded in upper bits */
33692f27 1080#define VM_FAULT_SIGSEGV 0x0040
f33ea7f4 1081
83c54070
NP
1082#define VM_FAULT_NOPAGE 0x0100 /* ->fault installed the pte, not return page */
1083#define VM_FAULT_LOCKED 0x0200 /* ->fault locked the returned page */
d065bd81 1084#define VM_FAULT_RETRY 0x0400 /* ->fault blocked, must retry */
c0292554 1085#define VM_FAULT_FALLBACK 0x0800 /* huge page fault failed, fall back to small */
bc2466e4 1086#define VM_FAULT_DAX_LOCKED 0x1000 /* ->fault has locked DAX entry */
1da177e4 1087
aa50d3a7
AK
1088#define VM_FAULT_HWPOISON_LARGE_MASK 0xf000 /* encodes hpage index for large hwpoison */
1089
33692f27
LT
1090#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | \
1091 VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE | \
1092 VM_FAULT_FALLBACK)
aa50d3a7
AK
1093
1094/* Encode hstate index for a hwpoisoned large page */
1095#define VM_FAULT_SET_HINDEX(x) ((x) << 12)
1096#define VM_FAULT_GET_HINDEX(x) (((x) >> 12) & 0xf)
d0217ac0 1097
1c0fe6e3
NP
1098/*
1099 * Can be called by the pagefault handler when it gets a VM_FAULT_OOM.
1100 */
1101extern void pagefault_out_of_memory(void);
1102
1da177e4
LT
1103#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
1104
ddd588b5 1105/*
7bf02ea2 1106 * Flags passed to show_mem() and show_free_areas() to suppress output in
ddd588b5
DR
1107 * various contexts.
1108 */
4b59e6c4 1109#define SHOW_MEM_FILTER_NODES (0x0001u) /* disallowed nodes */
ddd588b5 1110
7bf02ea2
DR
1111extern void show_free_areas(unsigned int flags);
1112extern bool skip_free_areas_node(unsigned int flags, int nid);
1da177e4 1113
1da177e4 1114int shmem_zero_setup(struct vm_area_struct *);
0cd6144a
JW
1115#ifdef CONFIG_SHMEM
1116bool shmem_mapping(struct address_space *mapping);
1117#else
1118static inline bool shmem_mapping(struct address_space *mapping)
1119{
1120 return false;
1121}
1122#endif
1da177e4 1123
7f43add4 1124extern bool can_do_mlock(void);
1da177e4
LT
1125extern int user_shm_lock(size_t, struct user_struct *);
1126extern void user_shm_unlock(size_t, struct user_struct *);
1127
1128/*
1129 * Parameter block passed down to zap_pte_range in exceptional cases.
1130 */
1131struct zap_details {
1da177e4
LT
1132 struct address_space *check_mapping; /* Check page->mapping if set */
1133 pgoff_t first_index; /* Lowest page->index to unmap */
1134 pgoff_t last_index; /* Highest page->index to unmap */
aac45363
MH
1135 bool ignore_dirty; /* Ignore dirty pages */
1136 bool check_swap_entries; /* Check also swap entries */
1da177e4
LT
1137};
1138
7e675137
NP
1139struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
1140 pte_t pte);
28093f9f
GS
1141struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
1142 pmd_t pmd);
7e675137 1143
c627f9cc
JS
1144int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
1145 unsigned long size);
14f5ff5d 1146void zap_page_range(struct vm_area_struct *vma, unsigned long address,
1da177e4 1147 unsigned long size, struct zap_details *);
4f74d2c8
LT
1148void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
1149 unsigned long start, unsigned long end);
e6473092
MM
1150
1151/**
1152 * mm_walk - callbacks for walk_page_range
e6473092 1153 * @pmd_entry: if set, called for each non-empty PMD (3rd-level) entry
03319327
DH
1154 * this handler is required to be able to handle
1155 * pmd_trans_huge() pmds. They may simply choose to
1156 * split_huge_page() instead of handling it explicitly.
e6473092
MM
1157 * @pte_entry: if set, called for each non-empty PTE (4th-level) entry
1158 * @pte_hole: if set, called for each hole at all levels
5dc37642 1159 * @hugetlb_entry: if set, called for each hugetlb entry
fafaa426
NH
1160 * @test_walk: caller specific callback function to determine whether
1161 * we walk over the current vma or not. A positive returned
1162 * value means "do page table walk over the current vma,"
1163 * and a negative one means "abort current page table walk
1164 * right now." 0 means "skip the current vma."
1165 * @mm: mm_struct representing the target process of page table walk
1166 * @vma: vma currently walked (NULL if walking outside vmas)
1167 * @private: private data for callbacks' usage
e6473092 1168 *
fafaa426 1169 * (see the comment on walk_page_range() for more details)
e6473092
MM
1170 */
1171struct mm_walk {
0f157a5b
AM
1172 int (*pmd_entry)(pmd_t *pmd, unsigned long addr,
1173 unsigned long next, struct mm_walk *walk);
1174 int (*pte_entry)(pte_t *pte, unsigned long addr,
1175 unsigned long next, struct mm_walk *walk);
1176 int (*pte_hole)(unsigned long addr, unsigned long next,
1177 struct mm_walk *walk);
1178 int (*hugetlb_entry)(pte_t *pte, unsigned long hmask,
1179 unsigned long addr, unsigned long next,
1180 struct mm_walk *walk);
fafaa426
NH
1181 int (*test_walk)(unsigned long addr, unsigned long next,
1182 struct mm_walk *walk);
2165009b 1183 struct mm_struct *mm;
fafaa426 1184 struct vm_area_struct *vma;
2165009b 1185 void *private;
e6473092
MM
1186};
1187
2165009b
DH
1188int walk_page_range(unsigned long addr, unsigned long end,
1189 struct mm_walk *walk);
900fc5f1 1190int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk);
42b77728 1191void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
3bf5ee95 1192 unsigned long end, unsigned long floor, unsigned long ceiling);
1da177e4
LT
1193int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
1194 struct vm_area_struct *vma);
1da177e4
LT
1195void unmap_mapping_range(struct address_space *mapping,
1196 loff_t const holebegin, loff_t const holelen, int even_cows);
3b6748e2
JW
1197int follow_pfn(struct vm_area_struct *vma, unsigned long address,
1198 unsigned long *pfn);
d87fe660 1199int follow_phys(struct vm_area_struct *vma, unsigned long address,
1200 unsigned int flags, unsigned long *prot, resource_size_t *phys);
28b2ee20
RR
1201int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
1202 void *buf, int len, int write);
1da177e4
LT
1203
1204static inline void unmap_shared_mapping_range(struct address_space *mapping,
1205 loff_t const holebegin, loff_t const holelen)
1206{
1207 unmap_mapping_range(mapping, holebegin, holelen, 0);
1208}
1209
7caef267 1210extern void truncate_pagecache(struct inode *inode, loff_t new);
2c27c65e 1211extern void truncate_setsize(struct inode *inode, loff_t newsize);
90a80202 1212void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
623e3db9 1213void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
750b4987 1214int truncate_inode_page(struct address_space *mapping, struct page *page);
25718736 1215int generic_error_remove_page(struct address_space *mapping, struct page *page);
83f78668
WF
1216int invalidate_inode_page(struct page *page);
1217
7ee1dd3f 1218#ifdef CONFIG_MMU
83c54070 1219extern int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
d06063cc 1220 unsigned long address, unsigned int flags);
5c723ba5 1221extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
4a9e1cda
DD
1222 unsigned long address, unsigned int fault_flags,
1223 bool *unlocked);
7ee1dd3f
DH
1224#else
1225static inline int handle_mm_fault(struct mm_struct *mm,
1226 struct vm_area_struct *vma, unsigned long address,
d06063cc 1227 unsigned int flags)
7ee1dd3f
DH
1228{
1229 /* should never happen if there's no MMU */
1230 BUG();
1231 return VM_FAULT_SIGBUS;
1232}
5c723ba5
PZ
1233static inline int fixup_user_fault(struct task_struct *tsk,
1234 struct mm_struct *mm, unsigned long address,
4a9e1cda 1235 unsigned int fault_flags, bool *unlocked)
5c723ba5
PZ
1236{
1237 /* should never happen if there's no MMU */
1238 BUG();
1239 return -EFAULT;
1240}
7ee1dd3f 1241#endif
f33ea7f4 1242
1da177e4 1243extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
5ddd36b9
SW
1244extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
1245 void *buf, int len, int write);
1da177e4 1246
28a35716
ML
1247long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
1248 unsigned long start, unsigned long nr_pages,
1249 unsigned int foll_flags, struct page **pages,
1250 struct vm_area_struct **vmas, int *nonblocking);
1e987790
DH
1251long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
1252 unsigned long start, unsigned long nr_pages,
1253 int write, int force, struct page **pages,
1254 struct vm_area_struct **vmas);
c12d2da5 1255long get_user_pages(unsigned long start, unsigned long nr_pages,
cde70140
DH
1256 int write, int force, struct page **pages,
1257 struct vm_area_struct **vmas);
c12d2da5 1258long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
cde70140 1259 int write, int force, struct page **pages, int *locked);
0fd71a56
AA
1260long __get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
1261 unsigned long start, unsigned long nr_pages,
1262 int write, int force, struct page **pages,
1263 unsigned int gup_flags);
c12d2da5 1264long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
f0818f47 1265 int write, int force, struct page **pages);
d2bf6be8
NP
1266int get_user_pages_fast(unsigned long start, int nr_pages, int write,
1267 struct page **pages);
8025e5dd
JK
1268
1269/* Container for pinned pfns / pages */
1270struct frame_vector {
1271 unsigned int nr_allocated; /* Number of frames we have space for */
1272 unsigned int nr_frames; /* Number of frames stored in ptrs array */
1273 bool got_ref; /* Did we pin pages by getting page ref? */
1274 bool is_pfns; /* Does array contain pages or pfns? */
1275 void *ptrs[0]; /* Array of pinned pfns / pages. Use
1276 * pfns_vector_pages() or pfns_vector_pfns()
1277 * for access */
1278};
1279
1280struct frame_vector *frame_vector_create(unsigned int nr_frames);
1281void frame_vector_destroy(struct frame_vector *vec);
1282int get_vaddr_frames(unsigned long start, unsigned int nr_pfns,
1283 bool write, bool force, struct frame_vector *vec);
1284void put_vaddr_frames(struct frame_vector *vec);
1285int frame_vector_to_pages(struct frame_vector *vec);
1286void frame_vector_to_pfns(struct frame_vector *vec);
1287
1288static inline unsigned int frame_vector_count(struct frame_vector *vec)
1289{
1290 return vec->nr_frames;
1291}
1292
1293static inline struct page **frame_vector_pages(struct frame_vector *vec)
1294{
1295 if (vec->is_pfns) {
1296 int err = frame_vector_to_pages(vec);
1297
1298 if (err)
1299 return ERR_PTR(err);
1300 }
1301 return (struct page **)(vec->ptrs);
1302}
1303
1304static inline unsigned long *frame_vector_pfns(struct frame_vector *vec)
1305{
1306 if (!vec->is_pfns)
1307 frame_vector_to_pfns(vec);
1308 return (unsigned long *)(vec->ptrs);
1309}
1310
18022c5d
MG
1311struct kvec;
1312int get_kernel_pages(const struct kvec *iov, int nr_pages, int write,
1313 struct page **pages);
1314int get_kernel_page(unsigned long start, int write, struct page **pages);
f3e8fccd 1315struct page *get_dump_page(unsigned long addr);
1da177e4 1316
cf9a2ae8 1317extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
d47992f8
LC
1318extern void do_invalidatepage(struct page *page, unsigned int offset,
1319 unsigned int length);
cf9a2ae8 1320
1da177e4 1321int __set_page_dirty_nobuffers(struct page *page);
76719325 1322int __set_page_dirty_no_writeback(struct page *page);
1da177e4
LT
1323int redirty_page_for_writepage(struct writeback_control *wbc,
1324 struct page *page);
62cccb8c 1325void account_page_dirtied(struct page *page, struct address_space *mapping);
c4843a75 1326void account_page_cleaned(struct page *page, struct address_space *mapping,
62cccb8c 1327 struct bdi_writeback *wb);
b3c97528 1328int set_page_dirty(struct page *page);
1da177e4 1329int set_page_dirty_lock(struct page *page);
11f81bec 1330void cancel_dirty_page(struct page *page);
1da177e4 1331int clear_page_dirty_for_io(struct page *page);
b9ea2515 1332
a9090253 1333int get_cmdline(struct task_struct *task, char *buffer, int buflen);
1da177e4 1334
39aa3cb3 1335/* Is the vma a continuation of the stack vma above it? */
a09a79f6 1336static inline int vma_growsdown(struct vm_area_struct *vma, unsigned long addr)
39aa3cb3
SB
1337{
1338 return vma && (vma->vm_end == addr) && (vma->vm_flags & VM_GROWSDOWN);
1339}
1340
b5330628
ON
1341static inline bool vma_is_anonymous(struct vm_area_struct *vma)
1342{
1343 return !vma->vm_ops;
1344}
1345
a09a79f6
MP
1346static inline int stack_guard_page_start(struct vm_area_struct *vma,
1347 unsigned long addr)
1348{
1349 return (vma->vm_flags & VM_GROWSDOWN) &&
1350 (vma->vm_start == addr) &&
1351 !vma_growsdown(vma->vm_prev, addr);
1352}
1353
1354/* Is the vma a continuation of the stack vma below it? */
1355static inline int vma_growsup(struct vm_area_struct *vma, unsigned long addr)
1356{
1357 return vma && (vma->vm_start == addr) && (vma->vm_flags & VM_GROWSUP);
1358}
1359
1360static inline int stack_guard_page_end(struct vm_area_struct *vma,
1361 unsigned long addr)
1362{
1363 return (vma->vm_flags & VM_GROWSUP) &&
1364 (vma->vm_end == addr) &&
1365 !vma_growsup(vma->vm_next, addr);
1366}
1367
65376df5 1368int vma_is_stack_for_task(struct vm_area_struct *vma, struct task_struct *t);
b7643757 1369
b6a2fea3
OW
1370extern unsigned long move_page_tables(struct vm_area_struct *vma,
1371 unsigned long old_addr, struct vm_area_struct *new_vma,
38a76013
ML
1372 unsigned long new_addr, unsigned long len,
1373 bool need_rmap_locks);
7da4d641
PZ
1374extern unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
1375 unsigned long end, pgprot_t newprot,
4b10e7d5 1376 int dirty_accountable, int prot_numa);
b6a2fea3
OW
1377extern int mprotect_fixup(struct vm_area_struct *vma,
1378 struct vm_area_struct **pprev, unsigned long start,
1379 unsigned long end, unsigned long newflags);
1da177e4 1380
465a454f
PZ
1381/*
1382 * doesn't attempt to fault and will return short.
1383 */
1384int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
1385 struct page **pages);
d559db08
KH
1386/*
1387 * per-process(per-mm_struct) statistics.
1388 */
d559db08
KH
1389static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
1390{
69c97823
KK
1391 long val = atomic_long_read(&mm->rss_stat.count[member]);
1392
1393#ifdef SPLIT_RSS_COUNTING
1394 /*
1395 * counter is updated in asynchronous manner and may go to minus.
1396 * But it's never be expected number for users.
1397 */
1398 if (val < 0)
1399 val = 0;
172703b0 1400#endif
69c97823
KK
1401 return (unsigned long)val;
1402}
d559db08
KH
1403
1404static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
1405{
172703b0 1406 atomic_long_add(value, &mm->rss_stat.count[member]);
d559db08
KH
1407}
1408
1409static inline void inc_mm_counter(struct mm_struct *mm, int member)
1410{
172703b0 1411 atomic_long_inc(&mm->rss_stat.count[member]);
d559db08
KH
1412}
1413
1414static inline void dec_mm_counter(struct mm_struct *mm, int member)
1415{
172703b0 1416 atomic_long_dec(&mm->rss_stat.count[member]);
d559db08
KH
1417}
1418
eca56ff9
JM
1419/* Optimized variant when page is already known not to be PageAnon */
1420static inline int mm_counter_file(struct page *page)
1421{
1422 if (PageSwapBacked(page))
1423 return MM_SHMEMPAGES;
1424 return MM_FILEPAGES;
1425}
1426
1427static inline int mm_counter(struct page *page)
1428{
1429 if (PageAnon(page))
1430 return MM_ANONPAGES;
1431 return mm_counter_file(page);
1432}
1433
d559db08
KH
1434static inline unsigned long get_mm_rss(struct mm_struct *mm)
1435{
1436 return get_mm_counter(mm, MM_FILEPAGES) +
eca56ff9
JM
1437 get_mm_counter(mm, MM_ANONPAGES) +
1438 get_mm_counter(mm, MM_SHMEMPAGES);
d559db08
KH
1439}
1440
1441static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
1442{
1443 return max(mm->hiwater_rss, get_mm_rss(mm));
1444}
1445
1446static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm)
1447{
1448 return max(mm->hiwater_vm, mm->total_vm);
1449}
1450
1451static inline void update_hiwater_rss(struct mm_struct *mm)
1452{
1453 unsigned long _rss = get_mm_rss(mm);
1454
1455 if ((mm)->hiwater_rss < _rss)
1456 (mm)->hiwater_rss = _rss;
1457}
1458
1459static inline void update_hiwater_vm(struct mm_struct *mm)
1460{
1461 if (mm->hiwater_vm < mm->total_vm)
1462 mm->hiwater_vm = mm->total_vm;
1463}
1464
695f0559
PC
1465static inline void reset_mm_hiwater_rss(struct mm_struct *mm)
1466{
1467 mm->hiwater_rss = get_mm_rss(mm);
1468}
1469
d559db08
KH
1470static inline void setmax_mm_hiwater_rss(unsigned long *maxrss,
1471 struct mm_struct *mm)
1472{
1473 unsigned long hiwater_rss = get_mm_hiwater_rss(mm);
1474
1475 if (*maxrss < hiwater_rss)
1476 *maxrss = hiwater_rss;
1477}
1478
53bddb4e 1479#if defined(SPLIT_RSS_COUNTING)
05af2e10 1480void sync_mm_rss(struct mm_struct *mm);
53bddb4e 1481#else
05af2e10 1482static inline void sync_mm_rss(struct mm_struct *mm)
53bddb4e
KH
1483{
1484}
1485#endif
465a454f 1486
3565fce3
DW
1487#ifndef __HAVE_ARCH_PTE_DEVMAP
1488static inline int pte_devmap(pte_t pte)
1489{
1490 return 0;
1491}
1492#endif
1493
4e950f6f 1494int vma_wants_writenotify(struct vm_area_struct *vma);
d08b3851 1495
25ca1d6c
NK
1496extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
1497 spinlock_t **ptl);
1498static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
1499 spinlock_t **ptl)
1500{
1501 pte_t *ptep;
1502 __cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl));
1503 return ptep;
1504}
c9cfcddf 1505
5f22df00
NP
1506#ifdef __PAGETABLE_PUD_FOLDED
1507static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
1508 unsigned long address)
1509{
1510 return 0;
1511}
1512#else
1bb3630e 1513int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
5f22df00
NP
1514#endif
1515
2d2f5119 1516#if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU)
5f22df00
NP
1517static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
1518 unsigned long address)
1519{
1520 return 0;
1521}
dc6c9a35 1522
2d2f5119
KS
1523static inline void mm_nr_pmds_init(struct mm_struct *mm) {}
1524
dc6c9a35
KS
1525static inline unsigned long mm_nr_pmds(struct mm_struct *mm)
1526{
1527 return 0;
1528}
1529
1530static inline void mm_inc_nr_pmds(struct mm_struct *mm) {}
1531static inline void mm_dec_nr_pmds(struct mm_struct *mm) {}
1532
5f22df00 1533#else
1bb3630e 1534int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
dc6c9a35 1535
2d2f5119
KS
1536static inline void mm_nr_pmds_init(struct mm_struct *mm)
1537{
1538 atomic_long_set(&mm->nr_pmds, 0);
1539}
1540
dc6c9a35
KS
1541static inline unsigned long mm_nr_pmds(struct mm_struct *mm)
1542{
1543 return atomic_long_read(&mm->nr_pmds);
1544}
1545
1546static inline void mm_inc_nr_pmds(struct mm_struct *mm)
1547{
1548 atomic_long_inc(&mm->nr_pmds);
1549}
1550
1551static inline void mm_dec_nr_pmds(struct mm_struct *mm)
1552{
1553 atomic_long_dec(&mm->nr_pmds);
1554}
5f22df00
NP
1555#endif
1556
3ed3a4f0 1557int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address);
1bb3630e
HD
1558int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
1559
1da177e4
LT
1560/*
1561 * The following ifdef needed to get the 4level-fixup.h header to work.
1562 * Remove it when 4level-fixup.h has been removed.
1563 */
1bb3630e 1564#if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
1da177e4
LT
1565static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
1566{
1bb3630e
HD
1567 return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
1568 NULL: pud_offset(pgd, address);
1da177e4
LT
1569}
1570
1571static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
1572{
1bb3630e
HD
1573 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
1574 NULL: pmd_offset(pud, address);
1da177e4 1575}
1bb3630e
HD
1576#endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */
1577
57c1ffce 1578#if USE_SPLIT_PTE_PTLOCKS
597d795a 1579#if ALLOC_SPLIT_PTLOCKS
b35f1819 1580void __init ptlock_cache_init(void);
539edb58
PZ
1581extern bool ptlock_alloc(struct page *page);
1582extern void ptlock_free(struct page *page);
1583
1584static inline spinlock_t *ptlock_ptr(struct page *page)
1585{
1586 return page->ptl;
1587}
597d795a 1588#else /* ALLOC_SPLIT_PTLOCKS */
b35f1819
KS
1589static inline void ptlock_cache_init(void)
1590{
1591}
1592
49076ec2
KS
1593static inline bool ptlock_alloc(struct page *page)
1594{
49076ec2
KS
1595 return true;
1596}
539edb58 1597
49076ec2
KS
1598static inline void ptlock_free(struct page *page)
1599{
49076ec2
KS
1600}
1601
1602static inline spinlock_t *ptlock_ptr(struct page *page)
1603{
539edb58 1604 return &page->ptl;
49076ec2 1605}
597d795a 1606#endif /* ALLOC_SPLIT_PTLOCKS */
49076ec2
KS
1607
1608static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
1609{
1610 return ptlock_ptr(pmd_page(*pmd));
1611}
1612
1613static inline bool ptlock_init(struct page *page)
1614{
1615 /*
1616 * prep_new_page() initialize page->private (and therefore page->ptl)
1617 * with 0. Make sure nobody took it in use in between.
1618 *
1619 * It can happen if arch try to use slab for page table allocation:
1d798ca3 1620 * slab code uses page->slab_cache, which share storage with page->ptl.
49076ec2 1621 */
309381fe 1622 VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page);
49076ec2
KS
1623 if (!ptlock_alloc(page))
1624 return false;
1625 spin_lock_init(ptlock_ptr(page));
1626 return true;
1627}
1628
1629/* Reset page->mapping so free_pages_check won't complain. */
1630static inline void pte_lock_deinit(struct page *page)
1631{
1632 page->mapping = NULL;
1633 ptlock_free(page);
1634}
1635
57c1ffce 1636#else /* !USE_SPLIT_PTE_PTLOCKS */
4c21e2f2
HD
1637/*
1638 * We use mm->page_table_lock to guard all pagetable pages of the mm.
1639 */
49076ec2
KS
1640static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
1641{
1642 return &mm->page_table_lock;
1643}
b35f1819 1644static inline void ptlock_cache_init(void) {}
49076ec2
KS
1645static inline bool ptlock_init(struct page *page) { return true; }
1646static inline void pte_lock_deinit(struct page *page) {}
57c1ffce 1647#endif /* USE_SPLIT_PTE_PTLOCKS */
4c21e2f2 1648
b35f1819
KS
1649static inline void pgtable_init(void)
1650{
1651 ptlock_cache_init();
1652 pgtable_cache_init();
1653}
1654
390f44e2 1655static inline bool pgtable_page_ctor(struct page *page)
2f569afd 1656{
706874e9
VD
1657 if (!ptlock_init(page))
1658 return false;
2f569afd 1659 inc_zone_page_state(page, NR_PAGETABLE);
706874e9 1660 return true;
2f569afd
MS
1661}
1662
1663static inline void pgtable_page_dtor(struct page *page)
1664{
1665 pte_lock_deinit(page);
1666 dec_zone_page_state(page, NR_PAGETABLE);
1667}
1668
c74df32c
HD
1669#define pte_offset_map_lock(mm, pmd, address, ptlp) \
1670({ \
4c21e2f2 1671 spinlock_t *__ptl = pte_lockptr(mm, pmd); \
c74df32c
HD
1672 pte_t *__pte = pte_offset_map(pmd, address); \
1673 *(ptlp) = __ptl; \
1674 spin_lock(__ptl); \
1675 __pte; \
1676})
1677
1678#define pte_unmap_unlock(pte, ptl) do { \
1679 spin_unlock(ptl); \
1680 pte_unmap(pte); \
1681} while (0)
1682
3ed3a4f0
KS
1683#define pte_alloc(mm, pmd, address) \
1684 (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd, address))
1685
1686#define pte_alloc_map(mm, pmd, address) \
1687 (pte_alloc(mm, pmd, address) ? NULL : pte_offset_map(pmd, address))
1bb3630e 1688
c74df32c 1689#define pte_alloc_map_lock(mm, pmd, address, ptlp) \
3ed3a4f0
KS
1690 (pte_alloc(mm, pmd, address) ? \
1691 NULL : pte_offset_map_lock(mm, pmd, address, ptlp))
c74df32c 1692
1bb3630e 1693#define pte_alloc_kernel(pmd, address) \
8ac1f832 1694 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
1bb3630e 1695 NULL: pte_offset_kernel(pmd, address))
1da177e4 1696
e009bb30
KS
1697#if USE_SPLIT_PMD_PTLOCKS
1698
634391ac
MS
1699static struct page *pmd_to_page(pmd_t *pmd)
1700{
1701 unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1);
1702 return virt_to_page((void *)((unsigned long) pmd & mask));
1703}
1704
e009bb30
KS
1705static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
1706{
634391ac 1707 return ptlock_ptr(pmd_to_page(pmd));
e009bb30
KS
1708}
1709
1710static inline bool pgtable_pmd_page_ctor(struct page *page)
1711{
e009bb30
KS
1712#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1713 page->pmd_huge_pte = NULL;
1714#endif
49076ec2 1715 return ptlock_init(page);
e009bb30
KS
1716}
1717
1718static inline void pgtable_pmd_page_dtor(struct page *page)
1719{
1720#ifdef CONFIG_TRANSPARENT_HUGEPAGE
309381fe 1721 VM_BUG_ON_PAGE(page->pmd_huge_pte, page);
e009bb30 1722#endif
49076ec2 1723 ptlock_free(page);
e009bb30
KS
1724}
1725
634391ac 1726#define pmd_huge_pte(mm, pmd) (pmd_to_page(pmd)->pmd_huge_pte)
e009bb30
KS
1727
1728#else
1729
9a86cb7b
KS
1730static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
1731{
1732 return &mm->page_table_lock;
1733}
1734
e009bb30
KS
1735static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; }
1736static inline void pgtable_pmd_page_dtor(struct page *page) {}
1737
c389a250 1738#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
9a86cb7b 1739
e009bb30
KS
1740#endif
1741
9a86cb7b
KS
1742static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
1743{
1744 spinlock_t *ptl = pmd_lockptr(mm, pmd);
1745 spin_lock(ptl);
1746 return ptl;
1747}
1748
1da177e4 1749extern void free_area_init(unsigned long * zones_size);
9109fb7b
JW
1750extern void free_area_init_node(int nid, unsigned long * zones_size,
1751 unsigned long zone_start_pfn, unsigned long *zholes_size);
49a7f04a
DH
1752extern void free_initmem(void);
1753
69afade7
JL
1754/*
1755 * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK)
1756 * into the buddy system. The freed pages will be poisoned with pattern
dbe67df4 1757 * "poison" if it's within range [0, UCHAR_MAX].
69afade7
JL
1758 * Return pages freed into the buddy system.
1759 */
11199692 1760extern unsigned long free_reserved_area(void *start, void *end,
69afade7 1761 int poison, char *s);
c3d5f5f0 1762
cfa11e08
JL
1763#ifdef CONFIG_HIGHMEM
1764/*
1765 * Free a highmem page into the buddy system, adjusting totalhigh_pages
1766 * and totalram_pages.
1767 */
1768extern void free_highmem_page(struct page *page);
1769#endif
69afade7 1770
c3d5f5f0 1771extern void adjust_managed_page_count(struct page *page, long count);
7ee3d4e8 1772extern void mem_init_print_info(const char *str);
69afade7 1773
4b50bcc7 1774extern void reserve_bootmem_region(phys_addr_t start, phys_addr_t end);
92923ca3 1775
69afade7
JL
1776/* Free the reserved page into the buddy system, so it gets managed. */
1777static inline void __free_reserved_page(struct page *page)
1778{
1779 ClearPageReserved(page);
1780 init_page_count(page);
1781 __free_page(page);
1782}
1783
1784static inline void free_reserved_page(struct page *page)
1785{
1786 __free_reserved_page(page);
1787 adjust_managed_page_count(page, 1);
1788}
1789
1790static inline void mark_page_reserved(struct page *page)
1791{
1792 SetPageReserved(page);
1793 adjust_managed_page_count(page, -1);
1794}
1795
1796/*
1797 * Default method to free all the __init memory into the buddy system.
dbe67df4
JL
1798 * The freed pages will be poisoned with pattern "poison" if it's within
1799 * range [0, UCHAR_MAX].
1800 * Return pages freed into the buddy system.
69afade7
JL
1801 */
1802static inline unsigned long free_initmem_default(int poison)
1803{
1804 extern char __init_begin[], __init_end[];
1805
11199692 1806 return free_reserved_area(&__init_begin, &__init_end,
69afade7
JL
1807 poison, "unused kernel");
1808}
1809
7ee3d4e8
JL
1810static inline unsigned long get_num_physpages(void)
1811{
1812 int nid;
1813 unsigned long phys_pages = 0;
1814
1815 for_each_online_node(nid)
1816 phys_pages += node_present_pages(nid);
1817
1818 return phys_pages;
1819}
1820
0ee332c1 1821#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
c713216d 1822/*
0ee332c1 1823 * With CONFIG_HAVE_MEMBLOCK_NODE_MAP set, an architecture may initialise its
c713216d
MG
1824 * zones, allocate the backing mem_map and account for memory holes in a more
1825 * architecture independent manner. This is a substitute for creating the
1826 * zone_sizes[] and zholes_size[] arrays and passing them to
1827 * free_area_init_node()
1828 *
1829 * An architecture is expected to register range of page frames backed by
0ee332c1 1830 * physical memory with memblock_add[_node]() before calling
c713216d
MG
1831 * free_area_init_nodes() passing in the PFN each zone ends at. At a basic
1832 * usage, an architecture is expected to do something like
1833 *
1834 * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn,
1835 * max_highmem_pfn};
1836 * for_each_valid_physical_page_range()
0ee332c1 1837 * memblock_add_node(base, size, nid)
c713216d
MG
1838 * free_area_init_nodes(max_zone_pfns);
1839 *
0ee332c1
TH
1840 * free_bootmem_with_active_regions() calls free_bootmem_node() for each
1841 * registered physical page range. Similarly
1842 * sparse_memory_present_with_active_regions() calls memory_present() for
1843 * each range when SPARSEMEM is enabled.
c713216d
MG
1844 *
1845 * See mm/page_alloc.c for more information on each function exposed by
0ee332c1 1846 * CONFIG_HAVE_MEMBLOCK_NODE_MAP.
c713216d
MG
1847 */
1848extern void free_area_init_nodes(unsigned long *max_zone_pfn);
1e01979c 1849unsigned long node_map_pfn_alignment(void);
32996250
YL
1850unsigned long __absent_pages_in_range(int nid, unsigned long start_pfn,
1851 unsigned long end_pfn);
c713216d
MG
1852extern unsigned long absent_pages_in_range(unsigned long start_pfn,
1853 unsigned long end_pfn);
1854extern void get_pfn_range_for_nid(unsigned int nid,
1855 unsigned long *start_pfn, unsigned long *end_pfn);
1856extern unsigned long find_min_pfn_with_active_regions(void);
c713216d
MG
1857extern void free_bootmem_with_active_regions(int nid,
1858 unsigned long max_low_pfn);
1859extern void sparse_memory_present_with_active_regions(int nid);
f2dbcfa7 1860
0ee332c1 1861#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
f2dbcfa7 1862
0ee332c1 1863#if !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) && \
f2dbcfa7 1864 !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID)
8a942fde
MG
1865static inline int __early_pfn_to_nid(unsigned long pfn,
1866 struct mminit_pfnnid_cache *state)
f2dbcfa7
KH
1867{
1868 return 0;
1869}
1870#else
1871/* please see mm/page_alloc.c */
1872extern int __meminit early_pfn_to_nid(unsigned long pfn);
f2dbcfa7 1873/* there is a per-arch backend function. */
8a942fde
MG
1874extern int __meminit __early_pfn_to_nid(unsigned long pfn,
1875 struct mminit_pfnnid_cache *state);
f2dbcfa7
KH
1876#endif
1877
0e0b864e 1878extern void set_dma_reserve(unsigned long new_dma_reserve);
a2f3aa02
DH
1879extern void memmap_init_zone(unsigned long, int, unsigned long,
1880 unsigned long, enum memmap_context);
bc75d33f 1881extern void setup_per_zone_wmarks(void);
1b79acc9 1882extern int __meminit init_per_zone_wmark_min(void);
1da177e4 1883extern void mem_init(void);
8feae131 1884extern void __init mmap_init(void);
b2b755b5 1885extern void show_mem(unsigned int flags);
d02bd27b 1886extern long si_mem_available(void);
1da177e4
LT
1887extern void si_meminfo(struct sysinfo * val);
1888extern void si_meminfo_node(struct sysinfo *val, int nid);
1889
3ee9a4f0 1890extern __printf(3, 4)
d00181b9
KS
1891void warn_alloc_failed(gfp_t gfp_mask, unsigned int order,
1892 const char *fmt, ...);
a238ab5b 1893
e7c8d5c9 1894extern void setup_per_cpu_pageset(void);
e7c8d5c9 1895
112067f0 1896extern void zone_pcp_update(struct zone *zone);
340175b7 1897extern void zone_pcp_reset(struct zone *zone);
112067f0 1898
75f7ad8e
PS
1899/* page_alloc.c */
1900extern int min_free_kbytes;
795ae7a0 1901extern int watermark_scale_factor;
75f7ad8e 1902
8feae131 1903/* nommu.c */
33e5d769 1904extern atomic_long_t mmap_pages_allocated;
7e660872 1905extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
8feae131 1906
6b2dbba8 1907/* interval_tree.c */
6b2dbba8
ML
1908void vma_interval_tree_insert(struct vm_area_struct *node,
1909 struct rb_root *root);
9826a516
ML
1910void vma_interval_tree_insert_after(struct vm_area_struct *node,
1911 struct vm_area_struct *prev,
1912 struct rb_root *root);
6b2dbba8
ML
1913void vma_interval_tree_remove(struct vm_area_struct *node,
1914 struct rb_root *root);
1915struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root *root,
1916 unsigned long start, unsigned long last);
1917struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node,
1918 unsigned long start, unsigned long last);
1919
1920#define vma_interval_tree_foreach(vma, root, start, last) \
1921 for (vma = vma_interval_tree_iter_first(root, start, last); \
1922 vma; vma = vma_interval_tree_iter_next(vma, start, last))
1da177e4 1923
bf181b9f
ML
1924void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
1925 struct rb_root *root);
1926void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
1927 struct rb_root *root);
1928struct anon_vma_chain *anon_vma_interval_tree_iter_first(
1929 struct rb_root *root, unsigned long start, unsigned long last);
1930struct anon_vma_chain *anon_vma_interval_tree_iter_next(
1931 struct anon_vma_chain *node, unsigned long start, unsigned long last);
ed8ea815
ML
1932#ifdef CONFIG_DEBUG_VM_RB
1933void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
1934#endif
bf181b9f
ML
1935
1936#define anon_vma_interval_tree_foreach(avc, root, start, last) \
1937 for (avc = anon_vma_interval_tree_iter_first(root, start, last); \
1938 avc; avc = anon_vma_interval_tree_iter_next(avc, start, last))
1939
1da177e4 1940/* mmap.c */
34b4e4aa 1941extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
5beb4930 1942extern int vma_adjust(struct vm_area_struct *vma, unsigned long start,
1da177e4
LT
1943 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert);
1944extern struct vm_area_struct *vma_merge(struct mm_struct *,
1945 struct vm_area_struct *prev, unsigned long addr, unsigned long end,
1946 unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
19a809af 1947 struct mempolicy *, struct vm_userfaultfd_ctx);
1da177e4
LT
1948extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
1949extern int split_vma(struct mm_struct *,
1950 struct vm_area_struct *, unsigned long addr, int new_below);
1951extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
1952extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
1953 struct rb_node **, struct rb_node *);
a8fb5618 1954extern void unlink_file_vma(struct vm_area_struct *);
1da177e4 1955extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
38a76013
ML
1956 unsigned long addr, unsigned long len, pgoff_t pgoff,
1957 bool *need_rmap_locks);
1da177e4 1958extern void exit_mmap(struct mm_struct *);
925d1c40 1959
9c599024
CG
1960static inline int check_data_rlimit(unsigned long rlim,
1961 unsigned long new,
1962 unsigned long start,
1963 unsigned long end_data,
1964 unsigned long start_data)
1965{
1966 if (rlim < RLIM_INFINITY) {
1967 if (((new - start) + (end_data - start_data)) > rlim)
1968 return -ENOSPC;
1969 }
1970
1971 return 0;
1972}
1973
7906d00c
AA
1974extern int mm_take_all_locks(struct mm_struct *mm);
1975extern void mm_drop_all_locks(struct mm_struct *mm);
1976
38646013
JS
1977extern void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
1978extern struct file *get_mm_exe_file(struct mm_struct *mm);
925d1c40 1979
84638335
KK
1980extern bool may_expand_vm(struct mm_struct *, vm_flags_t, unsigned long npages);
1981extern void vm_stat_account(struct mm_struct *, vm_flags_t, long npages);
1982
3935ed6a
SS
1983extern struct vm_area_struct *_install_special_mapping(struct mm_struct *mm,
1984 unsigned long addr, unsigned long len,
a62c34bd
AL
1985 unsigned long flags,
1986 const struct vm_special_mapping *spec);
1987/* This is an obsolete alternative to _install_special_mapping. */
fa5dc22f
RM
1988extern int install_special_mapping(struct mm_struct *mm,
1989 unsigned long addr, unsigned long len,
1990 unsigned long flags, struct page **pages);
1da177e4
LT
1991
1992extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
1993
0165ab44 1994extern unsigned long mmap_region(struct file *file, unsigned long addr,
c22c0d63 1995 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff);
1fcfd8db 1996extern unsigned long do_mmap(struct file *file, unsigned long addr,
bebeb3d6 1997 unsigned long len, unsigned long prot, unsigned long flags,
1fcfd8db 1998 vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate);
1da177e4
LT
1999extern int do_munmap(struct mm_struct *, unsigned long, size_t);
2000
1fcfd8db
ON
2001static inline unsigned long
2002do_mmap_pgoff(struct file *file, unsigned long addr,
2003 unsigned long len, unsigned long prot, unsigned long flags,
2004 unsigned long pgoff, unsigned long *populate)
2005{
2006 return do_mmap(file, addr, len, prot, flags, 0, pgoff, populate);
2007}
2008
bebeb3d6
ML
2009#ifdef CONFIG_MMU
2010extern int __mm_populate(unsigned long addr, unsigned long len,
2011 int ignore_errors);
2012static inline void mm_populate(unsigned long addr, unsigned long len)
2013{
2014 /* Ignore errors */
2015 (void) __mm_populate(addr, len, 1);
2016}
2017#else
2018static inline void mm_populate(unsigned long addr, unsigned long len) {}
2019#endif
2020
e4eb1ff6 2021/* These take the mm semaphore themselves */
5d22fc25 2022extern int __must_check vm_brk(unsigned long, unsigned long);
bfce281c 2023extern int vm_munmap(unsigned long, size_t);
9fbeb5ab 2024extern unsigned long __must_check vm_mmap(struct file *, unsigned long,
6be5ceb0
LT
2025 unsigned long, unsigned long,
2026 unsigned long, unsigned long);
1da177e4 2027
db4fbfb9
ML
2028struct vm_unmapped_area_info {
2029#define VM_UNMAPPED_AREA_TOPDOWN 1
2030 unsigned long flags;
2031 unsigned long length;
2032 unsigned long low_limit;
2033 unsigned long high_limit;
2034 unsigned long align_mask;
2035 unsigned long align_offset;
2036};
2037
2038extern unsigned long unmapped_area(struct vm_unmapped_area_info *info);
2039extern unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info);
2040
2041/*
2042 * Search for an unmapped address range.
2043 *
2044 * We are looking for a range that:
2045 * - does not intersect with any VMA;
2046 * - is contained within the [low_limit, high_limit) interval;
2047 * - is at least the desired size.
2048 * - satisfies (begin_addr & align_mask) == (align_offset & align_mask)
2049 */
2050static inline unsigned long
2051vm_unmapped_area(struct vm_unmapped_area_info *info)
2052{
cdd7875e 2053 if (info->flags & VM_UNMAPPED_AREA_TOPDOWN)
db4fbfb9 2054 return unmapped_area_topdown(info);
cdd7875e
BP
2055 else
2056 return unmapped_area(info);
db4fbfb9
ML
2057}
2058
85821aab 2059/* truncate.c */
1da177e4 2060extern void truncate_inode_pages(struct address_space *, loff_t);
d7339071
HR
2061extern void truncate_inode_pages_range(struct address_space *,
2062 loff_t lstart, loff_t lend);
91b0abe3 2063extern void truncate_inode_pages_final(struct address_space *);
1da177e4
LT
2064
2065/* generic vm_area_ops exported for stackable file systems */
d0217ac0 2066extern int filemap_fault(struct vm_area_struct *, struct vm_fault *);
f1820361 2067extern void filemap_map_pages(struct vm_area_struct *vma, struct vm_fault *vmf);
4fcf1c62 2068extern int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf);
1da177e4
LT
2069
2070/* mm/page-writeback.c */
2071int write_one_page(struct page *page, int wait);
1cf6e7d8 2072void task_dirty_inc(struct task_struct *tsk);
1da177e4
LT
2073
2074/* readahead.c */
2075#define VM_MAX_READAHEAD 128 /* kbytes */
2076#define VM_MIN_READAHEAD 16 /* kbytes (includes current page) */
1da177e4 2077
1da177e4 2078int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
7361f4d8 2079 pgoff_t offset, unsigned long nr_to_read);
cf914a7d
RR
2080
2081void page_cache_sync_readahead(struct address_space *mapping,
2082 struct file_ra_state *ra,
2083 struct file *filp,
2084 pgoff_t offset,
2085 unsigned long size);
2086
2087void page_cache_async_readahead(struct address_space *mapping,
2088 struct file_ra_state *ra,
2089 struct file *filp,
2090 struct page *pg,
2091 pgoff_t offset,
2092 unsigned long size);
2093
d05f3169 2094/* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */
46dea3d0 2095extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
d05f3169
MH
2096
2097/* CONFIG_STACK_GROWSUP still needs to to grow downwards at some places */
2098extern int expand_downwards(struct vm_area_struct *vma,
2099 unsigned long address);
8ca3eb08 2100#if VM_GROWSUP
46dea3d0 2101extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
8ca3eb08 2102#else
fee7e49d 2103 #define expand_upwards(vma, address) (0)
9ab88515 2104#endif
1da177e4
LT
2105
2106/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
2107extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
2108extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
2109 struct vm_area_struct **pprev);
2110
2111/* Look up the first VMA which intersects the interval start_addr..end_addr-1,
2112 NULL if none. Assume start_addr < end_addr. */
2113static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
2114{
2115 struct vm_area_struct * vma = find_vma(mm,start_addr);
2116
2117 if (vma && end_addr <= vma->vm_start)
2118 vma = NULL;
2119 return vma;
2120}
2121
2122static inline unsigned long vma_pages(struct vm_area_struct *vma)
2123{
2124 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
2125}
2126
640708a2
PE
2127/* Look up the first VMA which exactly match the interval vm_start ... vm_end */
2128static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
2129 unsigned long vm_start, unsigned long vm_end)
2130{
2131 struct vm_area_struct *vma = find_vma(mm, vm_start);
2132
2133 if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end))
2134 vma = NULL;
2135
2136 return vma;
2137}
2138
bad849b3 2139#ifdef CONFIG_MMU
804af2cf 2140pgprot_t vm_get_page_prot(unsigned long vm_flags);
64e45507 2141void vma_set_page_prot(struct vm_area_struct *vma);
bad849b3
DH
2142#else
2143static inline pgprot_t vm_get_page_prot(unsigned long vm_flags)
2144{
2145 return __pgprot(0);
2146}
64e45507
PF
2147static inline void vma_set_page_prot(struct vm_area_struct *vma)
2148{
2149 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
2150}
bad849b3
DH
2151#endif
2152
5877231f 2153#ifdef CONFIG_NUMA_BALANCING
4b10e7d5 2154unsigned long change_prot_numa(struct vm_area_struct *vma,
b24f53a0
LS
2155 unsigned long start, unsigned long end);
2156#endif
2157
deceb6cd 2158struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
deceb6cd
HD
2159int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
2160 unsigned long pfn, unsigned long size, pgprot_t);
a145dd41 2161int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
e0dc0d8f
NP
2162int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
2163 unsigned long pfn);
1745cbc5
AL
2164int vm_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
2165 unsigned long pfn, pgprot_t pgprot);
423bad60 2166int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
01c8f1c4 2167 pfn_t pfn);
b4cbb197
LT
2168int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len);
2169
deceb6cd 2170
240aadee
ML
2171struct page *follow_page_mask(struct vm_area_struct *vma,
2172 unsigned long address, unsigned int foll_flags,
2173 unsigned int *page_mask);
2174
2175static inline struct page *follow_page(struct vm_area_struct *vma,
2176 unsigned long address, unsigned int foll_flags)
2177{
2178 unsigned int unused_page_mask;
2179 return follow_page_mask(vma, address, foll_flags, &unused_page_mask);
2180}
2181
deceb6cd
HD
2182#define FOLL_WRITE 0x01 /* check pte is writable */
2183#define FOLL_TOUCH 0x02 /* mark page accessed */
2184#define FOLL_GET 0x04 /* do get_page on page */
8e4b9a60 2185#define FOLL_DUMP 0x08 /* give error on hole if it would be zero */
58fa879e 2186#define FOLL_FORCE 0x10 /* get_user_pages read/write w/o permission */
318b275f
GN
2187#define FOLL_NOWAIT 0x20 /* if a disk transfer is needed, start the IO
2188 * and return without waiting upon it */
84d33df2 2189#define FOLL_POPULATE 0x40 /* fault in page */
500d65d4 2190#define FOLL_SPLIT 0x80 /* don't return transhuge pages, split them */
69ebb83e 2191#define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
0b9d7052 2192#define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
5117b3b8 2193#define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */
234b239b 2194#define FOLL_TRIED 0x800 /* a retry, previous pass started an IO */
de60f5f1 2195#define FOLL_MLOCK 0x1000 /* lock present pages */
1e987790 2196#define FOLL_REMOTE 0x2000 /* we are working on non-current tsk/mm */
1da177e4 2197
2f569afd 2198typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
aee16b3c
JF
2199 void *data);
2200extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
2201 unsigned long size, pte_fn_t fn, void *data);
2202
1da177e4 2203
8823b1db
LA
2204#ifdef CONFIG_PAGE_POISONING
2205extern bool page_poisoning_enabled(void);
2206extern void kernel_poison_pages(struct page *page, int numpages, int enable);
1414c7f4 2207extern bool page_is_poisoned(struct page *page);
8823b1db
LA
2208#else
2209static inline bool page_poisoning_enabled(void) { return false; }
2210static inline void kernel_poison_pages(struct page *page, int numpages,
2211 int enable) { }
1414c7f4 2212static inline bool page_is_poisoned(struct page *page) { return false; }
8823b1db
LA
2213#endif
2214
12d6f21e 2215#ifdef CONFIG_DEBUG_PAGEALLOC
031bc574
JK
2216extern bool _debug_pagealloc_enabled;
2217extern void __kernel_map_pages(struct page *page, int numpages, int enable);
2218
2219static inline bool debug_pagealloc_enabled(void)
2220{
2221 return _debug_pagealloc_enabled;
2222}
2223
2224static inline void
2225kernel_map_pages(struct page *page, int numpages, int enable)
2226{
2227 if (!debug_pagealloc_enabled())
2228 return;
2229
2230 __kernel_map_pages(page, numpages, enable);
2231}
8a235efa
RW
2232#ifdef CONFIG_HIBERNATION
2233extern bool kernel_page_present(struct page *page);
40b44137
JK
2234#endif /* CONFIG_HIBERNATION */
2235#else /* CONFIG_DEBUG_PAGEALLOC */
1da177e4 2236static inline void
9858db50 2237kernel_map_pages(struct page *page, int numpages, int enable) {}
8a235efa
RW
2238#ifdef CONFIG_HIBERNATION
2239static inline bool kernel_page_present(struct page *page) { return true; }
40b44137
JK
2240#endif /* CONFIG_HIBERNATION */
2241static inline bool debug_pagealloc_enabled(void)
2242{
2243 return false;
2244}
2245#endif /* CONFIG_DEBUG_PAGEALLOC */
1da177e4 2246
a6c19dfe 2247#ifdef __HAVE_ARCH_GATE_AREA
31db58b3 2248extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm);
a6c19dfe
AL
2249extern int in_gate_area_no_mm(unsigned long addr);
2250extern int in_gate_area(struct mm_struct *mm, unsigned long addr);
1da177e4 2251#else
a6c19dfe
AL
2252static inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
2253{
2254 return NULL;
2255}
2256static inline int in_gate_area_no_mm(unsigned long addr) { return 0; }
2257static inline int in_gate_area(struct mm_struct *mm, unsigned long addr)
2258{
2259 return 0;
2260}
1da177e4
LT
2261#endif /* __HAVE_ARCH_GATE_AREA */
2262
146732ce
JT
2263#ifdef CONFIG_SYSCTL
2264extern int sysctl_drop_caches;
8d65af78 2265int drop_caches_sysctl_handler(struct ctl_table *, int,
9d0243bc 2266 void __user *, size_t *, loff_t *);
146732ce
JT
2267#endif
2268
cb731d6c
VD
2269void drop_slab(void);
2270void drop_slab_node(int nid);
9d0243bc 2271
7a9166e3
LY
2272#ifndef CONFIG_MMU
2273#define randomize_va_space 0
2274#else
a62eaf15 2275extern int randomize_va_space;
7a9166e3 2276#endif
a62eaf15 2277
045e72ac 2278const char * arch_vma_name(struct vm_area_struct *vma);
03252919 2279void print_vma_addr(char *prefix, unsigned long rip);
e6e5494c 2280
9bdac914
YL
2281void sparse_mem_maps_populate_node(struct page **map_map,
2282 unsigned long pnum_begin,
2283 unsigned long pnum_end,
2284 unsigned long map_count,
2285 int nodeid);
2286
98f3cfc1 2287struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
29c71111
AW
2288pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
2289pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
2290pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
2291pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
8f6aac41 2292void *vmemmap_alloc_block(unsigned long size, int node);
4b94ffdc
DW
2293struct vmem_altmap;
2294void *__vmemmap_alloc_block_buf(unsigned long size, int node,
2295 struct vmem_altmap *altmap);
2296static inline void *vmemmap_alloc_block_buf(unsigned long size, int node)
2297{
2298 return __vmemmap_alloc_block_buf(size, node, NULL);
2299}
2300
8f6aac41 2301void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
0aad818b
JW
2302int vmemmap_populate_basepages(unsigned long start, unsigned long end,
2303 int node);
2304int vmemmap_populate(unsigned long start, unsigned long end, int node);
c2b91e2e 2305void vmemmap_populate_print_last(void);
0197518c 2306#ifdef CONFIG_MEMORY_HOTPLUG
0aad818b 2307void vmemmap_free(unsigned long start, unsigned long end);
0197518c 2308#endif
46723bfa
YI
2309void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
2310 unsigned long size);
6a46079c 2311
82ba011b
AK
2312enum mf_flags {
2313 MF_COUNT_INCREASED = 1 << 0,
7329bbeb 2314 MF_ACTION_REQUIRED = 1 << 1,
6751ed65 2315 MF_MUST_KILL = 1 << 2,
cf870c70 2316 MF_SOFT_OFFLINE = 1 << 3,
82ba011b 2317};
cd42f4a3 2318extern int memory_failure(unsigned long pfn, int trapno, int flags);
ea8f5fb8 2319extern void memory_failure_queue(unsigned long pfn, int trapno, int flags);
847ce401 2320extern int unpoison_memory(unsigned long pfn);
ead07f6a 2321extern int get_hwpoison_page(struct page *page);
4e41a30c 2322#define put_hwpoison_page(page) put_page(page)
6a46079c
AK
2323extern int sysctl_memory_failure_early_kill;
2324extern int sysctl_memory_failure_recovery;
facb6011 2325extern void shake_page(struct page *p, int access);
293c07e3 2326extern atomic_long_t num_poisoned_pages;
facb6011 2327extern int soft_offline_page(struct page *page, int flags);
6a46079c 2328
cc637b17
XX
2329
2330/*
2331 * Error handlers for various types of pages.
2332 */
cc3e2af4 2333enum mf_result {
cc637b17
XX
2334 MF_IGNORED, /* Error: cannot be handled */
2335 MF_FAILED, /* Error: handling failed */
2336 MF_DELAYED, /* Will be handled later */
2337 MF_RECOVERED, /* Successfully recovered */
2338};
2339
2340enum mf_action_page_type {
2341 MF_MSG_KERNEL,
2342 MF_MSG_KERNEL_HIGH_ORDER,
2343 MF_MSG_SLAB,
2344 MF_MSG_DIFFERENT_COMPOUND,
2345 MF_MSG_POISONED_HUGE,
2346 MF_MSG_HUGE,
2347 MF_MSG_FREE_HUGE,
2348 MF_MSG_UNMAP_FAILED,
2349 MF_MSG_DIRTY_SWAPCACHE,
2350 MF_MSG_CLEAN_SWAPCACHE,
2351 MF_MSG_DIRTY_MLOCKED_LRU,
2352 MF_MSG_CLEAN_MLOCKED_LRU,
2353 MF_MSG_DIRTY_UNEVICTABLE_LRU,
2354 MF_MSG_CLEAN_UNEVICTABLE_LRU,
2355 MF_MSG_DIRTY_LRU,
2356 MF_MSG_CLEAN_LRU,
2357 MF_MSG_TRUNCATED_LRU,
2358 MF_MSG_BUDDY,
2359 MF_MSG_BUDDY_2ND,
2360 MF_MSG_UNKNOWN,
2361};
2362
47ad8475
AA
2363#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
2364extern void clear_huge_page(struct page *page,
2365 unsigned long addr,
2366 unsigned int pages_per_huge_page);
2367extern void copy_user_huge_page(struct page *dst, struct page *src,
2368 unsigned long addr, struct vm_area_struct *vma,
2369 unsigned int pages_per_huge_page);
2370#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
2371
e30825f1
JK
2372extern struct page_ext_operations debug_guardpage_ops;
2373extern struct page_ext_operations page_poisoning_ops;
2374
c0a32fc5
SG
2375#ifdef CONFIG_DEBUG_PAGEALLOC
2376extern unsigned int _debug_guardpage_minorder;
e30825f1 2377extern bool _debug_guardpage_enabled;
c0a32fc5
SG
2378
2379static inline unsigned int debug_guardpage_minorder(void)
2380{
2381 return _debug_guardpage_minorder;
2382}
2383
e30825f1
JK
2384static inline bool debug_guardpage_enabled(void)
2385{
2386 return _debug_guardpage_enabled;
2387}
2388
c0a32fc5
SG
2389static inline bool page_is_guard(struct page *page)
2390{
e30825f1
JK
2391 struct page_ext *page_ext;
2392
2393 if (!debug_guardpage_enabled())
2394 return false;
2395
2396 page_ext = lookup_page_ext(page);
0bb2fd13
YS
2397 if (unlikely(!page_ext))
2398 return false;
2399
e30825f1 2400 return test_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
c0a32fc5
SG
2401}
2402#else
2403static inline unsigned int debug_guardpage_minorder(void) { return 0; }
e30825f1 2404static inline bool debug_guardpage_enabled(void) { return false; }
c0a32fc5
SG
2405static inline bool page_is_guard(struct page *page) { return false; }
2406#endif /* CONFIG_DEBUG_PAGEALLOC */
2407
f9872caf
CS
2408#if MAX_NUMNODES > 1
2409void __init setup_nr_node_ids(void);
2410#else
2411static inline void setup_nr_node_ids(void) {}
2412#endif
2413
1da177e4
LT
2414#endif /* __KERNEL__ */
2415#endif /* _LINUX_MM_H */