[linux-block.git] / include / linux / mmu_notifier.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_MMU_NOTIFIER_H
#define _LINUX_MMU_NOTIFIER_H

#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/mm_types.h>
#include <linux/srcu.h>

struct mmu_notifier;
struct mmu_notifier_ops;

#ifdef CONFIG_MMU_NOTIFIER

/*
 * The mmu notifier_mm structure is allocated and installed in
 * mm->mmu_notifier_mm inside the mm_take_all_locks() protected
 * critical section and it's released only when mm_count reaches zero
 * in mmdrop().
 */
struct mmu_notifier_mm {
	/* all mmu notifiers registerd in this mm are queued in this list */
	struct hlist_head list;
	/* to serialize the list modifications and hlist_unhashed */
	spinlock_t lock;
};

struct mmu_notifier_range {
	struct mm_struct *mm;
	unsigned long start;
	unsigned long end;
	bool blockable;
};

struct mmu_notifier_ops {
	/*
	 * Called either by mmu_notifier_unregister or when the mm is
	 * being destroyed by exit_mmap, always before all pages are
	 * freed. This can run concurrently with other mmu notifier
	 * methods (the ones invoked outside the mm context) and it
	 * should tear down all secondary mmu mappings and freeze the
	 * secondary mmu. If this method isn't implemented you've to
	 * be sure that nothing could possibly write to the pages
	 * through the secondary mmu by the time the last thread with
	 * tsk->mm == mm exits.
	 *
	 * As side note: the pages freed after ->release returns could
	 * be immediately reallocated by the gart at an alias physical
	 * address with a different cache model, so if ->release isn't
	 * implemented because all _software_ driven memory accesses
	 * through the secondary mmu are terminated by the time the
	 * last thread of this mm quits, you've also to be sure that
	 * speculative _hardware_ operations can't allocate dirty
	 * cachelines in the cpu that could not be snooped and made
	 * coherent with the other read and write operations happening
	 * through the gart alias address, so leading to memory
	 * corruption.
	 */
	void (*release)(struct mmu_notifier *mn,
			struct mm_struct *mm);

	/*
	 * clear_flush_young is called after the VM is
	 * test-and-clearing the young/accessed bitflag in the
	 * pte. This way the VM will provide proper aging to the
	 * accesses to the page through the secondary MMUs and not
	 * only to the ones through the Linux pte.
	 * Start-end is necessary in case the secondary MMU is mapping the page
	 * at a smaller granularity than the primary MMU.
	 */
	int (*clear_flush_young)(struct mmu_notifier *mn,
				 struct mm_struct *mm,
				 unsigned long start,
				 unsigned long end);

	/*
	 * clear_young is a lightweight version of clear_flush_young. Like the
	 * latter, it is supposed to test-and-clear the young/accessed bitflag
	 * in the secondary pte, but it may omit flushing the secondary tlb.
	 */
	int (*clear_young)(struct mmu_notifier *mn,
			   struct mm_struct *mm,
			   unsigned long start,
			   unsigned long end);

	/*
	 * test_young is called to check the young/accessed bitflag in
	 * the secondary pte. This is used to know if the page is
	 * frequently used without actually clearing the flag or tearing
	 * down the secondary mapping on the page.
	 */
	int (*test_young)(struct mmu_notifier *mn,
			  struct mm_struct *mm,
			  unsigned long address);

	/*
	 * change_pte is called in cases that pte mapping to page is changed:
	 * for example, when ksm remaps pte to point to a new shared page.
	 */
	void (*change_pte)(struct mmu_notifier *mn,
			   struct mm_struct *mm,
			   unsigned long address,
			   pte_t pte);

	/*
	 * invalidate_range_start() and invalidate_range_end() must be
	 * paired and are called only when the mmap_sem and/or the
	 * locks protecting the reverse maps are held. If the subsystem
	 * can't guarantee that no additional references are taken to
	 * the pages in the range, it has to implement the
	 * invalidate_range() notifier to remove any references taken
	 * after invalidate_range_start().
	 *
	 * Invalidation of multiple concurrent ranges may be
	 * optionally permitted by the driver. Either way the
	 * establishment of sptes is forbidden in the range passed to
	 * invalidate_range_begin/end for the whole duration of the
	 * invalidate_range_begin/end critical section.
	 *
	 * invalidate_range_start() is called when all pages in the
	 * range are still mapped and have at least a refcount of one.
	 *
	 * invalidate_range_end() is called when all pages in the
	 * range have been unmapped and the pages have been freed by
	 * the VM.
	 *
	 * The VM will remove the page table entries and potentially
	 * the page between invalidate_range_start() and
	 * invalidate_range_end(). If the page must not be freed
	 * because of pending I/O or other circumstances then the
	 * invalidate_range_start() callback (or the initial mapping
	 * by the driver) must make sure that the refcount is kept
	 * elevated.
	 *
	 * If the driver increases the refcount when the pages are
	 * initially mapped into an address space then either
	 * invalidate_range_start() or invalidate_range_end() may
	 * decrease the refcount. If the refcount is decreased on
	 * invalidate_range_start() then the VM can free pages as page
	 * table entries are removed.  If the refcount is only
	 * droppped on invalidate_range_end() then the driver itself
	 * will drop the last refcount but it must take care to flush
	 * any secondary tlb before doing the final free on the
	 * page. Pages will no longer be referenced by the linux
	 * address space but may still be referenced by sptes until
	 * the last refcount is dropped.
	 *
	 * If blockable argument is set to false then the callback cannot
	 * sleep and has to return with -EAGAIN. 0 should be returned
	 * otherwise. Please note that if invalidate_range_start approves
	 * a non-blocking behavior then the same applies to
	 * invalidate_range_end.
	 *
	 */
	int (*invalidate_range_start)(struct mmu_notifier *mn,
				      const struct mmu_notifier_range *range);
	void (*invalidate_range_end)(struct mmu_notifier *mn,
				     const struct mmu_notifier_range *range);

	/*
	 * invalidate_range() is either called between
	 * invalidate_range_start() and invalidate_range_end() when the
	 * VM has to free pages that where unmapped, but before the
	 * pages are actually freed, or outside of _start()/_end() when
	 * a (remote) TLB is necessary.
	 *
	 * If invalidate_range() is used to manage a non-CPU TLB with
	 * shared page-tables, it not necessary to implement the
	 * invalidate_range_start()/end() notifiers, as
	 * invalidate_range() alread catches the points in time when an
	 * external TLB range needs to be flushed. For more in depth
	 * discussion on this see Documentation/vm/mmu_notifier.rst
	 *
	 * Note that this function might be called with just a sub-range
	 * of what was passed to invalidate_range_start()/end(), if
	 * called between those functions.
	 */
	void (*invalidate_range)(struct mmu_notifier *mn, struct mm_struct *mm,
				 unsigned long start, unsigned long end);
};

/*
 * The notifier chains are protected by mmap_sem and/or the reverse map
 * semaphores. Notifier chains are only changed when all reverse maps and
 * the mmap_sem locks are taken.
 *
 * Therefore notifier chains can only be traversed when either
 *
 * 1. mmap_sem is held.
 * 2. One of the reverse map locks is held (i_mmap_rwsem or anon_vma->rwsem).
 * 3. No other concurrent thread can access the list (release)
 */
struct mmu_notifier {
	struct hlist_node hlist;
	const struct mmu_notifier_ops *ops;
};

static inline int mm_has_notifiers(struct mm_struct *mm)
{
	return unlikely(mm->mmu_notifier_mm);
}

extern int mmu_notifier_register(struct mmu_notifier *mn,
				 struct mm_struct *mm);
extern int __mmu_notifier_register(struct mmu_notifier *mn,
				   struct mm_struct *mm);
extern void mmu_notifier_unregister(struct mmu_notifier *mn,
				    struct mm_struct *mm);
extern void mmu_notifier_unregister_no_release(struct mmu_notifier *mn,
					       struct mm_struct *mm);
extern void __mmu_notifier_mm_destroy(struct mm_struct *mm);
extern void __mmu_notifier_release(struct mm_struct *mm);
extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
					  unsigned long start,
					  unsigned long end);
extern int __mmu_notifier_clear_young(struct mm_struct *mm,
				      unsigned long start,
				      unsigned long end);
extern int __mmu_notifier_test_young(struct mm_struct *mm,
				     unsigned long address);
extern void __mmu_notifier_change_pte(struct mm_struct *mm,
				      unsigned long address, pte_t pte);
extern int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *r);
extern void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *r,
				  bool only_end);
extern void __mmu_notifier_invalidate_range(struct mm_struct *mm,
				  unsigned long start, unsigned long end);

static inline void mmu_notifier_release(struct mm_struct *mm)
{
	if (mm_has_notifiers(mm))
		__mmu_notifier_release(mm);
}

static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm,
					  unsigned long start,
					  unsigned long end)
{
	if (mm_has_notifiers(mm))
		return __mmu_notifier_clear_flush_young(mm, start, end);
	return 0;
}

static inline int mmu_notifier_clear_young(struct mm_struct *mm,
					   unsigned long start,
					   unsigned long end)
{
	if (mm_has_notifiers(mm))
		return __mmu_notifier_clear_young(mm, start, end);
	return 0;
}

static inline int mmu_notifier_test_young(struct mm_struct *mm,
					  unsigned long address)
{
	if (mm_has_notifiers(mm))
		return __mmu_notifier_test_young(mm, address);
	return 0;
}

static inline void mmu_notifier_change_pte(struct mm_struct *mm,
					   unsigned long address, pte_t pte)
{
	if (mm_has_notifiers(mm))
		__mmu_notifier_change_pte(mm, address, pte);
}

static inline void
mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
{
	if (mm_has_notifiers(range->mm)) {
		range->blockable = true;
		__mmu_notifier_invalidate_range_start(range);
	}
}

static inline int
mmu_notifier_invalidate_range_start_nonblock(struct mmu_notifier_range *range)
{
	if (mm_has_notifiers(range->mm)) {
		range->blockable = false;
		return __mmu_notifier_invalidate_range_start(range);
	}
	return 0;
}

static inline void
mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range)
{
	if (mm_has_notifiers(range->mm))
		__mmu_notifier_invalidate_range_end(range, false);
}

static inline void
mmu_notifier_invalidate_range_only_end(struct mmu_notifier_range *range)
{
	if (mm_has_notifiers(range->mm))
		__mmu_notifier_invalidate_range_end(range, true);
}

static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
				  unsigned long start, unsigned long end)
{
	if (mm_has_notifiers(mm))
		__mmu_notifier_invalidate_range(mm, start, end);
}

static inline void mmu_notifier_mm_init(struct mm_struct *mm)
{
	mm->mmu_notifier_mm = NULL;
}

static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
{
	if (mm_has_notifiers(mm))
		__mmu_notifier_mm_destroy(mm);
}


static inline void mmu_notifier_range_init(struct mmu_notifier_range *range,
					   struct mm_struct *mm,
					   unsigned long start,
					   unsigned long end)
{
	range->mm = mm;
	range->start = start;
	range->end = end;
}

#define ptep_clear_flush_young_notify(__vma, __address, __ptep)		\
({									\
	int __young;							\
	struct vm_area_struct *___vma = __vma;				\
	unsigned long ___address = __address;				\
	__young = ptep_clear_flush_young(___vma, ___address, __ptep);	\
	__young |= mmu_notifier_clear_flush_young(___vma->vm_mm,	\
						  ___address,		\
						  ___address +		\
							PAGE_SIZE);	\
	__young;							\
})

#define pmdp_clear_flush_young_notify(__vma, __address, __pmdp)		\
({									\
	int __young;							\
	struct vm_area_struct *___vma = __vma;				\
	unsigned long ___address = __address;				\
	__young = pmdp_clear_flush_young(___vma, ___address, __pmdp);	\
	__young |= mmu_notifier_clear_flush_young(___vma->vm_mm,	\
						  ___address,		\
						  ___address +		\
							PMD_SIZE);	\
	__young;							\
})

#define ptep_clear_young_notify(__vma, __address, __ptep)		\
({									\
	int __young;							\
	struct vm_area_struct *___vma = __vma;				\
	unsigned long ___address = __address;				\
	__young = ptep_test_and_clear_young(___vma, ___address, __ptep);\
	__young |= mmu_notifier_clear_young(___vma->vm_mm, ___address,	\
					    ___address + PAGE_SIZE);	\
	__young;							\
})

#define pmdp_clear_young_notify(__vma, __address, __pmdp)		\
({									\
	int __young;							\
	struct vm_area_struct *___vma = __vma;				\
	unsigned long ___address = __address;				\
	__young = pmdp_test_and_clear_young(___vma, ___address, __pmdp);\
	__young |= mmu_notifier_clear_young(___vma->vm_mm, ___address,	\
					    ___address + PMD_SIZE);	\
	__young;							\
})

#define	ptep_clear_flush_notify(__vma, __address, __ptep)		\
({									\
	unsigned long ___addr = __address & PAGE_MASK;			\
	struct mm_struct *___mm = (__vma)->vm_mm;			\
	pte_t ___pte;							\
									\
	___pte = ptep_clear_flush(__vma, __address, __ptep);		\
	mmu_notifier_invalidate_range(___mm, ___addr,			\
					___addr + PAGE_SIZE);		\
									\
	___pte;								\
})

#define pmdp_huge_clear_flush_notify(__vma, __haddr, __pmd)		\
({									\
	unsigned long ___haddr = __haddr & HPAGE_PMD_MASK;		\
	struct mm_struct *___mm = (__vma)->vm_mm;			\
	pmd_t ___pmd;							\
									\
	___pmd = pmdp_huge_clear_flush(__vma, __haddr, __pmd);		\
	mmu_notifier_invalidate_range(___mm, ___haddr,			\
				      ___haddr + HPAGE_PMD_SIZE);	\
									\
	___pmd;								\
})

#define pudp_huge_clear_flush_notify(__vma, __haddr, __pud)		\
({									\
	unsigned long ___haddr = __haddr & HPAGE_PUD_MASK;		\
	struct mm_struct *___mm = (__vma)->vm_mm;			\
	pud_t ___pud;							\
									\
	___pud = pudp_huge_clear_flush(__vma, __haddr, __pud);		\
	mmu_notifier_invalidate_range(___mm, ___haddr,			\
				      ___haddr + HPAGE_PUD_SIZE);	\
									\
	___pud;								\
})

/*
 * set_pte_at_notify() sets the pte _after_ running the notifier.
 * This is safe to start by updating the secondary MMUs, because the primary MMU
 * pte invalidate must have already happened with a ptep_clear_flush() before
 * set_pte_at_notify() has been invoked.  Updating the secondary MMUs first is
 * required when we change both the protection of the mapping from read-only to
 * read-write and the pfn (like during copy on write page faults). Otherwise the
 * old page would remain mapped readonly in the secondary MMUs after the new
 * page is already writable by some CPU through the primary MMU.
 */
#define set_pte_at_notify(__mm, __address, __ptep, __pte)		\
({									\
	struct mm_struct *___mm = __mm;					\
	unsigned long ___address = __address;				\
	pte_t ___pte = __pte;						\
									\
	mmu_notifier_change_pte(___mm, ___address, ___pte);		\
	set_pte_at(___mm, ___address, __ptep, ___pte);			\
})

extern void mmu_notifier_call_srcu(struct rcu_head *rcu,
				   void (*func)(struct rcu_head *rcu));

#else /* CONFIG_MMU_NOTIFIER */

struct mmu_notifier_range {
	unsigned long start;
	unsigned long end;
};

static inline void _mmu_notifier_range_init(struct mmu_notifier_range *range,
					    unsigned long start,
					    unsigned long end)
{
	range->start = start;
	range->end = end;
}

#define mmu_notifier_range_init(range, mm, start, end) \
	_mmu_notifier_range_init(range, start, end)


static inline int mm_has_notifiers(struct mm_struct *mm)
{
	return 0;
}

static inline void mmu_notifier_release(struct mm_struct *mm)
{
}

static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm,
					  unsigned long start,
					  unsigned long end)
{
	return 0;
}

static inline int mmu_notifier_test_young(struct mm_struct *mm,
					  unsigned long address)
{
	return 0;
}

static inline void mmu_notifier_change_pte(struct mm_struct *mm,
					   unsigned long address, pte_t pte)
{
}

static inline void
mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
{
}

static inline int
mmu_notifier_invalidate_range_start_nonblock(struct mmu_notifier_range *range)
{
	return 0;
}

static inline
void mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range)
{
}

static inline void
mmu_notifier_invalidate_range_only_end(struct mmu_notifier_range *range)
{
}

static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
				  unsigned long start, unsigned long end)
{
}

static inline void mmu_notifier_mm_init(struct mm_struct *mm)
{
}

static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
{
}

#define ptep_clear_flush_young_notify ptep_clear_flush_young
#define pmdp_clear_flush_young_notify pmdp_clear_flush_young
#define ptep_clear_young_notify ptep_test_and_clear_young
#define pmdp_clear_young_notify pmdp_test_and_clear_young
#define	ptep_clear_flush_notify ptep_clear_flush
#define pmdp_huge_clear_flush_notify pmdp_huge_clear_flush
#define pudp_huge_clear_flush_notify pudp_huge_clear_flush
#define set_pte_at_notify set_pte_at

#endif /* CONFIG_MMU_NOTIFIER */

#endif /* _LINUX_MMU_NOTIFIER_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
cddb8a5c AA	2	#ifndef _LINUX_MMU_NOTIFIER_H
	3	#define _LINUX_MMU_NOTIFIER_H
	4
	5	#include <linux/list.h>
	6	#include <linux/spinlock.h>
	7	#include <linux/mm_types.h>
21a92735	8	#include <linux/srcu.h>
cddb8a5c AA	9
	10	struct mmu_notifier;
	11	struct mmu_notifier_ops;
	12
	13	#ifdef CONFIG_MMU_NOTIFIER
	14
	15	/*
	16	* The mmu notifier_mm structure is allocated and installed in
	17	* mm->mmu_notifier_mm inside the mm_take_all_locks() protected
	18	* critical section and it's released only when mm_count reaches zero
	19	* in mmdrop().
	20	*/
	21	struct mmu_notifier_mm {
	22	/* all mmu notifiers registerd in this mm are queued in this list */
	23	struct hlist_head list;
	24	/* to serialize the list modifications and hlist_unhashed */
	25	spinlock_t lock;
	26	};
	27
5d6527a7 JG	28	struct mmu_notifier_range {
	29	struct mm_struct *mm;
	30	unsigned long start;
	31	unsigned long end;
	32	bool blockable;
	33	};
	34
cddb8a5c AA	35	struct mmu_notifier_ops {
	36	/*
	37	* Called either by mmu_notifier_unregister or when the mm is
	38	* being destroyed by exit_mmap, always before all pages are
	39	* freed. This can run concurrently with other mmu notifier
	40	* methods (the ones invoked outside the mm context) and it
	41	* should tear down all secondary mmu mappings and freeze the
	42	* secondary mmu. If this method isn't implemented you've to
	43	* be sure that nothing could possibly write to the pages
	44	* through the secondary mmu by the time the last thread with
	45	* tsk->mm == mm exits.
	46	*
	47	* As side note: the pages freed after ->release returns could
	48	* be immediately reallocated by the gart at an alias physical
	49	* address with a different cache model, so if ->release isn't
	50	* implemented because all _software_ driven memory accesses
	51	* through the secondary mmu are terminated by the time the
	52	* last thread of this mm quits, you've also to be sure that
	53	* speculative _hardware_ operations can't allocate dirty
	54	* cachelines in the cpu that could not be snooped and made
	55	* coherent with the other read and write operations happening
	56	* through the gart alias address, so leading to memory
	57	* corruption.
	58	*/
	59	void (release)(struct mmu_notifier mn,
	60	struct mm_struct *mm);
	61
	62	/*
	63	* clear_flush_young is called after the VM is
	64	* test-and-clearing the young/accessed bitflag in the
	65	* pte. This way the VM will provide proper aging to the
	66	* accesses to the page through the secondary MMUs and not
	67	* only to the ones through the Linux pte.
57128468 ALC	68	* Start-end is necessary in case the secondary MMU is mapping the page
57128468 ALC	69	* at a smaller granularity than the primary MMU.
cddb8a5c AA	70	*/
	71	int (clear_flush_young)(struct mmu_notifier mn,
	72	struct mm_struct *mm,
57128468 ALC	73	unsigned long start,
57128468 ALC	74	unsigned long end);
cddb8a5c	75
1d7715c6 VD	76	/*
	77	* clear_young is a lightweight version of clear_flush_young. Like the
	78	* latter, it is supposed to test-and-clear the young/accessed bitflag
	79	* in the secondary pte, but it may omit flushing the secondary tlb.
	80	*/
	81	int (clear_young)(struct mmu_notifier mn,
	82	struct mm_struct *mm,
	83	unsigned long start,
	84	unsigned long end);
	85
8ee53820 AA	86	/*
	87	* test_young is called to check the young/accessed bitflag in
	88	* the secondary pte. This is used to know if the page is
	89	* frequently used without actually clearing the flag or tearing
	90	* down the secondary mapping on the page.
	91	*/
	92	int (test_young)(struct mmu_notifier mn,
	93	struct mm_struct *mm,
	94	unsigned long address);
	95
828502d3 IE	96	/*
	97	* change_pte is called in cases that pte mapping to page is changed:
	98	* for example, when ksm remaps pte to point to a new shared page.
	99	*/
	100	void (change_pte)(struct mmu_notifier mn,
	101	struct mm_struct *mm,
	102	unsigned long address,
	103	pte_t pte);
	104
cddb8a5c AA	105	/*
	106	* invalidate_range_start() and invalidate_range_end() must be
	107	* paired and are called only when the mmap_sem and/or the
0f0a327f JR	108	* locks protecting the reverse maps are held. If the subsystem
	109	* can't guarantee that no additional references are taken to
	110	* the pages in the range, it has to implement the
	111	* invalidate_range() notifier to remove any references taken
	112	* after invalidate_range_start().
cddb8a5c AA	113	*
	114	* Invalidation of multiple concurrent ranges may be
	115	* optionally permitted by the driver. Either way the
	116	* establishment of sptes is forbidden in the range passed to
	117	* invalidate_range_begin/end for the whole duration of the
	118	* invalidate_range_begin/end critical section.
	119	*
	120	* invalidate_range_start() is called when all pages in the
	121	* range are still mapped and have at least a refcount of one.
	122	*
	123	* invalidate_range_end() is called when all pages in the
	124	* range have been unmapped and the pages have been freed by
	125	* the VM.
	126	*
	127	* The VM will remove the page table entries and potentially
	128	* the page between invalidate_range_start() and
	129	* invalidate_range_end(). If the page must not be freed
	130	* because of pending I/O or other circumstances then the
	131	* invalidate_range_start() callback (or the initial mapping
	132	* by the driver) must make sure that the refcount is kept
	133	* elevated.
	134	*
	135	* If the driver increases the refcount when the pages are
	136	* initially mapped into an address space then either
	137	* invalidate_range_start() or invalidate_range_end() may
	138	* decrease the refcount. If the refcount is decreased on
	139	* invalidate_range_start() then the VM can free pages as page
	140	* table entries are removed. If the refcount is only
	141	* droppped on invalidate_range_end() then the driver itself
	142	* will drop the last refcount but it must take care to flush
	143	* any secondary tlb before doing the final free on the
	144	* page. Pages will no longer be referenced by the linux
	145	* address space but may still be referenced by sptes until
	146	* the last refcount is dropped.
5ff7091f	147	*
93065ac7 MH	148	* If blockable argument is set to false then the callback cannot
93065ac7 MH	149	* sleep and has to return with -EAGAIN. 0 should be returned
33490af3 MH	150	* otherwise. Please note that if invalidate_range_start approves
	151	* a non-blocking behavior then the same applies to
	152	* invalidate_range_end.
93065ac7	153	*
cddb8a5c	154	*/
93065ac7	155	int (invalidate_range_start)(struct mmu_notifier mn,
5d6527a7	156	const struct mmu_notifier_range *range);
cddb8a5c	157	void (invalidate_range_end)(struct mmu_notifier mn,
5d6527a7	158	const struct mmu_notifier_range *range);
0f0a327f JR	159
	160	/*
	161	* invalidate_range() is either called between
	162	* invalidate_range_start() and invalidate_range_end() when the
	163	* VM has to free pages that where unmapped, but before the
	164	* pages are actually freed, or outside of _start()/_end() when
	165	* a (remote) TLB is necessary.
	166	*
	167	* If invalidate_range() is used to manage a non-CPU TLB with
	168	* shared page-tables, it not necessary to implement the
	169	* invalidate_range_start()/end() notifiers, as
	170	* invalidate_range() alread catches the points in time when an
0f10851e	171	* external TLB range needs to be flushed. For more in depth
ad56b738	172	* discussion on this see Documentation/vm/mmu_notifier.rst
0f0a327f	173	*
0f0a327f JR	174	* Note that this function might be called with just a sub-range
	175	* of what was passed to invalidate_range_start()/end(), if
	176	* called between those functions.
	177	*/
	178	void (invalidate_range)(struct mmu_notifier mn, struct mm_struct *mm,
	179	unsigned long start, unsigned long end);
cddb8a5c AA	180	};
	181
	182	/*
	183	* The notifier chains are protected by mmap_sem and/or the reverse map
	184	* semaphores. Notifier chains are only changed when all reverse maps and
	185	* the mmap_sem locks are taken.
	186	*
	187	* Therefore notifier chains can only be traversed when either
	188	*
	189	* 1. mmap_sem is held.
c8c06efa	190	* 2. One of the reverse map locks is held (i_mmap_rwsem or anon_vma->rwsem).
cddb8a5c AA	191	* 3. No other concurrent thread can access the list (release)
	192	*/
	193	struct mmu_notifier {
	194	struct hlist_node hlist;
	195	const struct mmu_notifier_ops *ops;
	196	};
	197
	198	static inline int mm_has_notifiers(struct mm_struct *mm)
	199	{
	200	return unlikely(mm->mmu_notifier_mm);
	201	}
	202
	203	extern int mmu_notifier_register(struct mmu_notifier *mn,
	204	struct mm_struct *mm);
	205	extern int __mmu_notifier_register(struct mmu_notifier *mn,
	206	struct mm_struct *mm);
	207	extern void mmu_notifier_unregister(struct mmu_notifier *mn,
	208	struct mm_struct *mm);
b972216e PZ	209	extern void mmu_notifier_unregister_no_release(struct mmu_notifier *mn,
b972216e PZ	210	struct mm_struct *mm);
cddb8a5c AA	211	extern void __mmu_notifier_mm_destroy(struct mm_struct *mm);
	212	extern void __mmu_notifier_release(struct mm_struct *mm);
	213	extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
57128468 ALC	214	unsigned long start,
57128468 ALC	215	unsigned long end);
1d7715c6 VD	216	extern int __mmu_notifier_clear_young(struct mm_struct *mm,
	217	unsigned long start,
	218	unsigned long end);
8ee53820 AA	219	extern int __mmu_notifier_test_young(struct mm_struct *mm,
8ee53820 AA	220	unsigned long address);
828502d3 IE	221	extern void __mmu_notifier_change_pte(struct mm_struct *mm,
828502d3 IE	222	unsigned long address, pte_t pte);
ac46d4f3 JG	223	extern int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *r);
ac46d4f3 JG	224	extern void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *r,
4645b9fe	225	bool only_end);
0f0a327f JR	226	extern void __mmu_notifier_invalidate_range(struct mm_struct *mm,
0f0a327f JR	227	unsigned long start, unsigned long end);
cddb8a5c AA	228
	229	static inline void mmu_notifier_release(struct mm_struct *mm)
	230	{
	231	if (mm_has_notifiers(mm))
	232	__mmu_notifier_release(mm);
	233	}
	234
	235	static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm,
57128468 ALC	236	unsigned long start,
57128468 ALC	237	unsigned long end)
cddb8a5c AA	238	{
cddb8a5c AA	239	if (mm_has_notifiers(mm))
57128468	240	return __mmu_notifier_clear_flush_young(mm, start, end);
cddb8a5c AA	241	return 0;
	242	}
	243
1d7715c6 VD	244	static inline int mmu_notifier_clear_young(struct mm_struct *mm,
	245	unsigned long start,
	246	unsigned long end)
	247	{
	248	if (mm_has_notifiers(mm))
	249	return __mmu_notifier_clear_young(mm, start, end);
	250	return 0;
	251	}
	252
8ee53820 AA	253	static inline int mmu_notifier_test_young(struct mm_struct *mm,
	254	unsigned long address)
	255	{
	256	if (mm_has_notifiers(mm))
	257	return __mmu_notifier_test_young(mm, address);
	258	return 0;
	259	}
	260
828502d3 IE	261	static inline void mmu_notifier_change_pte(struct mm_struct *mm,
	262	unsigned long address, pte_t pte)
	263	{
	264	if (mm_has_notifiers(mm))
	265	__mmu_notifier_change_pte(mm, address, pte);
	266	}
	267
ac46d4f3 JG	268	static inline void
ac46d4f3 JG	269	mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
cddb8a5c	270	{
ac46d4f3 JG	271	if (mm_has_notifiers(range->mm)) {
	272	range->blockable = true;
	273	__mmu_notifier_invalidate_range_start(range);
	274	}
93065ac7 MH	275	}
93065ac7 MH	276
ac46d4f3 JG	277	static inline int
ac46d4f3 JG	278	mmu_notifier_invalidate_range_start_nonblock(struct mmu_notifier_range *range)
93065ac7	279	{
ac46d4f3 JG	280	if (mm_has_notifiers(range->mm)) {
	281	range->blockable = false;
	282	return __mmu_notifier_invalidate_range_start(range);
	283	}
93065ac7	284	return 0;
cddb8a5c AA	285	}
cddb8a5c AA	286
ac46d4f3 JG	287	static inline void
ac46d4f3 JG	288	mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range)
cddb8a5c	289	{
ac46d4f3 JG	290	if (mm_has_notifiers(range->mm))
ac46d4f3 JG	291	__mmu_notifier_invalidate_range_end(range, false);
4645b9fe JG	292	}
4645b9fe JG	293
ac46d4f3 JG	294	static inline void
ac46d4f3 JG	295	mmu_notifier_invalidate_range_only_end(struct mmu_notifier_range *range)
4645b9fe	296	{
ac46d4f3 JG	297	if (mm_has_notifiers(range->mm))
ac46d4f3 JG	298	__mmu_notifier_invalidate_range_end(range, true);
cddb8a5c AA	299	}
cddb8a5c AA	300
1897bdc4 JR	301	static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
	302	unsigned long start, unsigned long end)
	303	{
0f0a327f JR	304	if (mm_has_notifiers(mm))
0f0a327f JR	305	__mmu_notifier_invalidate_range(mm, start, end);
1897bdc4 JR	306	}
1897bdc4 JR	307
cddb8a5c AA	308	static inline void mmu_notifier_mm_init(struct mm_struct *mm)
	309	{
	310	mm->mmu_notifier_mm = NULL;
	311	}
	312
	313	static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
	314	{
	315	if (mm_has_notifiers(mm))
	316	__mmu_notifier_mm_destroy(mm);
	317	}
	318
ac46d4f3 JG	319
	320	static inline void mmu_notifier_range_init(struct mmu_notifier_range *range,
	321	struct mm_struct *mm,
	322	unsigned long start,
	323	unsigned long end)
	324	{
	325	range->mm = mm;
	326	range->start = start;
	327	range->end = end;
	328	}
	329
cddb8a5c AA	330	#define ptep_clear_flush_young_notify(__vma, __address, __ptep) \
	331	({ \
	332	int __young; \
	333	struct vm_area_struct *___vma = __vma; \
	334	unsigned long ___address = __address; \
	335	__young = ptep_clear_flush_young(___vma, ___address, __ptep); \
	336	__young \|= mmu_notifier_clear_flush_young(___vma->vm_mm, \
57128468 ALC	337	___address, \
	338	___address + \
	339	PAGE_SIZE); \
cddb8a5c AA	340	__young; \
	341	})
	342
91a4ee26 AA	343	#define pmdp_clear_flush_young_notify(__vma, __address, __pmdp) \
	344	({ \
	345	int __young; \
	346	struct vm_area_struct *___vma = __vma; \
	347	unsigned long ___address = __address; \
	348	__young = pmdp_clear_flush_young(___vma, ___address, __pmdp); \
	349	__young \|= mmu_notifier_clear_flush_young(___vma->vm_mm, \
57128468 ALC	350	___address, \
	351	___address + \
	352	PMD_SIZE); \
91a4ee26 AA	353	__young; \
	354	})
	355
1d7715c6 VD	356	#define ptep_clear_young_notify(__vma, __address, __ptep) \
	357	({ \
	358	int __young; \
	359	struct vm_area_struct *___vma = __vma; \
	360	unsigned long ___address = __address; \
	361	__young = ptep_test_and_clear_young(___vma, ___address, __ptep);\
	362	__young \|= mmu_notifier_clear_young(___vma->vm_mm, ___address, \
	363	___address + PAGE_SIZE); \
	364	__young; \
	365	})
	366
	367	#define pmdp_clear_young_notify(__vma, __address, __pmdp) \
	368	({ \
	369	int __young; \
	370	struct vm_area_struct *___vma = __vma; \
	371	unsigned long ___address = __address; \
	372	__young = pmdp_test_and_clear_young(___vma, ___address, __pmdp);\
	373	__young \|= mmu_notifier_clear_young(___vma->vm_mm, ___address, \
	374	___address + PMD_SIZE); \
	375	__young; \
	376	})
	377
34ee645e JR	378	#define ptep_clear_flush_notify(__vma, __address, __ptep) \
	379	({ \
	380	unsigned long ___addr = __address & PAGE_MASK; \
	381	struct mm_struct *___mm = (__vma)->vm_mm; \
	382	pte_t ___pte; \
	383	\
	384	___pte = ptep_clear_flush(__vma, __address, __ptep); \
	385	mmu_notifier_invalidate_range(___mm, ___addr, \
	386	___addr + PAGE_SIZE); \
	387	\
	388	___pte; \
	389	})
	390
8809aa2d	391	#define pmdp_huge_clear_flush_notify(__vma, __haddr, __pmd) \
34ee645e JR	392	({ \
	393	unsigned long ___haddr = __haddr & HPAGE_PMD_MASK; \
	394	struct mm_struct *___mm = (__vma)->vm_mm; \
	395	pmd_t ___pmd; \
	396	\
8809aa2d	397	___pmd = pmdp_huge_clear_flush(__vma, __haddr, __pmd); \
34ee645e JR	398	mmu_notifier_invalidate_range(___mm, ___haddr, \
	399	___haddr + HPAGE_PMD_SIZE); \
	400	\
	401	___pmd; \
	402	})
	403
a00cc7d9 MW	404	#define pudp_huge_clear_flush_notify(__vma, __haddr, __pud) \
	405	({ \
	406	unsigned long ___haddr = __haddr & HPAGE_PUD_MASK; \
	407	struct mm_struct *___mm = (__vma)->vm_mm; \
	408	pud_t ___pud; \
	409	\
	410	___pud = pudp_huge_clear_flush(__vma, __haddr, __pud); \
	411	mmu_notifier_invalidate_range(___mm, ___haddr, \
	412	___haddr + HPAGE_PUD_SIZE); \
	413	\
	414	___pud; \
	415	})
	416
48af0d7c XG	417	/*
	418	* set_pte_at_notify() sets the pte _after_ running the notifier.
	419	* This is safe to start by updating the secondary MMUs, because the primary MMU
	420	* pte invalidate must have already happened with a ptep_clear_flush() before
	421	* set_pte_at_notify() has been invoked. Updating the secondary MMUs first is
	422	* required when we change both the protection of the mapping from read-only to
	423	* read-write and the pfn (like during copy on write page faults). Otherwise the
	424	* old page would remain mapped readonly in the secondary MMUs after the new
	425	* page is already writable by some CPU through the primary MMU.
	426	*/
828502d3 IE	427	#define set_pte_at_notify(__mm, __address, __ptep, __pte) \
	428	({ \
	429	struct mm_struct *___mm = __mm; \
	430	unsigned long ___address = __address; \
	431	pte_t ___pte = __pte; \
	432	\
828502d3	433	mmu_notifier_change_pte(___mm, ___address, ___pte); \
48af0d7c	434	set_pte_at(___mm, ___address, __ptep, ___pte); \
828502d3 IE	435	})
828502d3 IE	436
b972216e PZ	437	extern void mmu_notifier_call_srcu(struct rcu_head *rcu,
b972216e PZ	438	void (func)(struct rcu_head rcu));
b972216e	439
cddb8a5c AA	440	#else /* CONFIG_MMU_NOTIFIER */
cddb8a5c AA	441
ac46d4f3 JG	442	struct mmu_notifier_range {
	443	unsigned long start;
	444	unsigned long end;
	445	};
	446
	447	static inline void _mmu_notifier_range_init(struct mmu_notifier_range *range,
	448	unsigned long start,
	449	unsigned long end)
	450	{
	451	range->start = start;
	452	range->end = end;
	453	}
	454
	455	#define mmu_notifier_range_init(range, mm, start, end) \
	456	_mmu_notifier_range_init(range, start, end)
	457
	458
4d4bbd85 MH	459	static inline int mm_has_notifiers(struct mm_struct *mm)
	460	{
	461	return 0;
	462	}
	463
cddb8a5c AA	464	static inline void mmu_notifier_release(struct mm_struct *mm)
	465	{
	466	}
	467
	468	static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm,
57128468 ALC	469	unsigned long start,
57128468 ALC	470	unsigned long end)
8ee53820 AA	471	{
	472	return 0;
	473	}
	474
	475	static inline int mmu_notifier_test_young(struct mm_struct *mm,
	476	unsigned long address)
cddb8a5c AA	477	{
	478	return 0;
	479	}
	480
828502d3 IE	481	static inline void mmu_notifier_change_pte(struct mm_struct *mm,
	482	unsigned long address, pte_t pte)
	483	{
	484	}
	485
ac46d4f3 JG	486	static inline void
ac46d4f3 JG	487	mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
cddb8a5c AA	488	{
	489	}
	490
ac46d4f3 JG	491	static inline int
ac46d4f3 JG	492	mmu_notifier_invalidate_range_start_nonblock(struct mmu_notifier_range *range)
93065ac7 MH	493	{
	494	return 0;
	495	}
	496
ac46d4f3 JG	497	static inline
ac46d4f3 JG	498	void mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range)
cddb8a5c AA	499	{
	500	}
	501
ac46d4f3 JG	502	static inline void
ac46d4f3 JG	503	mmu_notifier_invalidate_range_only_end(struct mmu_notifier_range *range)
4645b9fe JG	504	{
	505	}
	506
1897bdc4 JR	507	static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
	508	unsigned long start, unsigned long end)
	509	{
	510	}
	511
cddb8a5c AA	512	static inline void mmu_notifier_mm_init(struct mm_struct *mm)
	513	{
	514	}
	515
	516	static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
	517	{
	518	}
	519
	520	#define ptep_clear_flush_young_notify ptep_clear_flush_young
91a4ee26	521	#define pmdp_clear_flush_young_notify pmdp_clear_flush_young
33c3fc71 VD	522	#define ptep_clear_young_notify ptep_test_and_clear_young
33c3fc71 VD	523	#define pmdp_clear_young_notify pmdp_test_and_clear_young
34ee645e	524	#define ptep_clear_flush_notify ptep_clear_flush
8809aa2d	525	#define pmdp_huge_clear_flush_notify pmdp_huge_clear_flush
a00cc7d9	526	#define pudp_huge_clear_flush_notify pudp_huge_clear_flush
828502d3	527	#define set_pte_at_notify set_pte_at
cddb8a5c AA	528
	529	#endif /* CONFIG_MMU_NOTIFIER */
	530
	531	#endif /* _LINUX_MMU_NOTIFIER_H */