Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
6e84f315 IM |
2 | #ifndef _LINUX_SCHED_MM_H |
3 | #define _LINUX_SCHED_MM_H | |
4 | ||
b8d6d80b IM |
5 | #include <linux/kernel.h> |
6 | #include <linux/atomic.h> | |
6e84f315 | 7 | #include <linux/sched.h> |
589ee628 | 8 | #include <linux/mm_types.h> |
fd771233 | 9 | #include <linux/gfp.h> |
6e84f315 | 10 | |
68e21be2 IM |
11 | /* |
12 | * Routines for handling mm_structs | |
13 | */ | |
14 | extern struct mm_struct * mm_alloc(void); | |
15 | ||
16 | /** | |
17 | * mmgrab() - Pin a &struct mm_struct. | |
18 | * @mm: The &struct mm_struct to pin. | |
19 | * | |
20 | * Make sure that @mm will not get freed even after the owning task | |
21 | * exits. This doesn't guarantee that the associated address space | |
22 | * will still exist later on and mmget_not_zero() has to be used before | |
23 | * accessing it. | |
24 | * | |
25 | * This is a preferred way to to pin @mm for a longer/unbounded amount | |
26 | * of time. | |
27 | * | |
28 | * Use mmdrop() to release the reference acquired by mmgrab(). | |
29 | * | |
30 | * See also <Documentation/vm/active_mm.txt> for an in-depth explanation | |
31 | * of &mm_struct.mm_count vs &mm_struct.mm_users. | |
32 | */ | |
33 | static inline void mmgrab(struct mm_struct *mm) | |
34 | { | |
35 | atomic_inc(&mm->mm_count); | |
36 | } | |
37 | ||
38 | /* mmdrop drops the mm and the page tables */ | |
39 | extern void __mmdrop(struct mm_struct *); | |
40 | static inline void mmdrop(struct mm_struct *mm) | |
41 | { | |
306e0604 MD |
42 | /* |
43 | * The implicit full barrier implied by atomic_dec_and_test() is | |
44 | * required by the membarrier system call before returning to | |
45 | * user-space, after storing to rq->curr. | |
46 | */ | |
68e21be2 IM |
47 | if (unlikely(atomic_dec_and_test(&mm->mm_count))) |
48 | __mmdrop(mm); | |
49 | } | |
50 | ||
51 | static inline void mmdrop_async_fn(struct work_struct *work) | |
52 | { | |
53 | struct mm_struct *mm = container_of(work, struct mm_struct, async_put_work); | |
54 | __mmdrop(mm); | |
55 | } | |
56 | ||
57 | static inline void mmdrop_async(struct mm_struct *mm) | |
58 | { | |
59 | if (unlikely(atomic_dec_and_test(&mm->mm_count))) { | |
60 | INIT_WORK(&mm->async_put_work, mmdrop_async_fn); | |
61 | schedule_work(&mm->async_put_work); | |
62 | } | |
63 | } | |
64 | ||
65 | /** | |
66 | * mmget() - Pin the address space associated with a &struct mm_struct. | |
67 | * @mm: The address space to pin. | |
68 | * | |
69 | * Make sure that the address space of the given &struct mm_struct doesn't | |
70 | * go away. This does not protect against parts of the address space being | |
71 | * modified or freed, however. | |
72 | * | |
73 | * Never use this function to pin this address space for an | |
74 | * unbounded/indefinite amount of time. | |
75 | * | |
76 | * Use mmput() to release the reference acquired by mmget(). | |
77 | * | |
78 | * See also <Documentation/vm/active_mm.txt> for an in-depth explanation | |
79 | * of &mm_struct.mm_count vs &mm_struct.mm_users. | |
80 | */ | |
81 | static inline void mmget(struct mm_struct *mm) | |
82 | { | |
83 | atomic_inc(&mm->mm_users); | |
84 | } | |
85 | ||
86 | static inline bool mmget_not_zero(struct mm_struct *mm) | |
87 | { | |
88 | return atomic_inc_not_zero(&mm->mm_users); | |
89 | } | |
90 | ||
91 | /* mmput gets rid of the mappings and all user-space */ | |
92 | extern void mmput(struct mm_struct *); | |
a1b2289c SY |
93 | #ifdef CONFIG_MMU |
94 | /* same as above but performs the slow path from the async context. Can | |
95 | * be called from the atomic context as well | |
96 | */ | |
97 | void mmput_async(struct mm_struct *); | |
98 | #endif | |
68e21be2 IM |
99 | |
100 | /* Grab a reference to a task's mm, if it is not already going away */ | |
101 | extern struct mm_struct *get_task_mm(struct task_struct *task); | |
102 | /* | |
103 | * Grab a reference to a task's mm, if it is not already going away | |
104 | * and ptrace_may_access with the mode parameter passed to it | |
105 | * succeeds. | |
106 | */ | |
107 | extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode); | |
108 | /* Remove the current tasks stale references to the old mm_struct */ | |
109 | extern void mm_release(struct task_struct *, struct mm_struct *); | |
110 | ||
4240c8bf IM |
111 | #ifdef CONFIG_MEMCG |
112 | extern void mm_update_next_owner(struct mm_struct *mm); | |
113 | #else | |
114 | static inline void mm_update_next_owner(struct mm_struct *mm) | |
115 | { | |
116 | } | |
117 | #endif /* CONFIG_MEMCG */ | |
118 | ||
119 | #ifdef CONFIG_MMU | |
120 | extern void arch_pick_mmap_layout(struct mm_struct *mm); | |
121 | extern unsigned long | |
122 | arch_get_unmapped_area(struct file *, unsigned long, unsigned long, | |
123 | unsigned long, unsigned long); | |
124 | extern unsigned long | |
125 | arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, | |
126 | unsigned long len, unsigned long pgoff, | |
127 | unsigned long flags); | |
128 | #else | |
129 | static inline void arch_pick_mmap_layout(struct mm_struct *mm) {} | |
130 | #endif | |
131 | ||
d026ce79 IM |
132 | static inline bool in_vfork(struct task_struct *tsk) |
133 | { | |
134 | bool ret; | |
135 | ||
136 | /* | |
137 | * need RCU to access ->real_parent if CLONE_VM was used along with | |
138 | * CLONE_PARENT. | |
139 | * | |
140 | * We check real_parent->mm == tsk->mm because CLONE_VFORK does not | |
141 | * imply CLONE_VM | |
142 | * | |
143 | * CLONE_VFORK can be used with CLONE_PARENT/CLONE_THREAD and thus | |
144 | * ->real_parent is not necessarily the task doing vfork(), so in | |
145 | * theory we can't rely on task_lock() if we want to dereference it. | |
146 | * | |
147 | * And in this case we can't trust the real_parent->mm == tsk->mm | |
148 | * check, it can be false negative. But we do not care, if init or | |
149 | * another oom-unkillable task does this it should blame itself. | |
150 | */ | |
151 | rcu_read_lock(); | |
152 | ret = tsk->vfork_done && tsk->real_parent->mm == tsk->mm; | |
153 | rcu_read_unlock(); | |
154 | ||
155 | return ret; | |
156 | } | |
157 | ||
7dea19f9 MH |
158 | /* |
159 | * Applies per-task gfp context to the given allocation flags. | |
160 | * PF_MEMALLOC_NOIO implies GFP_NOIO | |
161 | * PF_MEMALLOC_NOFS implies GFP_NOFS | |
74444eda | 162 | */ |
7dea19f9 | 163 | static inline gfp_t current_gfp_context(gfp_t flags) |
74444eda | 164 | { |
7dea19f9 MH |
165 | /* |
166 | * NOIO implies both NOIO and NOFS and it is a weaker context | |
167 | * so always make sure it makes precendence | |
168 | */ | |
74444eda IM |
169 | if (unlikely(current->flags & PF_MEMALLOC_NOIO)) |
170 | flags &= ~(__GFP_IO | __GFP_FS); | |
7dea19f9 MH |
171 | else if (unlikely(current->flags & PF_MEMALLOC_NOFS)) |
172 | flags &= ~__GFP_FS; | |
74444eda IM |
173 | return flags; |
174 | } | |
175 | ||
d92a8cfc PZ |
176 | #ifdef CONFIG_LOCKDEP |
177 | extern void fs_reclaim_acquire(gfp_t gfp_mask); | |
178 | extern void fs_reclaim_release(gfp_t gfp_mask); | |
179 | #else | |
180 | static inline void fs_reclaim_acquire(gfp_t gfp_mask) { } | |
181 | static inline void fs_reclaim_release(gfp_t gfp_mask) { } | |
182 | #endif | |
183 | ||
74444eda IM |
184 | static inline unsigned int memalloc_noio_save(void) |
185 | { | |
186 | unsigned int flags = current->flags & PF_MEMALLOC_NOIO; | |
187 | current->flags |= PF_MEMALLOC_NOIO; | |
188 | return flags; | |
189 | } | |
190 | ||
191 | static inline void memalloc_noio_restore(unsigned int flags) | |
192 | { | |
193 | current->flags = (current->flags & ~PF_MEMALLOC_NOIO) | flags; | |
194 | } | |
195 | ||
7dea19f9 MH |
196 | static inline unsigned int memalloc_nofs_save(void) |
197 | { | |
198 | unsigned int flags = current->flags & PF_MEMALLOC_NOFS; | |
199 | current->flags |= PF_MEMALLOC_NOFS; | |
200 | return flags; | |
201 | } | |
202 | ||
203 | static inline void memalloc_nofs_restore(unsigned int flags) | |
204 | { | |
205 | current->flags = (current->flags & ~PF_MEMALLOC_NOFS) | flags; | |
206 | } | |
207 | ||
499118e9 VB |
208 | static inline unsigned int memalloc_noreclaim_save(void) |
209 | { | |
210 | unsigned int flags = current->flags & PF_MEMALLOC; | |
211 | current->flags |= PF_MEMALLOC; | |
212 | return flags; | |
213 | } | |
214 | ||
215 | static inline void memalloc_noreclaim_restore(unsigned int flags) | |
216 | { | |
217 | current->flags = (current->flags & ~PF_MEMALLOC) | flags; | |
218 | } | |
219 | ||
a961e409 MD |
220 | #ifdef CONFIG_MEMBARRIER |
221 | enum { | |
c5f58bd5 MD |
222 | MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY = (1U << 0), |
223 | MEMBARRIER_STATE_PRIVATE_EXPEDITED = (1U << 1), | |
224 | MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY = (1U << 2), | |
225 | MEMBARRIER_STATE_GLOBAL_EXPEDITED = (1U << 3), | |
a961e409 MD |
226 | }; |
227 | ||
3ccfebed MD |
228 | #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS |
229 | #include <asm/membarrier.h> | |
230 | #endif | |
231 | ||
a961e409 MD |
232 | static inline void membarrier_execve(struct task_struct *t) |
233 | { | |
234 | atomic_set(&t->mm->membarrier_state, 0); | |
235 | } | |
236 | #else | |
3ccfebed MD |
237 | #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS |
238 | static inline void membarrier_arch_switch_mm(struct mm_struct *prev, | |
239 | struct mm_struct *next, | |
240 | struct task_struct *tsk) | |
241 | { | |
242 | } | |
243 | #endif | |
a961e409 MD |
244 | static inline void membarrier_execve(struct task_struct *t) |
245 | { | |
246 | } | |
247 | #endif | |
248 | ||
6e84f315 | 249 | #endif /* _LINUX_SCHED_MM_H */ |