Commit | Line | Data |
---|---|---|
9740ca4e ML |
1 | #ifndef _LINUX_MMAP_LOCK_H |
2 | #define _LINUX_MMAP_LOCK_H | |
3 | ||
2b5067a8 AR |
4 | #include <linux/lockdep.h> |
5 | #include <linux/mm_types.h> | |
42fc5414 | 6 | #include <linux/mmdebug.h> |
2b5067a8 AR |
7 | #include <linux/rwsem.h> |
8 | #include <linux/tracepoint-defs.h> | |
9 | #include <linux/types.h> | |
42fc5414 | 10 | |
14c3656b | 11 | #define MMAP_LOCK_INITIALIZER(name) \ |
da1c55f1 | 12 | .mmap_lock = __RWSEM_INITIALIZER((name).mmap_lock), |
14c3656b | 13 | |
2b5067a8 AR |
14 | DECLARE_TRACEPOINT(mmap_lock_start_locking); |
15 | DECLARE_TRACEPOINT(mmap_lock_acquire_returned); | |
16 | DECLARE_TRACEPOINT(mmap_lock_released); | |
17 | ||
18 | #ifdef CONFIG_TRACING | |
19 | ||
20 | void __mmap_lock_do_trace_start_locking(struct mm_struct *mm, bool write); | |
21 | void __mmap_lock_do_trace_acquire_returned(struct mm_struct *mm, bool write, | |
22 | bool success); | |
23 | void __mmap_lock_do_trace_released(struct mm_struct *mm, bool write); | |
24 | ||
25 | static inline void __mmap_lock_trace_start_locking(struct mm_struct *mm, | |
26 | bool write) | |
27 | { | |
28 | if (tracepoint_enabled(mmap_lock_start_locking)) | |
29 | __mmap_lock_do_trace_start_locking(mm, write); | |
30 | } | |
31 | ||
32 | static inline void __mmap_lock_trace_acquire_returned(struct mm_struct *mm, | |
33 | bool write, bool success) | |
34 | { | |
35 | if (tracepoint_enabled(mmap_lock_acquire_returned)) | |
36 | __mmap_lock_do_trace_acquire_returned(mm, write, success); | |
37 | } | |
38 | ||
39 | static inline void __mmap_lock_trace_released(struct mm_struct *mm, bool write) | |
40 | { | |
41 | if (tracepoint_enabled(mmap_lock_released)) | |
42 | __mmap_lock_do_trace_released(mm, write); | |
43 | } | |
44 | ||
45 | #else /* !CONFIG_TRACING */ | |
46 | ||
47 | static inline void __mmap_lock_trace_start_locking(struct mm_struct *mm, | |
48 | bool write) | |
49 | { | |
50 | } | |
51 | ||
52 | static inline void __mmap_lock_trace_acquire_returned(struct mm_struct *mm, | |
53 | bool write, bool success) | |
54 | { | |
55 | } | |
56 | ||
57 | static inline void __mmap_lock_trace_released(struct mm_struct *mm, bool write) | |
58 | { | |
59 | } | |
60 | ||
61 | #endif /* CONFIG_TRACING */ | |
62 | ||
ba168b52 | 63 | static inline void mmap_assert_locked(const struct mm_struct *mm) |
438b6e12 | 64 | { |
ba168b52 | 65 | rwsem_assert_held(&mm->mmap_lock); |
438b6e12 SB |
66 | } |
67 | ||
ba168b52 | 68 | static inline void mmap_assert_write_locked(const struct mm_struct *mm) |
438b6e12 | 69 | { |
ba168b52 | 70 | rwsem_assert_held_write(&mm->mmap_lock); |
438b6e12 SB |
71 | } |
72 | ||
5e31275c | 73 | #ifdef CONFIG_PER_VMA_LOCK |
90717566 JH |
74 | /* |
75 | * Drop all currently-held per-VMA locks. | |
76 | * This is called from the mmap_lock implementation directly before releasing | |
77 | * a write-locked mmap_lock (or downgrading it to read-locked). | |
78 | * This should normally NOT be called manually from other places. | |
79 | * If you want to call this manually anyway, keep in mind that this will release | |
80 | * *all* VMA write locks, including ones from further up the stack. | |
81 | */ | |
5e31275c SB |
82 | static inline void vma_end_write_all(struct mm_struct *mm) |
83 | { | |
84 | mmap_assert_write_locked(mm); | |
b1f02b95 JH |
85 | /* |
86 | * Nobody can concurrently modify mm->mm_lock_seq due to exclusive | |
87 | * mmap_lock being held. | |
88 | * We need RELEASE semantics here to ensure that preceding stores into | |
89 | * the VMA take effect before we unlock it with this store. | |
90 | * Pairs with ACQUIRE semantics in vma_start_read(). | |
91 | */ | |
92 | smp_store_release(&mm->mm_lock_seq, mm->mm_lock_seq + 1); | |
5e31275c SB |
93 | } |
94 | #else | |
95 | static inline void vma_end_write_all(struct mm_struct *mm) {} | |
96 | #endif | |
97 | ||
9740ca4e ML |
98 | static inline void mmap_init_lock(struct mm_struct *mm) |
99 | { | |
da1c55f1 | 100 | init_rwsem(&mm->mmap_lock); |
9740ca4e ML |
101 | } |
102 | ||
103 | static inline void mmap_write_lock(struct mm_struct *mm) | |
104 | { | |
2b5067a8 | 105 | __mmap_lock_trace_start_locking(mm, true); |
da1c55f1 | 106 | down_write(&mm->mmap_lock); |
2b5067a8 | 107 | __mmap_lock_trace_acquire_returned(mm, true, true); |
9740ca4e ML |
108 | } |
109 | ||
aaa2cc56 ML |
110 | static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass) |
111 | { | |
2b5067a8 | 112 | __mmap_lock_trace_start_locking(mm, true); |
da1c55f1 | 113 | down_write_nested(&mm->mmap_lock, subclass); |
2b5067a8 | 114 | __mmap_lock_trace_acquire_returned(mm, true, true); |
aaa2cc56 ML |
115 | } |
116 | ||
9740ca4e ML |
117 | static inline int mmap_write_lock_killable(struct mm_struct *mm) |
118 | { | |
2b5067a8 AR |
119 | int ret; |
120 | ||
121 | __mmap_lock_trace_start_locking(mm, true); | |
122 | ret = down_write_killable(&mm->mmap_lock); | |
123 | __mmap_lock_trace_acquire_returned(mm, true, ret == 0); | |
124 | return ret; | |
9740ca4e ML |
125 | } |
126 | ||
9740ca4e ML |
127 | static inline void mmap_write_unlock(struct mm_struct *mm) |
128 | { | |
2b5067a8 | 129 | __mmap_lock_trace_released(mm, true); |
5e31275c | 130 | vma_end_write_all(mm); |
10994316 | 131 | up_write(&mm->mmap_lock); |
9740ca4e ML |
132 | } |
133 | ||
134 | static inline void mmap_write_downgrade(struct mm_struct *mm) | |
135 | { | |
2b5067a8 | 136 | __mmap_lock_trace_acquire_returned(mm, false, true); |
5e31275c | 137 | vma_end_write_all(mm); |
10994316 | 138 | downgrade_write(&mm->mmap_lock); |
9740ca4e ML |
139 | } |
140 | ||
141 | static inline void mmap_read_lock(struct mm_struct *mm) | |
142 | { | |
2b5067a8 | 143 | __mmap_lock_trace_start_locking(mm, false); |
da1c55f1 | 144 | down_read(&mm->mmap_lock); |
2b5067a8 | 145 | __mmap_lock_trace_acquire_returned(mm, false, true); |
9740ca4e ML |
146 | } |
147 | ||
148 | static inline int mmap_read_lock_killable(struct mm_struct *mm) | |
149 | { | |
2b5067a8 AR |
150 | int ret; |
151 | ||
152 | __mmap_lock_trace_start_locking(mm, false); | |
153 | ret = down_read_killable(&mm->mmap_lock); | |
154 | __mmap_lock_trace_acquire_returned(mm, false, ret == 0); | |
155 | return ret; | |
9740ca4e ML |
156 | } |
157 | ||
158 | static inline bool mmap_read_trylock(struct mm_struct *mm) | |
159 | { | |
2b5067a8 AR |
160 | bool ret; |
161 | ||
162 | __mmap_lock_trace_start_locking(mm, false); | |
163 | ret = down_read_trylock(&mm->mmap_lock) != 0; | |
164 | __mmap_lock_trace_acquire_returned(mm, false, ret); | |
165 | return ret; | |
9740ca4e ML |
166 | } |
167 | ||
168 | static inline void mmap_read_unlock(struct mm_struct *mm) | |
169 | { | |
2b5067a8 | 170 | __mmap_lock_trace_released(mm, false); |
10994316 | 171 | up_read(&mm->mmap_lock); |
9740ca4e ML |
172 | } |
173 | ||
0cc55a02 ML |
174 | static inline void mmap_read_unlock_non_owner(struct mm_struct *mm) |
175 | { | |
2b5067a8 | 176 | __mmap_lock_trace_released(mm, false); |
10994316 | 177 | up_read_non_owner(&mm->mmap_lock); |
0cc55a02 ML |
178 | } |
179 | ||
07e5bfe6 CC |
180 | static inline int mmap_lock_is_contended(struct mm_struct *mm) |
181 | { | |
182 | return rwsem_is_contended(&mm->mmap_lock); | |
183 | } | |
184 | ||
9740ca4e | 185 | #endif /* _LINUX_MMAP_LOCK_H */ |