Commit | Line | Data |
---|---|---|
9740ca4e ML |
1 | #ifndef _LINUX_MMAP_LOCK_H |
2 | #define _LINUX_MMAP_LOCK_H | |
3 | ||
2b5067a8 AR |
4 | #include <linux/lockdep.h> |
5 | #include <linux/mm_types.h> | |
42fc5414 | 6 | #include <linux/mmdebug.h> |
2b5067a8 AR |
7 | #include <linux/rwsem.h> |
8 | #include <linux/tracepoint-defs.h> | |
9 | #include <linux/types.h> | |
42fc5414 | 10 | |
14c3656b | 11 | #define MMAP_LOCK_INITIALIZER(name) \ |
da1c55f1 | 12 | .mmap_lock = __RWSEM_INITIALIZER((name).mmap_lock), |
14c3656b | 13 | |
2b5067a8 AR |
14 | DECLARE_TRACEPOINT(mmap_lock_start_locking); |
15 | DECLARE_TRACEPOINT(mmap_lock_acquire_returned); | |
16 | DECLARE_TRACEPOINT(mmap_lock_released); | |
17 | ||
18 | #ifdef CONFIG_TRACING | |
19 | ||
20 | void __mmap_lock_do_trace_start_locking(struct mm_struct *mm, bool write); | |
21 | void __mmap_lock_do_trace_acquire_returned(struct mm_struct *mm, bool write, | |
22 | bool success); | |
23 | void __mmap_lock_do_trace_released(struct mm_struct *mm, bool write); | |
24 | ||
25 | static inline void __mmap_lock_trace_start_locking(struct mm_struct *mm, | |
26 | bool write) | |
27 | { | |
28 | if (tracepoint_enabled(mmap_lock_start_locking)) | |
29 | __mmap_lock_do_trace_start_locking(mm, write); | |
30 | } | |
31 | ||
32 | static inline void __mmap_lock_trace_acquire_returned(struct mm_struct *mm, | |
33 | bool write, bool success) | |
34 | { | |
35 | if (tracepoint_enabled(mmap_lock_acquire_returned)) | |
36 | __mmap_lock_do_trace_acquire_returned(mm, write, success); | |
37 | } | |
38 | ||
39 | static inline void __mmap_lock_trace_released(struct mm_struct *mm, bool write) | |
40 | { | |
41 | if (tracepoint_enabled(mmap_lock_released)) | |
42 | __mmap_lock_do_trace_released(mm, write); | |
43 | } | |
44 | ||
45 | #else /* !CONFIG_TRACING */ | |
46 | ||
47 | static inline void __mmap_lock_trace_start_locking(struct mm_struct *mm, | |
48 | bool write) | |
49 | { | |
50 | } | |
51 | ||
52 | static inline void __mmap_lock_trace_acquire_returned(struct mm_struct *mm, | |
53 | bool write, bool success) | |
54 | { | |
55 | } | |
56 | ||
57 | static inline void __mmap_lock_trace_released(struct mm_struct *mm, bool write) | |
58 | { | |
59 | } | |
60 | ||
61 | #endif /* CONFIG_TRACING */ | |
62 | ||
438b6e12 SB |
63 | static inline void mmap_assert_locked(struct mm_struct *mm) |
64 | { | |
65 | lockdep_assert_held(&mm->mmap_lock); | |
66 | VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm); | |
67 | } | |
68 | ||
69 | static inline void mmap_assert_write_locked(struct mm_struct *mm) | |
70 | { | |
71 | lockdep_assert_held_write(&mm->mmap_lock); | |
72 | VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm); | |
73 | } | |
74 | ||
5e31275c SB |
75 | #ifdef CONFIG_PER_VMA_LOCK |
76 | static inline void vma_end_write_all(struct mm_struct *mm) | |
77 | { | |
78 | mmap_assert_write_locked(mm); | |
79 | /* No races during update due to exclusive mmap_lock being held */ | |
80 | WRITE_ONCE(mm->mm_lock_seq, mm->mm_lock_seq + 1); | |
81 | } | |
82 | #else | |
83 | static inline void vma_end_write_all(struct mm_struct *mm) {} | |
84 | #endif | |
85 | ||
9740ca4e ML |
86 | static inline void mmap_init_lock(struct mm_struct *mm) |
87 | { | |
da1c55f1 | 88 | init_rwsem(&mm->mmap_lock); |
9740ca4e ML |
89 | } |
90 | ||
91 | static inline void mmap_write_lock(struct mm_struct *mm) | |
92 | { | |
2b5067a8 | 93 | __mmap_lock_trace_start_locking(mm, true); |
da1c55f1 | 94 | down_write(&mm->mmap_lock); |
2b5067a8 | 95 | __mmap_lock_trace_acquire_returned(mm, true, true); |
9740ca4e ML |
96 | } |
97 | ||
aaa2cc56 ML |
98 | static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass) |
99 | { | |
2b5067a8 | 100 | __mmap_lock_trace_start_locking(mm, true); |
da1c55f1 | 101 | down_write_nested(&mm->mmap_lock, subclass); |
2b5067a8 | 102 | __mmap_lock_trace_acquire_returned(mm, true, true); |
aaa2cc56 ML |
103 | } |
104 | ||
9740ca4e ML |
105 | static inline int mmap_write_lock_killable(struct mm_struct *mm) |
106 | { | |
2b5067a8 AR |
107 | int ret; |
108 | ||
109 | __mmap_lock_trace_start_locking(mm, true); | |
110 | ret = down_write_killable(&mm->mmap_lock); | |
111 | __mmap_lock_trace_acquire_returned(mm, true, ret == 0); | |
112 | return ret; | |
9740ca4e ML |
113 | } |
114 | ||
115 | static inline bool mmap_write_trylock(struct mm_struct *mm) | |
116 | { | |
2b5067a8 AR |
117 | bool ret; |
118 | ||
119 | __mmap_lock_trace_start_locking(mm, true); | |
120 | ret = down_write_trylock(&mm->mmap_lock) != 0; | |
121 | __mmap_lock_trace_acquire_returned(mm, true, ret); | |
122 | return ret; | |
9740ca4e ML |
123 | } |
124 | ||
125 | static inline void mmap_write_unlock(struct mm_struct *mm) | |
126 | { | |
2b5067a8 | 127 | __mmap_lock_trace_released(mm, true); |
5e31275c | 128 | vma_end_write_all(mm); |
10994316 | 129 | up_write(&mm->mmap_lock); |
9740ca4e ML |
130 | } |
131 | ||
132 | static inline void mmap_write_downgrade(struct mm_struct *mm) | |
133 | { | |
2b5067a8 | 134 | __mmap_lock_trace_acquire_returned(mm, false, true); |
5e31275c | 135 | vma_end_write_all(mm); |
10994316 | 136 | downgrade_write(&mm->mmap_lock); |
9740ca4e ML |
137 | } |
138 | ||
139 | static inline void mmap_read_lock(struct mm_struct *mm) | |
140 | { | |
2b5067a8 | 141 | __mmap_lock_trace_start_locking(mm, false); |
da1c55f1 | 142 | down_read(&mm->mmap_lock); |
2b5067a8 | 143 | __mmap_lock_trace_acquire_returned(mm, false, true); |
9740ca4e ML |
144 | } |
145 | ||
146 | static inline int mmap_read_lock_killable(struct mm_struct *mm) | |
147 | { | |
2b5067a8 AR |
148 | int ret; |
149 | ||
150 | __mmap_lock_trace_start_locking(mm, false); | |
151 | ret = down_read_killable(&mm->mmap_lock); | |
152 | __mmap_lock_trace_acquire_returned(mm, false, ret == 0); | |
153 | return ret; | |
9740ca4e ML |
154 | } |
155 | ||
156 | static inline bool mmap_read_trylock(struct mm_struct *mm) | |
157 | { | |
2b5067a8 AR |
158 | bool ret; |
159 | ||
160 | __mmap_lock_trace_start_locking(mm, false); | |
161 | ret = down_read_trylock(&mm->mmap_lock) != 0; | |
162 | __mmap_lock_trace_acquire_returned(mm, false, ret); | |
163 | return ret; | |
9740ca4e ML |
164 | } |
165 | ||
166 | static inline void mmap_read_unlock(struct mm_struct *mm) | |
167 | { | |
2b5067a8 | 168 | __mmap_lock_trace_released(mm, false); |
10994316 | 169 | up_read(&mm->mmap_lock); |
9740ca4e ML |
170 | } |
171 | ||
0cc55a02 ML |
172 | static inline void mmap_read_unlock_non_owner(struct mm_struct *mm) |
173 | { | |
2b5067a8 | 174 | __mmap_lock_trace_released(mm, false); |
10994316 | 175 | up_read_non_owner(&mm->mmap_lock); |
0cc55a02 ML |
176 | } |
177 | ||
07e5bfe6 CC |
178 | static inline int mmap_lock_is_contended(struct mm_struct *mm) |
179 | { | |
180 | return rwsem_is_contended(&mm->mmap_lock); | |
181 | } | |
182 | ||
9740ca4e | 183 | #endif /* _LINUX_MMAP_LOCK_H */ |