Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
1da177e4 LT |
2 | /* |
3 | * include/linux/backing-dev.h | |
4 | * | |
5 | * low-level device information and state which is propagated up through | |
6 | * to high-level code. | |
7 | */ | |
8 | ||
9 | #ifndef _LINUX_BACKING_DEV_H | |
10 | #define _LINUX_BACKING_DEV_H | |
11 | ||
cf0ca9fe | 12 | #include <linux/kernel.h> |
e4ad08fe | 13 | #include <linux/fs.h> |
03ba3782 | 14 | #include <linux/sched.h> |
68f23b89 | 15 | #include <linux/device.h> |
03ba3782 | 16 | #include <linux/writeback.h> |
66114cad | 17 | #include <linux/backing-dev-defs.h> |
a13f35e8 | 18 | #include <linux/slab.h> |
de1414a6 | 19 | |
d03f6cdc JK |
20 | static inline struct backing_dev_info *bdi_get(struct backing_dev_info *bdi) |
21 | { | |
22 | kref_get(&bdi->refcnt); | |
23 | return bdi; | |
24 | } | |
25 | ||
34f8fe50 | 26 | struct backing_dev_info *bdi_get_by_id(u64 id); |
d03f6cdc | 27 | void bdi_put(struct backing_dev_info *bdi); |
b2e8fb6e | 28 | |
7c4cc300 JK |
29 | __printf(2, 3) |
30 | int bdi_register(struct backing_dev_info *bdi, const char *fmt, ...); | |
a93f00b3 | 31 | __printf(2, 0) |
7c4cc300 JK |
32 | int bdi_register_va(struct backing_dev_info *bdi, const char *fmt, |
33 | va_list args); | |
3c5d202b | 34 | void bdi_set_owner(struct backing_dev_info *bdi, struct device *owner); |
b02176f3 TH |
35 | void bdi_unregister(struct backing_dev_info *bdi); |
36 | ||
aef33c2f | 37 | struct backing_dev_info *bdi_alloc(int node_id); |
b02176f3 | 38 | |
9ecf4866 | 39 | void wb_start_background_writeback(struct bdi_writeback *wb); |
f0054bb1 | 40 | void wb_workfn(struct work_struct *work); |
f0054bb1 | 41 | void wb_wakeup_delayed(struct bdi_writeback *wb); |
cf0ca9fe | 42 | |
5b9cce4c TH |
43 | void wb_wait_for_completion(struct wb_completion *done); |
44 | ||
03ba3782 | 45 | extern spinlock_t bdi_lock; |
66f3b8e2 JA |
46 | extern struct list_head bdi_list; |
47 | ||
839a8e86 | 48 | extern struct workqueue_struct *bdi_wq; |
d3f77dfd | 49 | extern struct workqueue_struct *bdi_async_bio_wq; |
839a8e86 | 50 | |
d6c10f1f | 51 | static inline bool wb_has_dirty_io(struct bdi_writeback *wb) |
03ba3782 | 52 | { |
d6c10f1f | 53 | return test_bit(WB_has_dirty_io, &wb->state); |
03ba3782 JA |
54 | } |
55 | ||
95a46c65 TH |
56 | static inline bool bdi_has_dirty_io(struct backing_dev_info *bdi) |
57 | { | |
58 | /* | |
59 | * @bdi->tot_write_bandwidth is guaranteed to be > 0 if there are | |
60 | * any dirty wbs. See wb_update_write_bandwidth(). | |
61 | */ | |
62 | return atomic_long_read(&bdi->tot_write_bandwidth); | |
03ba3782 JA |
63 | } |
64 | ||
bd3488e7 | 65 | static inline void wb_stat_mod(struct bdi_writeback *wb, |
93f78d88 | 66 | enum wb_stat_item item, s64 amount) |
b2e8fb6e | 67 | { |
104b4e51 | 68 | percpu_counter_add_batch(&wb->stat[item], amount, WB_STAT_BATCH); |
b2e8fb6e PZ |
69 | } |
70 | ||
93f78d88 | 71 | static inline void inc_wb_stat(struct bdi_writeback *wb, enum wb_stat_item item) |
b2e8fb6e | 72 | { |
bd3488e7 | 73 | wb_stat_mod(wb, item, 1); |
b2e8fb6e PZ |
74 | } |
75 | ||
93f78d88 | 76 | static inline void dec_wb_stat(struct bdi_writeback *wb, enum wb_stat_item item) |
b2e8fb6e | 77 | { |
bd3488e7 | 78 | wb_stat_mod(wb, item, -1); |
b2e8fb6e PZ |
79 | } |
80 | ||
93f78d88 | 81 | static inline s64 wb_stat(struct bdi_writeback *wb, enum wb_stat_item item) |
b2e8fb6e | 82 | { |
93f78d88 | 83 | return percpu_counter_read_positive(&wb->stat[item]); |
b2e8fb6e PZ |
84 | } |
85 | ||
93f78d88 | 86 | static inline s64 wb_stat_sum(struct bdi_writeback *wb, enum wb_stat_item item) |
e0bf68dd | 87 | { |
e3d3910a | 88 | return percpu_counter_sum_positive(&wb->stat[item]); |
e0bf68dd PZ |
89 | } |
90 | ||
93f78d88 | 91 | extern void wb_writeout_inc(struct bdi_writeback *wb); |
dd5656e5 | 92 | |
b2e8fb6e PZ |
93 | /* |
94 | * maximal error of a stat counter. | |
95 | */ | |
2bce774e | 96 | static inline unsigned long wb_stat_error(void) |
e0bf68dd | 97 | { |
b2e8fb6e | 98 | #ifdef CONFIG_SMP |
93f78d88 | 99 | return nr_cpu_ids * WB_STAT_BATCH; |
b2e8fb6e PZ |
100 | #else |
101 | return 1; | |
102 | #endif | |
e0bf68dd | 103 | } |
1da177e4 | 104 | |
ae82291e SR |
105 | /* BDI ratio is expressed as part per 1000000 for finer granularity. */ |
106 | #define BDI_RATIO_SCALE 10000 | |
107 | ||
712c00d6 | 108 | u64 bdi_get_min_bytes(struct backing_dev_info *bdi); |
00df7d51 | 109 | u64 bdi_get_max_bytes(struct backing_dev_info *bdi); |
189d3c4a | 110 | int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio); |
a42dde04 | 111 | int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio); |
2c44af4f | 112 | int bdi_set_min_ratio_no_scale(struct backing_dev_info *bdi, unsigned int min_ratio); |
4e230b40 | 113 | int bdi_set_max_ratio_no_scale(struct backing_dev_info *bdi, unsigned int max_ratio); |
803c9805 | 114 | int bdi_set_min_bytes(struct backing_dev_info *bdi, u64 min_bytes); |
1bf27e98 | 115 | int bdi_set_max_bytes(struct backing_dev_info *bdi, u64 max_bytes); |
8e9d5ead | 116 | int bdi_set_strict_limit(struct backing_dev_info *bdi, unsigned int strict_limit); |
189d3c4a | 117 | |
1da177e4 LT |
118 | /* |
119 | * Flags in backing_dev_info::capability | |
e4ad08fe | 120 | * |
f56753ac CH |
121 | * BDI_CAP_WRITEBACK: Supports dirty page writeback, and dirty pages |
122 | * should contribute to accounting | |
123 | * BDI_CAP_WRITEBACK_ACCT: Automatically account writeback pages | |
124 | * BDI_CAP_STRICTLIMIT: Keep number of dirty pages below bdi threshold | |
1da177e4 | 125 | */ |
f56753ac CH |
126 | #define BDI_CAP_WRITEBACK (1 << 0) |
127 | #define BDI_CAP_WRITEBACK_ACCT (1 << 1) | |
128 | #define BDI_CAP_STRICTLIMIT (1 << 2) | |
e4ad08fe | 129 | |
5129a469 | 130 | extern struct backing_dev_info noop_backing_dev_info; |
1da177e4 | 131 | |
4bca7e80 JK |
132 | int bdi_init(struct backing_dev_info *bdi); |
133 | ||
bc05873d TH |
134 | /** |
135 | * writeback_in_progress - determine whether there is writeback in progress | |
136 | * @wb: bdi_writeback of interest | |
137 | * | |
138 | * Determine whether there is writeback waiting to be handled against a | |
139 | * bdi_writeback. | |
140 | */ | |
141 | static inline bool writeback_in_progress(struct bdi_writeback *wb) | |
1da177e4 | 142 | { |
bc05873d | 143 | return test_bit(WB_writeback_running, &wb->state); |
1da177e4 LT |
144 | } |
145 | ||
ccdf7741 | 146 | struct backing_dev_info *inode_to_bdi(struct inode *inode); |
1da177e4 | 147 | |
f56753ac | 148 | static inline bool mapping_can_writeback(struct address_space *mapping) |
e4ad08fe | 149 | { |
f56753ac | 150 | return inode_to_bdi(mapping->host)->capabilities & BDI_CAP_WRITEBACK; |
e4ad08fe | 151 | } |
1da177e4 | 152 | |
89e9b9e0 TH |
153 | #ifdef CONFIG_CGROUP_WRITEBACK |
154 | ||
ed288dc0 TH |
155 | struct bdi_writeback *wb_get_lookup(struct backing_dev_info *bdi, |
156 | struct cgroup_subsys_state *memcg_css); | |
52ebea74 TH |
157 | struct bdi_writeback *wb_get_create(struct backing_dev_info *bdi, |
158 | struct cgroup_subsys_state *memcg_css, | |
159 | gfp_t gfp); | |
52ebea74 | 160 | void wb_memcg_offline(struct mem_cgroup *memcg); |
dec223c9 | 161 | void wb_blkcg_offline(struct cgroup_subsys_state *css); |
52ebea74 | 162 | |
89e9b9e0 TH |
163 | /** |
164 | * inode_cgwb_enabled - test whether cgroup writeback is enabled on an inode | |
165 | * @inode: inode of interest | |
166 | * | |
ed7b6b4f CH |
167 | * Cgroup writeback requires support from the filesystem. Also, both memcg and |
168 | * iocg have to be on the default hierarchy. Test whether all conditions are | |
169 | * met. | |
9badce00 TH |
170 | * |
171 | * Note that the test result may change dynamically on the same inode | |
172 | * depending on how memcg and iocg are configured. | |
89e9b9e0 TH |
173 | */ |
174 | static inline bool inode_cgwb_enabled(struct inode *inode) | |
175 | { | |
176 | struct backing_dev_info *bdi = inode_to_bdi(inode); | |
177 | ||
c0522908 TH |
178 | return cgroup_subsys_on_dfl(memory_cgrp_subsys) && |
179 | cgroup_subsys_on_dfl(io_cgrp_subsys) && | |
f56753ac | 180 | (bdi->capabilities & BDI_CAP_WRITEBACK) && |
46b15caa | 181 | (inode->i_sb->s_iflags & SB_I_CGROUPWB); |
89e9b9e0 TH |
182 | } |
183 | ||
52ebea74 TH |
184 | /** |
185 | * wb_find_current - find wb for %current on a bdi | |
186 | * @bdi: bdi of interest | |
187 | * | |
188 | * Find the wb of @bdi which matches both the memcg and blkcg of %current. | |
189 | * Must be called under rcu_read_lock() which protects the returend wb. | |
190 | * NULL if not found. | |
191 | */ | |
192 | static inline struct bdi_writeback *wb_find_current(struct backing_dev_info *bdi) | |
193 | { | |
194 | struct cgroup_subsys_state *memcg_css; | |
195 | struct bdi_writeback *wb; | |
196 | ||
197 | memcg_css = task_css(current, memory_cgrp_id); | |
198 | if (!memcg_css->parent) | |
199 | return &bdi->wb; | |
200 | ||
201 | wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id); | |
202 | ||
203 | /* | |
204 | * %current's blkcg equals the effective blkcg of its memcg. No | |
205 | * need to use the relatively expensive cgroup_get_e_css(). | |
206 | */ | |
c165b3e3 | 207 | if (likely(wb && wb->blkcg_css == task_css(current, io_cgrp_id))) |
52ebea74 TH |
208 | return wb; |
209 | return NULL; | |
210 | } | |
211 | ||
212 | /** | |
213 | * wb_get_create_current - get or create wb for %current on a bdi | |
214 | * @bdi: bdi of interest | |
215 | * @gfp: allocation mask | |
216 | * | |
217 | * Equivalent to wb_get_create() on %current's memcg. This function is | |
218 | * called from a relatively hot path and optimizes the common cases using | |
219 | * wb_find_current(). | |
220 | */ | |
221 | static inline struct bdi_writeback * | |
222 | wb_get_create_current(struct backing_dev_info *bdi, gfp_t gfp) | |
223 | { | |
224 | struct bdi_writeback *wb; | |
225 | ||
226 | rcu_read_lock(); | |
227 | wb = wb_find_current(bdi); | |
228 | if (wb && unlikely(!wb_tryget(wb))) | |
229 | wb = NULL; | |
230 | rcu_read_unlock(); | |
231 | ||
232 | if (unlikely(!wb)) { | |
233 | struct cgroup_subsys_state *memcg_css; | |
234 | ||
235 | memcg_css = task_get_css(current, memory_cgrp_id); | |
236 | wb = wb_get_create(bdi, memcg_css, gfp); | |
237 | css_put(memcg_css); | |
238 | } | |
239 | return wb; | |
240 | } | |
241 | ||
52ebea74 TH |
242 | /** |
243 | * inode_to_wb - determine the wb of an inode | |
244 | * @inode: inode of interest | |
245 | * | |
aaa2cacf | 246 | * Returns the wb @inode is currently associated with. The caller must be |
b93b0163 | 247 | * holding either @inode->i_lock, the i_pages lock, or the |
aaa2cacf | 248 | * associated wb's list_lock. |
52ebea74 | 249 | */ |
05b93801 | 250 | static inline struct bdi_writeback *inode_to_wb(const struct inode *inode) |
52ebea74 | 251 | { |
aaa2cacf TH |
252 | #ifdef CONFIG_LOCKDEP |
253 | WARN_ON_ONCE(debug_locks && | |
254 | (!lockdep_is_held(&inode->i_lock) && | |
b93b0163 | 255 | !lockdep_is_held(&inode->i_mapping->i_pages.xa_lock) && |
aaa2cacf TH |
256 | !lockdep_is_held(&inode->i_wb->list_lock))); |
257 | #endif | |
52ebea74 TH |
258 | return inode->i_wb; |
259 | } | |
260 | ||
fee468fd JK |
261 | static inline struct bdi_writeback *inode_to_wb_wbc( |
262 | struct inode *inode, | |
263 | struct writeback_control *wbc) | |
264 | { | |
265 | /* | |
266 | * If wbc does not have inode attached, it means cgroup writeback was | |
267 | * disabled when wbc started. Just use the default wb in that case. | |
268 | */ | |
269 | return wbc->wb ? wbc->wb : &inode_to_bdi(inode)->wb; | |
270 | } | |
271 | ||
682aa8e1 TH |
272 | /** |
273 | * unlocked_inode_to_wb_begin - begin unlocked inode wb access transaction | |
274 | * @inode: target inode | |
2e898e4c | 275 | * @cookie: output param, to be passed to the end function |
682aa8e1 TH |
276 | * |
277 | * The caller wants to access the wb associated with @inode but isn't | |
b93b0163 | 278 | * holding inode->i_lock, the i_pages lock or wb->list_lock. This |
682aa8e1 TH |
279 | * function determines the wb associated with @inode and ensures that the |
280 | * association doesn't change until the transaction is finished with | |
281 | * unlocked_inode_to_wb_end(). | |
282 | * | |
2e898e4c GT |
283 | * The caller must call unlocked_inode_to_wb_end() with *@cookie afterwards and |
284 | * can't sleep during the transaction. IRQs may or may not be disabled on | |
285 | * return. | |
682aa8e1 TH |
286 | */ |
287 | static inline struct bdi_writeback * | |
2e898e4c | 288 | unlocked_inode_to_wb_begin(struct inode *inode, struct wb_lock_cookie *cookie) |
682aa8e1 TH |
289 | { |
290 | rcu_read_lock(); | |
291 | ||
292 | /* | |
a9519def | 293 | * Paired with store_release in inode_switch_wbs_work_fn() and |
682aa8e1 TH |
294 | * ensures that we see the new wb if we see cleared I_WB_SWITCH. |
295 | */ | |
2e898e4c | 296 | cookie->locked = smp_load_acquire(&inode->i_state) & I_WB_SWITCH; |
682aa8e1 | 297 | |
2e898e4c GT |
298 | if (unlikely(cookie->locked)) |
299 | xa_lock_irqsave(&inode->i_mapping->i_pages, cookie->flags); | |
aaa2cacf TH |
300 | |
301 | /* | |
b93b0163 MW |
302 | * Protected by either !I_WB_SWITCH + rcu_read_lock() or the i_pages |
303 | * lock. inode_to_wb() will bark. Deref directly. | |
aaa2cacf TH |
304 | */ |
305 | return inode->i_wb; | |
682aa8e1 TH |
306 | } |
307 | ||
308 | /** | |
309 | * unlocked_inode_to_wb_end - end inode wb access transaction | |
310 | * @inode: target inode | |
2e898e4c | 311 | * @cookie: @cookie from unlocked_inode_to_wb_begin() |
682aa8e1 | 312 | */ |
2e898e4c GT |
313 | static inline void unlocked_inode_to_wb_end(struct inode *inode, |
314 | struct wb_lock_cookie *cookie) | |
682aa8e1 | 315 | { |
2e898e4c GT |
316 | if (unlikely(cookie->locked)) |
317 | xa_unlock_irqrestore(&inode->i_mapping->i_pages, cookie->flags); | |
682aa8e1 TH |
318 | |
319 | rcu_read_unlock(); | |
320 | } | |
321 | ||
89e9b9e0 TH |
322 | #else /* CONFIG_CGROUP_WRITEBACK */ |
323 | ||
324 | static inline bool inode_cgwb_enabled(struct inode *inode) | |
325 | { | |
326 | return false; | |
327 | } | |
328 | ||
52ebea74 TH |
329 | static inline struct bdi_writeback *wb_find_current(struct backing_dev_info *bdi) |
330 | { | |
331 | return &bdi->wb; | |
332 | } | |
333 | ||
334 | static inline struct bdi_writeback * | |
335 | wb_get_create_current(struct backing_dev_info *bdi, gfp_t gfp) | |
336 | { | |
337 | return &bdi->wb; | |
338 | } | |
339 | ||
52ebea74 TH |
340 | static inline struct bdi_writeback *inode_to_wb(struct inode *inode) |
341 | { | |
342 | return &inode_to_bdi(inode)->wb; | |
343 | } | |
344 | ||
fee468fd JK |
345 | static inline struct bdi_writeback *inode_to_wb_wbc( |
346 | struct inode *inode, | |
347 | struct writeback_control *wbc) | |
348 | { | |
349 | return inode_to_wb(inode); | |
350 | } | |
351 | ||
352 | ||
682aa8e1 | 353 | static inline struct bdi_writeback * |
2e898e4c | 354 | unlocked_inode_to_wb_begin(struct inode *inode, struct wb_lock_cookie *cookie) |
682aa8e1 TH |
355 | { |
356 | return inode_to_wb(inode); | |
357 | } | |
358 | ||
2e898e4c GT |
359 | static inline void unlocked_inode_to_wb_end(struct inode *inode, |
360 | struct wb_lock_cookie *cookie) | |
682aa8e1 TH |
361 | { |
362 | } | |
363 | ||
52ebea74 TH |
364 | static inline void wb_memcg_offline(struct mem_cgroup *memcg) |
365 | { | |
366 | } | |
367 | ||
dec223c9 | 368 | static inline void wb_blkcg_offline(struct cgroup_subsys_state *css) |
52ebea74 TH |
369 | { |
370 | } | |
371 | ||
89e9b9e0 TH |
372 | #endif /* CONFIG_CGROUP_WRITEBACK */ |
373 | ||
eb7ae5e0 | 374 | const char *bdi_dev_name(struct backing_dev_info *bdi); |
68f23b89 | 375 | |
89e9b9e0 | 376 | #endif /* _LINUX_BACKING_DEV_H */ |