Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
1da177e4 | 2 | /* |
f30c2269 | 3 | * include/linux/writeback.h |
1da177e4 LT |
4 | */ |
5 | #ifndef WRITEBACK_H | |
6 | #define WRITEBACK_H | |
7 | ||
e8edc6e0 | 8 | #include <linux/sched.h> |
a27bb332 | 9 | #include <linux/workqueue.h> |
f5ff8422 | 10 | #include <linux/fs.h> |
380c27ca | 11 | #include <linux/flex_proportions.h> |
21c6321f | 12 | #include <linux/backing-dev-defs.h> |
7637241e | 13 | #include <linux/blk_types.h> |
751e0d55 | 14 | #include <linux/pagevec.h> |
e8edc6e0 | 15 | |
2f8b5444 CH |
16 | struct bio; |
17 | ||
54848d73 WF |
18 | DECLARE_PER_CPU(int, dirty_throttle_leaks); |
19 | ||
ffd1f609 | 20 | /* |
ffd1f609 WF |
21 | * The global dirty threshold is normally equal to the global dirty limit, |
22 | * except when the system suddenly allocates a lot of anonymous memory and | |
23 | * knocks down the global dirty threshold quickly, in which case the global | |
24 | * dirty limit will follow down slowly to prevent livelocking all dirtier tasks. | |
25 | */ | |
1a12d8bd | 26 | #define DIRTY_SCOPE 8 |
ffd1f609 | 27 | |
1da177e4 LT |
28 | struct backing_dev_info; |
29 | ||
1da177e4 LT |
30 | /* |
31 | * fs/fs-writeback.c | |
32 | */ | |
33 | enum writeback_sync_modes { | |
34 | WB_SYNC_NONE, /* Don't wait on anything */ | |
35 | WB_SYNC_ALL, /* Wait on every mapping */ | |
1da177e4 LT |
36 | }; |
37 | ||
38 | /* | |
39 | * A control structure which tells the writeback code what to do. These are | |
40 | * always on the stack, and hence need no locking. They are always initialised | |
41 | * in a manner such that unspecified fields are set to zero. | |
42 | */ | |
43 | struct writeback_control { | |
751e0d55 | 44 | /* public fields that can be set and/or consumed by the caller: */ |
1da177e4 LT |
45 | long nr_to_write; /* Write this many pages, and decrement |
46 | this for each page written */ | |
47 | long pages_skipped; /* Pages which were not written */ | |
48 | ||
49 | /* | |
95468fd4 | 50 | * For a_ops->writepages(): if start or end are non-zero then this is |
1da177e4 LT |
51 | * a hint that the filesystem need only write out the pages inside that |
52 | * byterange. The byte at `end' is included in the writeout request. | |
53 | */ | |
111ebb6e OH |
54 | loff_t range_start; |
55 | loff_t range_end; | |
1da177e4 | 56 | |
4cd9069a RK |
57 | enum writeback_sync_modes sync_mode; |
58 | ||
22905f77 | 59 | unsigned for_kupdate:1; /* A kupdate writeback */ |
b17621fe | 60 | unsigned for_background:1; /* A background writeback */ |
6e6938b6 | 61 | unsigned tagged_writepages:1; /* tag-and-write to avoid livelock */ |
22905f77 | 62 | unsigned for_reclaim:1; /* Invoked from the page allocator */ |
111ebb6e | 63 | unsigned range_cyclic:1; /* range_start is cyclic */ |
7747bd4b | 64 | unsigned for_sync:1; /* sync(2) WB_SYNC_ALL writeback */ |
c9c4ff12 | 65 | unsigned unpinned_netfs_wb:1; /* Cleared I_PINNING_NETFS_WB */ |
27b36d8f TH |
66 | |
67 | /* | |
68 | * When writeback IOs are bounced through async layers, only the | |
69 | * initial synchronous phase should be accounted towards inode | |
70 | * cgroup ownership arbitration to avoid confusion. Later stages | |
71 | * can set the following flag to disable the accounting. | |
72 | */ | |
73 | unsigned no_cgroup_owner:1; | |
74 | ||
2282679f N |
75 | /* To enable batching of swap writes to non-block-device backends, |
76 | * "plug" can be set point to a 'struct swap_iocb *'. When all swap | |
77 | * writes have been submitted, if with swap_iocb is not NULL, | |
78 | * swap_write_unplug() should be called. | |
79 | */ | |
80 | struct swap_iocb **swap_plug; | |
81 | ||
751e0d55 MWO |
82 | /* internal fields used by the ->writepages implementation: */ |
83 | struct folio_batch fbatch; | |
84 | pgoff_t index; | |
cdc150b5 | 85 | int saved_err; |
751e0d55 | 86 | |
b16b1deb TH |
87 | #ifdef CONFIG_CGROUP_WRITEBACK |
88 | struct bdi_writeback *wb; /* wb this writeback is issued under */ | |
2a814908 TH |
89 | struct inode *inode; /* inode being written out */ |
90 | ||
91 | /* foreign inode detection, see wbc_detach_inode() */ | |
92 | int wb_id; /* current wb id */ | |
93 | int wb_lcand_id; /* last foreign candidate wb id */ | |
94 | int wb_tcand_id; /* this foreign candidate wb id */ | |
95 | size_t wb_bytes; /* bytes written by current wb */ | |
96 | size_t wb_lcand_bytes; /* bytes written by last candidate */ | |
97 | size_t wb_tcand_bytes; /* bytes written by this candidate */ | |
b16b1deb | 98 | #endif |
1da177e4 LT |
99 | }; |
100 | ||
f8e6e4bd | 101 | static inline blk_opf_t wbc_to_write_flags(struct writeback_control *wbc) |
7637241e | 102 | { |
f8e6e4bd | 103 | blk_opf_t flags = 0; |
d3f77dfd | 104 | |
7637241e | 105 | if (wbc->sync_mode == WB_SYNC_ALL) |
d3f77dfd | 106 | flags |= REQ_SYNC; |
13edd5e7 | 107 | else if (wbc->for_kupdate || wbc->for_background) |
d3f77dfd | 108 | flags |= REQ_BACKGROUND; |
7637241e | 109 | |
d3f77dfd | 110 | return flags; |
7637241e JA |
111 | } |
112 | ||
653c45c6 | 113 | #ifdef CONFIG_CGROUP_WRITEBACK |
348332e0 CH |
114 | #define wbc_blkcg_css(wbc) \ |
115 | ((wbc)->wb ? (wbc)->wb->blkcg_css : blkcg_root_css) | |
116 | #else | |
117 | #define wbc_blkcg_css(wbc) (blkcg_root_css) | |
118 | #endif /* CONFIG_CGROUP_WRITEBACK */ | |
653c45c6 | 119 | |
380c27ca TH |
120 | /* |
121 | * A wb_domain represents a domain that wb's (bdi_writeback's) belong to | |
122 | * and are measured against each other in. There always is one global | |
123 | * domain, global_wb_domain, that every wb in the system is a member of. | |
124 | * This allows measuring the relative bandwidth of each wb to distribute | |
125 | * dirtyable memory accordingly. | |
126 | */ | |
127 | struct wb_domain { | |
dcc25ae7 TH |
128 | spinlock_t lock; |
129 | ||
380c27ca TH |
130 | /* |
131 | * Scale the writeback cache size proportional to the relative | |
132 | * writeout speed. | |
133 | * | |
134 | * We do this by keeping a floating proportion between BDIs, based | |
135 | * on page writeback completions [end_page_writeback()]. Those | |
136 | * devices that write out pages fastest will get the larger share, | |
137 | * while the slower will get a smaller share. | |
138 | * | |
139 | * We use page writeout completions because we are interested in | |
140 | * getting rid of dirty pages. Having them written out is the | |
141 | * primary goal. | |
142 | * | |
143 | * We introduce a concept of time, a period over which we measure | |
144 | * these events, because demand can/will vary over time. The length | |
145 | * of this period itself is measured in page writeback completions. | |
146 | */ | |
147 | struct fprop_global completions; | |
148 | struct timer_list period_timer; /* timer for aging of completions */ | |
149 | unsigned long period_time; | |
dcc25ae7 TH |
150 | |
151 | /* | |
152 | * The dirtyable memory and dirty threshold could be suddenly | |
153 | * knocked down by a large amount (eg. on the startup of KVM in a | |
154 | * swapless system). This may throw the system into deep dirty | |
155 | * exceeded state and throttle heavy/light dirtiers alike. To | |
156 | * retain good responsiveness, maintain global_dirty_limit for | |
157 | * tracking slowly down to the knocked down dirty threshold. | |
158 | * | |
159 | * Both fields are protected by ->lock. | |
160 | */ | |
161 | unsigned long dirty_limit_tstamp; | |
162 | unsigned long dirty_limit; | |
380c27ca TH |
163 | }; |
164 | ||
2529bb3a TH |
165 | /** |
166 | * wb_domain_size_changed - memory available to a wb_domain has changed | |
167 | * @dom: wb_domain of interest | |
168 | * | |
169 | * This function should be called when the amount of memory available to | |
170 | * @dom has changed. It resets @dom's dirty limit parameters to prevent | |
171 | * the past values which don't match the current configuration from skewing | |
172 | * dirty throttling. Without this, when memory size of a wb_domain is | |
173 | * greatly reduced, the dirty throttling logic may allow too many pages to | |
174 | * be dirtied leading to consecutive unnecessary OOMs and may get stuck in | |
175 | * that situation. | |
176 | */ | |
177 | static inline void wb_domain_size_changed(struct wb_domain *dom) | |
178 | { | |
179 | spin_lock(&dom->lock); | |
180 | dom->dirty_limit_tstamp = jiffies; | |
181 | dom->dirty_limit = 0; | |
182 | spin_unlock(&dom->lock); | |
183 | } | |
184 | ||
1da177e4 LT |
185 | /* |
186 | * fs/fs-writeback.c | |
187 | */ | |
03ba3782 | 188 | struct bdi_writeback; |
0e175a18 CW |
189 | void writeback_inodes_sb(struct super_block *, enum wb_reason reason); |
190 | void writeback_inodes_sb_nr(struct super_block *, unsigned long nr, | |
191 | enum wb_reason reason); | |
8264c321 | 192 | void try_to_writeback_inodes_sb(struct super_block *sb, enum wb_reason reason); |
0dc83bd3 | 193 | void sync_inodes_sb(struct super_block *); |
9ba4b2df | 194 | void wakeup_flusher_threads(enum wb_reason reason); |
595043e5 JA |
195 | void wakeup_flusher_threads_bdi(struct backing_dev_info *bdi, |
196 | enum wb_reason reason); | |
169ebd90 | 197 | void inode_wait_for_writeback(struct inode *inode); |
4301efa4 | 198 | void inode_io_list_del(struct inode *inode); |
1da177e4 LT |
199 | |
200 | /* writeback.h requires fs.h; it, too, is not included from here. */ | |
201 | static inline void wait_on_inode(struct inode *inode) | |
202 | { | |
74316201 | 203 | wait_on_bit(&inode->i_state, __I_NEW, TASK_UNINTERRUPTIBLE); |
1da177e4 | 204 | } |
1c0eeaf5 | 205 | |
21c6321f TH |
206 | #ifdef CONFIG_CGROUP_WRITEBACK |
207 | ||
b16b1deb TH |
208 | #include <linux/cgroup.h> |
209 | #include <linux/bio.h> | |
210 | ||
9cfb816b | 211 | void __inode_attach_wb(struct inode *inode, struct folio *folio); |
b16b1deb TH |
212 | void wbc_attach_and_unlock_inode(struct writeback_control *wbc, |
213 | struct inode *inode) | |
214 | __releases(&inode->i_lock); | |
215 | void wbc_detach_inode(struct writeback_control *wbc); | |
34e51a5e TH |
216 | void wbc_account_cgroup_owner(struct writeback_control *wbc, struct page *page, |
217 | size_t bytes); | |
7490a2d2 | 218 | int cgroup_writeback_by_id(u64 bdi_id, int memcg_id, |
d62241c7 | 219 | enum wb_reason reason, struct wb_completion *done); |
a1a0e23e | 220 | void cgroup_writeback_umount(void); |
c22d70a1 | 221 | bool cleanup_offline_cgwb(struct bdi_writeback *wb); |
21c6321f TH |
222 | |
223 | /** | |
224 | * inode_attach_wb - associate an inode with its wb | |
225 | * @inode: inode of interest | |
9cfb816b | 226 | * @folio: folio being dirtied (may be NULL) |
21c6321f TH |
227 | * |
228 | * If @inode doesn't have its wb, associate it with the wb matching the | |
9cfb816b | 229 | * memcg of @folio or, if @folio is NULL, %current. May be called w/ or w/o |
21c6321f TH |
230 | * @inode->i_lock. |
231 | */ | |
9cfb816b | 232 | static inline void inode_attach_wb(struct inode *inode, struct folio *folio) |
21c6321f TH |
233 | { |
234 | if (!inode->i_wb) | |
9cfb816b | 235 | __inode_attach_wb(inode, folio); |
21c6321f TH |
236 | } |
237 | ||
238 | /** | |
239 | * inode_detach_wb - disassociate an inode from its wb | |
240 | * @inode: inode of interest | |
241 | * | |
242 | * @inode is being freed. Detach from its wb. | |
243 | */ | |
244 | static inline void inode_detach_wb(struct inode *inode) | |
245 | { | |
246 | if (inode->i_wb) { | |
f759741d | 247 | WARN_ON_ONCE(!(inode->i_state & I_CLEAR)); |
21c6321f TH |
248 | wb_put(inode->i_wb); |
249 | inode->i_wb = NULL; | |
250 | } | |
251 | } | |
252 | ||
b16b1deb TH |
253 | /** |
254 | * wbc_attach_fdatawrite_inode - associate wbc and inode for fdatawrite | |
255 | * @wbc: writeback_control of interest | |
256 | * @inode: target inode | |
257 | * | |
258 | * This function is to be used by __filemap_fdatawrite_range(), which is an | |
259 | * alternative entry point into writeback code, and first ensures @inode is | |
260 | * associated with a bdi_writeback and attaches it to @wbc. | |
261 | */ | |
262 | static inline void wbc_attach_fdatawrite_inode(struct writeback_control *wbc, | |
263 | struct inode *inode) | |
264 | { | |
265 | spin_lock(&inode->i_lock); | |
266 | inode_attach_wb(inode, NULL); | |
267 | wbc_attach_and_unlock_inode(wbc, inode); | |
268 | } | |
269 | ||
270 | /** | |
271 | * wbc_init_bio - writeback specific initializtion of bio | |
272 | * @wbc: writeback_control for the writeback in progress | |
273 | * @bio: bio to be initialized | |
274 | * | |
275 | * @bio is a part of the writeback in progress controlled by @wbc. Perform | |
276 | * writeback specific initialization. This is used to apply the cgroup | |
fd42df30 DZ |
277 | * writeback context. Must be called after the bio has been associated with |
278 | * a device. | |
b16b1deb TH |
279 | */ |
280 | static inline void wbc_init_bio(struct writeback_control *wbc, struct bio *bio) | |
281 | { | |
282 | /* | |
283 | * pageout() path doesn't attach @wbc to the inode being written | |
284 | * out. This is intentional as we don't want the function to block | |
285 | * behind a slow cgroup. Ultimately, we want pageout() to kick off | |
286 | * regular writeback instead of writing things out itself. | |
287 | */ | |
288 | if (wbc->wb) | |
fd42df30 | 289 | bio_associate_blkg_from_css(bio, wbc->wb->blkcg_css); |
b16b1deb TH |
290 | } |
291 | ||
21c6321f TH |
292 | #else /* CONFIG_CGROUP_WRITEBACK */ |
293 | ||
9cfb816b | 294 | static inline void inode_attach_wb(struct inode *inode, struct folio *folio) |
21c6321f TH |
295 | { |
296 | } | |
297 | ||
298 | static inline void inode_detach_wb(struct inode *inode) | |
299 | { | |
300 | } | |
301 | ||
b16b1deb TH |
302 | static inline void wbc_attach_and_unlock_inode(struct writeback_control *wbc, |
303 | struct inode *inode) | |
304 | __releases(&inode->i_lock) | |
305 | { | |
306 | spin_unlock(&inode->i_lock); | |
307 | } | |
308 | ||
309 | static inline void wbc_attach_fdatawrite_inode(struct writeback_control *wbc, | |
310 | struct inode *inode) | |
311 | { | |
312 | } | |
313 | ||
314 | static inline void wbc_detach_inode(struct writeback_control *wbc) | |
315 | { | |
316 | } | |
317 | ||
318 | static inline void wbc_init_bio(struct writeback_control *wbc, struct bio *bio) | |
319 | { | |
320 | } | |
321 | ||
34e51a5e TH |
322 | static inline void wbc_account_cgroup_owner(struct writeback_control *wbc, |
323 | struct page *page, size_t bytes) | |
2a814908 TH |
324 | { |
325 | } | |
326 | ||
a1a0e23e TH |
327 | static inline void cgroup_writeback_umount(void) |
328 | { | |
329 | } | |
330 | ||
21c6321f TH |
331 | #endif /* CONFIG_CGROUP_WRITEBACK */ |
332 | ||
1da177e4 LT |
333 | /* |
334 | * mm/page-writeback.c | |
335 | */ | |
31373d09 | 336 | void laptop_io_completion(struct backing_dev_info *info); |
1da177e4 | 337 | void laptop_sync_completion(void); |
bca237a5 | 338 | void laptop_mode_timer_fn(struct timer_list *t); |
281e3726 | 339 | bool node_dirty_ok(struct pglist_data *pgdat); |
380c27ca | 340 | int wb_domain_init(struct wb_domain *dom, gfp_t gfp); |
841710aa TH |
341 | #ifdef CONFIG_CGROUP_WRITEBACK |
342 | void wb_domain_exit(struct wb_domain *dom); | |
343 | #endif | |
1da177e4 | 344 | |
dcc25ae7 | 345 | extern struct wb_domain global_wb_domain; |
c42843f2 | 346 | |
1da177e4 | 347 | /* These are exported to sysctl. */ |
704503d8 AD |
348 | extern unsigned int dirty_writeback_interval; |
349 | extern unsigned int dirty_expire_interval; | |
1efff914 | 350 | extern unsigned int dirtytime_expire_interval; |
1da177e4 LT |
351 | extern int laptop_mode; |
352 | ||
1efff914 | 353 | int dirtytime_interval_handler(struct ctl_table *table, int write, |
32927393 | 354 | void *buffer, size_t *lenp, loff_t *ppos); |
1da177e4 | 355 | |
16c4042f | 356 | void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty); |
0d960a38 | 357 | unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh); |
4b5bbc39 | 358 | unsigned long cgwb_calc_thresh(struct bdi_writeback *wb); |
a88a341a | 359 | |
45a2966f | 360 | void wb_update_bandwidth(struct bdi_writeback *wb); |
fe6c9c6e JK |
361 | |
362 | /* Invoke balance dirty pages in async mode. */ | |
363 | #define BDP_ASYNC 0x0001 | |
364 | ||
d0e1d66b | 365 | void balance_dirty_pages_ratelimited(struct address_space *mapping); |
fe6c9c6e JK |
366 | int balance_dirty_pages_ratelimited_flags(struct address_space *mapping, |
367 | unsigned int flags); | |
368 | ||
aa661bbe | 369 | bool wb_over_bg_thresh(struct bdi_writeback *wb); |
fa5a734e | 370 | |
cdc150b5 CH |
371 | struct folio *writeback_iter(struct address_space *mapping, |
372 | struct writeback_control *wbc, struct folio *folio, int *error); | |
373 | ||
d585bdbe | 374 | typedef int (*writepage_t)(struct folio *folio, struct writeback_control *wbc, |
0ea97180 MS |
375 | void *data); |
376 | ||
0ea97180 MS |
377 | int write_cache_pages(struct address_space *mapping, |
378 | struct writeback_control *wbc, writepage_t writepage, | |
379 | void *data); | |
1da177e4 | 380 | int do_writepages(struct address_space *mapping, struct writeback_control *wbc); |
2d1d43f6 | 381 | void writeback_set_ratelimit(void); |
92c09c04 NK |
382 | void tag_pages_for_writeback(struct address_space *mapping, |
383 | pgoff_t start, pgoff_t end); | |
1da177e4 | 384 | |
85d4d2eb | 385 | bool filemap_dirty_folio(struct address_space *mapping, struct folio *folio); |
cd78ab11 MWO |
386 | bool folio_redirty_for_writepage(struct writeback_control *, struct folio *); |
387 | bool redirty_page_for_writepage(struct writeback_control *, struct page *); | |
2f800fbd | 388 | |
6c60d2b5 DC |
389 | void sb_mark_inode_writeback(struct inode *inode); |
390 | void sb_clear_inode_writeback(struct inode *inode); | |
391 | ||
1da177e4 | 392 | #endif /* WRITEBACK_H */ |