Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
1da177e4 LT |
2 | /* |
3 | * linux/mm/swap.c | |
4 | * | |
5 | * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds | |
6 | */ | |
7 | ||
8 | /* | |
183ff22b | 9 | * This file contains the default values for the operation of the |
1da177e4 | 10 | * Linux VM subsystem. Fine-tuning documentation can be found in |
57043247 | 11 | * Documentation/admin-guide/sysctl/vm.rst. |
1da177e4 LT |
12 | * Started 18.12.91 |
13 | * Swap aging added 23.2.95, Stephen Tweedie. | |
14 | * Buffermem limits added 12.3.98, Rik van Riel. | |
15 | */ | |
16 | ||
17 | #include <linux/mm.h> | |
18 | #include <linux/sched.h> | |
19 | #include <linux/kernel_stat.h> | |
20 | #include <linux/swap.h> | |
21 | #include <linux/mman.h> | |
22 | #include <linux/pagemap.h> | |
23 | #include <linux/pagevec.h> | |
24 | #include <linux/init.h> | |
b95f1b31 | 25 | #include <linux/export.h> |
1da177e4 | 26 | #include <linux/mm_inline.h> |
1da177e4 | 27 | #include <linux/percpu_counter.h> |
3565fce3 | 28 | #include <linux/memremap.h> |
1da177e4 LT |
29 | #include <linux/percpu.h> |
30 | #include <linux/cpu.h> | |
31 | #include <linux/notifier.h> | |
e0bf68dd | 32 | #include <linux/backing-dev.h> |
66e1707b | 33 | #include <linux/memcontrol.h> |
5a0e3ad6 | 34 | #include <linux/gfp.h> |
a27bb332 | 35 | #include <linux/uio.h> |
822fc613 | 36 | #include <linux/hugetlb.h> |
33c3fc71 | 37 | #include <linux/page_idle.h> |
b01b2141 | 38 | #include <linux/local_lock.h> |
8cc621d2 | 39 | #include <linux/buffer_head.h> |
1da177e4 | 40 | |
64d6519d LS |
41 | #include "internal.h" |
42 | ||
c6286c98 MG |
43 | #define CREATE_TRACE_POINTS |
44 | #include <trace/events/pagemap.h> | |
45 | ||
1da177e4 LT |
46 | /* How many pages do we try to swap or page in/out together? */ |
47 | int page_cluster; | |
48 | ||
c2bc1681 | 49 | /* Protecting only lru_rotate.fbatch which requires disabling interrupts */ |
b01b2141 IM |
50 | struct lru_rotate { |
51 | local_lock_t lock; | |
c2bc1681 | 52 | struct folio_batch fbatch; |
b01b2141 IM |
53 | }; |
54 | static DEFINE_PER_CPU(struct lru_rotate, lru_rotate) = { | |
55 | .lock = INIT_LOCAL_LOCK(lock), | |
56 | }; | |
57 | ||
58 | /* | |
82ac64d8 | 59 | * The following folio batches are grouped together because they are protected |
b01b2141 IM |
60 | * by disabling preemption (and interrupts remain enabled). |
61 | */ | |
82ac64d8 | 62 | struct cpu_fbatches { |
b01b2141 | 63 | local_lock_t lock; |
70dea534 | 64 | struct folio_batch lru_add; |
7a3dbfe8 | 65 | struct folio_batch lru_deactivate_file; |
85cd7791 | 66 | struct folio_batch lru_deactivate; |
cec394ba | 67 | struct folio_batch lru_lazyfree; |
a4a921aa | 68 | #ifdef CONFIG_SMP |
3a44610b | 69 | struct folio_batch activate; |
a4a921aa | 70 | #endif |
b01b2141 | 71 | }; |
82ac64d8 | 72 | static DEFINE_PER_CPU(struct cpu_fbatches, cpu_fbatches) = { |
b01b2141 IM |
73 | .lock = INIT_LOCAL_LOCK(lock), |
74 | }; | |
902aaed0 | 75 | |
b221385b | 76 | /* |
b109b870 HD |
77 | * This path almost never happens for VM activity - pages are normally freed |
78 | * via pagevecs. But it gets used by networking - and for compound pages. | |
b221385b | 79 | */ |
188e8cae | 80 | static void __page_cache_release(struct folio *folio) |
b221385b | 81 | { |
188e8cae | 82 | if (folio_test_lru(folio)) { |
fa9add64 HD |
83 | struct lruvec *lruvec; |
84 | unsigned long flags; | |
b221385b | 85 | |
e809c3fe | 86 | lruvec = folio_lruvec_lock_irqsave(folio, &flags); |
188e8cae MWO |
87 | lruvec_del_folio(lruvec, folio); |
88 | __folio_clear_lru_flags(folio); | |
6168d0da | 89 | unlock_page_lruvec_irqrestore(lruvec, flags); |
b221385b | 90 | } |
188e8cae MWO |
91 | /* See comment on folio_test_mlocked in release_pages() */ |
92 | if (unlikely(folio_test_mlocked(folio))) { | |
93 | long nr_pages = folio_nr_pages(folio); | |
b109b870 | 94 | |
188e8cae MWO |
95 | __folio_clear_mlocked(folio); |
96 | zone_stat_mod_folio(folio, NR_MLOCK, -nr_pages); | |
b109b870 HD |
97 | count_vm_events(UNEVICTABLE_PGCLEARED, nr_pages); |
98 | } | |
91807063 AA |
99 | } |
100 | ||
83d99659 | 101 | static void __folio_put_small(struct folio *folio) |
91807063 | 102 | { |
188e8cae | 103 | __page_cache_release(folio); |
83d99659 MWO |
104 | mem_cgroup_uncharge(folio); |
105 | free_unref_page(&folio->page, 0); | |
b221385b AB |
106 | } |
107 | ||
5ef82fe7 | 108 | static void __folio_put_large(struct folio *folio) |
1da177e4 | 109 | { |
822fc613 NH |
110 | /* |
111 | * __page_cache_release() is supposed to be called for thp, not for | |
112 | * hugetlb. This is because hugetlb page does never have PageLRU set | |
113 | * (it's never listed to any LRU lists) and no memcg routines should | |
114 | * be called for hugetlb (it has a separate hugetlb_cgroup.) | |
115 | */ | |
5ef82fe7 | 116 | if (!folio_test_hugetlb(folio)) |
188e8cae | 117 | __page_cache_release(folio); |
5375336c | 118 | destroy_large_folio(folio); |
91807063 AA |
119 | } |
120 | ||
8d29c703 | 121 | void __folio_put(struct folio *folio) |
8519fb30 | 122 | { |
8d29c703 MWO |
123 | if (unlikely(folio_is_zone_device(folio))) |
124 | free_zone_device_page(&folio->page); | |
125 | else if (unlikely(folio_test_large(folio))) | |
5ef82fe7 | 126 | __folio_put_large(folio); |
ddc58f27 | 127 | else |
83d99659 | 128 | __folio_put_small(folio); |
1da177e4 | 129 | } |
8d29c703 | 130 | EXPORT_SYMBOL(__folio_put); |
70b50f94 | 131 | |
1d7ea732 | 132 | /** |
7682486b RD |
133 | * put_pages_list() - release a list of pages |
134 | * @pages: list of pages threaded on page->lru | |
1d7ea732 | 135 | * |
988c69f1 | 136 | * Release a list of pages which are strung together on page.lru. |
1d7ea732 AZ |
137 | */ |
138 | void put_pages_list(struct list_head *pages) | |
139 | { | |
2f58e5de | 140 | struct folio *folio, *next; |
988c69f1 | 141 | |
2f58e5de MWO |
142 | list_for_each_entry_safe(folio, next, pages, lru) { |
143 | if (!folio_put_testzero(folio)) { | |
144 | list_del(&folio->lru); | |
988c69f1 MWO |
145 | continue; |
146 | } | |
2f58e5de MWO |
147 | if (folio_test_large(folio)) { |
148 | list_del(&folio->lru); | |
5ef82fe7 | 149 | __folio_put_large(folio); |
988c69f1 MWO |
150 | continue; |
151 | } | |
2f58e5de | 152 | /* LRU flag must be clear because it's passed using the lru */ |
1d7ea732 | 153 | } |
988c69f1 MWO |
154 | |
155 | free_unref_page_list(pages); | |
3cd018b4 | 156 | INIT_LIST_HEAD(pages); |
1d7ea732 AZ |
157 | } |
158 | EXPORT_SYMBOL(put_pages_list); | |
159 | ||
18022c5d MG |
160 | /* |
161 | * get_kernel_pages() - pin kernel pages in memory | |
162 | * @kiov: An array of struct kvec structures | |
163 | * @nr_segs: number of segments to pin | |
164 | * @write: pinning for read/write, currently ignored | |
165 | * @pages: array that receives pointers to the pages pinned. | |
166 | * Should be at least nr_segs long. | |
167 | * | |
133d2743 ML |
168 | * Returns number of pages pinned. This may be fewer than the number requested. |
169 | * If nr_segs is 0 or negative, returns 0. If no pages were pinned, returns 0. | |
170 | * Each page returned must be released with a put_page() call when it is | |
171 | * finished with. | |
18022c5d MG |
172 | */ |
173 | int get_kernel_pages(const struct kvec *kiov, int nr_segs, int write, | |
174 | struct page **pages) | |
175 | { | |
176 | int seg; | |
177 | ||
178 | for (seg = 0; seg < nr_segs; seg++) { | |
179 | if (WARN_ON(kiov[seg].iov_len != PAGE_SIZE)) | |
180 | return seg; | |
181 | ||
5a178119 | 182 | pages[seg] = kmap_to_page(kiov[seg].iov_base); |
09cbfeaf | 183 | get_page(pages[seg]); |
18022c5d MG |
184 | } |
185 | ||
186 | return seg; | |
187 | } | |
188 | EXPORT_SYMBOL_GPL(get_kernel_pages); | |
189 | ||
c2bc1681 MWO |
190 | typedef void (*move_fn_t)(struct lruvec *lruvec, struct folio *folio); |
191 | ||
70dea534 | 192 | static void lru_add_fn(struct lruvec *lruvec, struct folio *folio) |
7d80dd09 MWO |
193 | { |
194 | int was_unevictable = folio_test_clear_unevictable(folio); | |
195 | long nr_pages = folio_nr_pages(folio); | |
196 | ||
197 | VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); | |
198 | ||
7d80dd09 MWO |
199 | /* |
200 | * Is an smp_mb__after_atomic() still required here, before | |
188e8cae | 201 | * folio_evictable() tests the mlocked flag, to rule out the possibility |
7d80dd09 | 202 | * of stranding an evictable folio on an unevictable LRU? I think |
188e8cae MWO |
203 | * not, because __munlock_page() only clears the mlocked flag |
204 | * while the LRU lock is held. | |
7d80dd09 MWO |
205 | * |
206 | * (That is not true of __page_cache_release(), and not necessarily | |
188e8cae MWO |
207 | * true of release_pages(): but those only clear the mlocked flag after |
208 | * folio_put_testzero() has excluded any other users of the folio.) | |
7d80dd09 MWO |
209 | */ |
210 | if (folio_evictable(folio)) { | |
211 | if (was_unevictable) | |
212 | __count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages); | |
213 | } else { | |
214 | folio_clear_active(folio); | |
215 | folio_set_unevictable(folio); | |
216 | /* | |
217 | * folio->mlock_count = !!folio_test_mlocked(folio)? | |
218 | * But that leaves __mlock_page() in doubt whether another | |
219 | * actor has already counted the mlock or not. Err on the | |
220 | * safe side, underestimate, let page reclaim fix it, rather | |
221 | * than leaving a page on the unevictable LRU indefinitely. | |
222 | */ | |
223 | folio->mlock_count = 0; | |
224 | if (!was_unevictable) | |
225 | __count_vm_events(UNEVICTABLE_PGCULLED, nr_pages); | |
226 | } | |
227 | ||
228 | lruvec_add_folio(lruvec, folio); | |
229 | trace_mm_lru_insertion(folio); | |
230 | } | |
231 | ||
c2bc1681 | 232 | static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn) |
902aaed0 HH |
233 | { |
234 | int i; | |
6168d0da | 235 | struct lruvec *lruvec = NULL; |
3dd7ae8e | 236 | unsigned long flags = 0; |
902aaed0 | 237 | |
c2bc1681 MWO |
238 | for (i = 0; i < folio_batch_count(fbatch); i++) { |
239 | struct folio *folio = fbatch->folios[i]; | |
3dd7ae8e | 240 | |
c2bc1681 | 241 | /* block memcg migration while the folio moves between lru */ |
70dea534 | 242 | if (move_fn != lru_add_fn && !folio_test_clear_lru(folio)) |
fc574c23 AS |
243 | continue; |
244 | ||
0de340cb | 245 | lruvec = folio_lruvec_relock_irqsave(folio, lruvec, &flags); |
c2bc1681 | 246 | move_fn(lruvec, folio); |
fc574c23 | 247 | |
c2bc1681 | 248 | folio_set_lru(folio); |
902aaed0 | 249 | } |
c2bc1681 | 250 | |
6168d0da AS |
251 | if (lruvec) |
252 | unlock_page_lruvec_irqrestore(lruvec, flags); | |
c2bc1681 MWO |
253 | folios_put(fbatch->folios, folio_batch_count(fbatch)); |
254 | folio_batch_init(fbatch); | |
d8505dee SL |
255 | } |
256 | ||
c2bc1681 MWO |
257 | static void folio_batch_add_and_move(struct folio_batch *fbatch, |
258 | struct folio *folio, move_fn_t move_fn) | |
3dd7ae8e | 259 | { |
c2bc1681 MWO |
260 | if (folio_batch_add(fbatch, folio) && !folio_test_large(folio) && |
261 | !lru_cache_disabled()) | |
262 | return; | |
263 | folio_batch_move_lru(fbatch, move_fn); | |
264 | } | |
575ced1c | 265 | |
c2bc1681 MWO |
266 | static void lru_move_tail_fn(struct lruvec *lruvec, struct folio *folio) |
267 | { | |
575ced1c MWO |
268 | if (!folio_test_unevictable(folio)) { |
269 | lruvec_del_folio(lruvec, folio); | |
270 | folio_clear_active(folio); | |
271 | lruvec_add_folio_tail(lruvec, folio); | |
272 | __count_vm_events(PGROTATED, folio_nr_pages(folio)); | |
3dd7ae8e SL |
273 | } |
274 | } | |
275 | ||
1da177e4 | 276 | /* |
575ced1c MWO |
277 | * Writeback is about to end against a folio which has been marked for |
278 | * immediate reclaim. If it still appears to be reclaimable, move it | |
279 | * to the tail of the inactive list. | |
c7c7b80c | 280 | * |
575ced1c | 281 | * folio_rotate_reclaimable() must disable IRQs, to prevent nasty races. |
1da177e4 | 282 | */ |
575ced1c | 283 | void folio_rotate_reclaimable(struct folio *folio) |
1da177e4 | 284 | { |
575ced1c MWO |
285 | if (!folio_test_locked(folio) && !folio_test_dirty(folio) && |
286 | !folio_test_unevictable(folio) && folio_test_lru(folio)) { | |
c2bc1681 | 287 | struct folio_batch *fbatch; |
ac6aadb2 MS |
288 | unsigned long flags; |
289 | ||
575ced1c | 290 | folio_get(folio); |
b01b2141 | 291 | local_lock_irqsave(&lru_rotate.lock, flags); |
c2bc1681 MWO |
292 | fbatch = this_cpu_ptr(&lru_rotate.fbatch); |
293 | folio_batch_add_and_move(fbatch, folio, lru_move_tail_fn); | |
b01b2141 | 294 | local_unlock_irqrestore(&lru_rotate.lock, flags); |
ac6aadb2 | 295 | } |
1da177e4 LT |
296 | } |
297 | ||
0538a82c JW |
298 | void lru_note_cost(struct lruvec *lruvec, bool file, |
299 | unsigned int nr_io, unsigned int nr_rotated) | |
3e2f41f1 | 300 | { |
0538a82c JW |
301 | unsigned long cost; |
302 | ||
303 | /* | |
304 | * Reflect the relative cost of incurring IO and spending CPU | |
305 | * time on rotations. This doesn't attempt to make a precise | |
306 | * comparison, it just says: if reloads are about comparable | |
307 | * between the LRU lists, or rotations are overwhelmingly | |
308 | * different between them, adjust scan balance for CPU work. | |
309 | */ | |
310 | cost = nr_io * SWAP_CLUSTER_MAX + nr_rotated; | |
311 | ||
7cf111bc JW |
312 | do { |
313 | unsigned long lrusize; | |
314 | ||
6168d0da AS |
315 | /* |
316 | * Hold lruvec->lru_lock is safe here, since | |
317 | * 1) The pinned lruvec in reclaim, or | |
318 | * 2) From a pre-LRU page during refault (which also holds the | |
319 | * rcu lock, so would be safe even if the page was on the LRU | |
320 | * and could move simultaneously to a new lruvec). | |
321 | */ | |
322 | spin_lock_irq(&lruvec->lru_lock); | |
7cf111bc | 323 | /* Record cost event */ |
96f8bf4f | 324 | if (file) |
0538a82c | 325 | lruvec->file_cost += cost; |
7cf111bc | 326 | else |
0538a82c | 327 | lruvec->anon_cost += cost; |
7cf111bc JW |
328 | |
329 | /* | |
330 | * Decay previous events | |
331 | * | |
332 | * Because workloads change over time (and to avoid | |
333 | * overflow) we keep these statistics as a floating | |
334 | * average, which ends up weighing recent refaults | |
335 | * more than old ones. | |
336 | */ | |
337 | lrusize = lruvec_page_state(lruvec, NR_INACTIVE_ANON) + | |
338 | lruvec_page_state(lruvec, NR_ACTIVE_ANON) + | |
339 | lruvec_page_state(lruvec, NR_INACTIVE_FILE) + | |
340 | lruvec_page_state(lruvec, NR_ACTIVE_FILE); | |
341 | ||
342 | if (lruvec->file_cost + lruvec->anon_cost > lrusize / 4) { | |
343 | lruvec->file_cost /= 2; | |
344 | lruvec->anon_cost /= 2; | |
345 | } | |
6168d0da | 346 | spin_unlock_irq(&lruvec->lru_lock); |
7cf111bc | 347 | } while ((lruvec = parent_lruvec(lruvec))); |
3e2f41f1 KM |
348 | } |
349 | ||
0538a82c | 350 | void lru_note_cost_refault(struct folio *folio) |
96f8bf4f | 351 | { |
0995d7e5 | 352 | lru_note_cost(folio_lruvec(folio), folio_is_file_lru(folio), |
0538a82c | 353 | folio_nr_pages(folio), 0); |
96f8bf4f JW |
354 | } |
355 | ||
3a44610b | 356 | static void folio_activate_fn(struct lruvec *lruvec, struct folio *folio) |
1da177e4 | 357 | { |
f2d27392 MWO |
358 | if (!folio_test_active(folio) && !folio_test_unevictable(folio)) { |
359 | long nr_pages = folio_nr_pages(folio); | |
744ed144 | 360 | |
f2d27392 MWO |
361 | lruvec_del_folio(lruvec, folio); |
362 | folio_set_active(folio); | |
363 | lruvec_add_folio(lruvec, folio); | |
364 | trace_mm_lru_activate(folio); | |
4f98a2fe | 365 | |
21e330fc SB |
366 | __count_vm_events(PGACTIVATE, nr_pages); |
367 | __count_memcg_events(lruvec_memcg(lruvec), PGACTIVATE, | |
368 | nr_pages); | |
1da177e4 | 369 | } |
eb709b0d SL |
370 | } |
371 | ||
372 | #ifdef CONFIG_SMP | |
3a44610b | 373 | static void folio_activate_drain(int cpu) |
f2d27392 | 374 | { |
82ac64d8 | 375 | struct folio_batch *fbatch = &per_cpu(cpu_fbatches.activate, cpu); |
f2d27392 | 376 | |
3a44610b MWO |
377 | if (folio_batch_count(fbatch)) |
378 | folio_batch_move_lru(fbatch, folio_activate_fn); | |
5fbc4616 CM |
379 | } |
380 | ||
018ee47f | 381 | void folio_activate(struct folio *folio) |
eb709b0d | 382 | { |
f2d27392 MWO |
383 | if (folio_test_lru(folio) && !folio_test_active(folio) && |
384 | !folio_test_unevictable(folio)) { | |
3a44610b | 385 | struct folio_batch *fbatch; |
eb709b0d | 386 | |
f2d27392 | 387 | folio_get(folio); |
82ac64d8 MWO |
388 | local_lock(&cpu_fbatches.lock); |
389 | fbatch = this_cpu_ptr(&cpu_fbatches.activate); | |
3a44610b | 390 | folio_batch_add_and_move(fbatch, folio, folio_activate_fn); |
82ac64d8 | 391 | local_unlock(&cpu_fbatches.lock); |
eb709b0d SL |
392 | } |
393 | } | |
394 | ||
395 | #else | |
3a44610b | 396 | static inline void folio_activate_drain(int cpu) |
eb709b0d SL |
397 | { |
398 | } | |
399 | ||
018ee47f | 400 | void folio_activate(struct folio *folio) |
eb709b0d | 401 | { |
6168d0da | 402 | struct lruvec *lruvec; |
eb709b0d | 403 | |
f2d27392 | 404 | if (folio_test_clear_lru(folio)) { |
e809c3fe | 405 | lruvec = folio_lruvec_lock_irq(folio); |
3a44610b | 406 | folio_activate_fn(lruvec, folio); |
6168d0da | 407 | unlock_page_lruvec_irq(lruvec); |
f2d27392 | 408 | folio_set_lru(folio); |
6168d0da | 409 | } |
1da177e4 | 410 | } |
eb709b0d | 411 | #endif |
1da177e4 | 412 | |
76580b65 | 413 | static void __lru_cache_activate_folio(struct folio *folio) |
059285a2 | 414 | { |
70dea534 | 415 | struct folio_batch *fbatch; |
059285a2 MG |
416 | int i; |
417 | ||
82ac64d8 MWO |
418 | local_lock(&cpu_fbatches.lock); |
419 | fbatch = this_cpu_ptr(&cpu_fbatches.lru_add); | |
b01b2141 | 420 | |
059285a2 | 421 | /* |
70dea534 MWO |
422 | * Search backwards on the optimistic assumption that the folio being |
423 | * activated has just been added to this batch. Note that only | |
424 | * the local batch is examined as a !LRU folio could be in the | |
059285a2 | 425 | * process of being released, reclaimed, migrated or on a remote |
70dea534 MWO |
426 | * batch that is currently being drained. Furthermore, marking |
427 | * a remote batch's folio active potentially hits a race where | |
428 | * a folio is marked active just after it is added to the inactive | |
059285a2 MG |
429 | * list causing accounting errors and BUG_ON checks to trigger. |
430 | */ | |
70dea534 MWO |
431 | for (i = folio_batch_count(fbatch) - 1; i >= 0; i--) { |
432 | struct folio *batch_folio = fbatch->folios[i]; | |
059285a2 | 433 | |
70dea534 | 434 | if (batch_folio == folio) { |
76580b65 | 435 | folio_set_active(folio); |
059285a2 MG |
436 | break; |
437 | } | |
438 | } | |
439 | ||
82ac64d8 | 440 | local_unlock(&cpu_fbatches.lock); |
059285a2 MG |
441 | } |
442 | ||
ac35a490 YZ |
443 | #ifdef CONFIG_LRU_GEN |
444 | static void folio_inc_refs(struct folio *folio) | |
445 | { | |
446 | unsigned long new_flags, old_flags = READ_ONCE(folio->flags); | |
447 | ||
448 | if (folio_test_unevictable(folio)) | |
449 | return; | |
450 | ||
451 | if (!folio_test_referenced(folio)) { | |
452 | folio_set_referenced(folio); | |
453 | return; | |
454 | } | |
455 | ||
456 | if (!folio_test_workingset(folio)) { | |
457 | folio_set_workingset(folio); | |
458 | return; | |
459 | } | |
460 | ||
461 | /* see the comment on MAX_NR_TIERS */ | |
462 | do { | |
463 | new_flags = old_flags & LRU_REFS_MASK; | |
464 | if (new_flags == LRU_REFS_MASK) | |
465 | break; | |
466 | ||
467 | new_flags += BIT(LRU_REFS_PGOFF); | |
468 | new_flags |= old_flags & ~LRU_REFS_MASK; | |
469 | } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); | |
470 | } | |
471 | #else | |
472 | static void folio_inc_refs(struct folio *folio) | |
473 | { | |
474 | } | |
475 | #endif /* CONFIG_LRU_GEN */ | |
476 | ||
1da177e4 LT |
477 | /* |
478 | * Mark a page as having seen activity. | |
479 | * | |
480 | * inactive,unreferenced -> inactive,referenced | |
481 | * inactive,referenced -> active,unreferenced | |
482 | * active,unreferenced -> active,referenced | |
eb39d618 HD |
483 | * |
484 | * When a newly allocated page is not yet visible, so safe for non-atomic ops, | |
485 | * __SetPageReferenced(page) may be substituted for mark_page_accessed(page). | |
1da177e4 | 486 | */ |
76580b65 | 487 | void folio_mark_accessed(struct folio *folio) |
1da177e4 | 488 | { |
ac35a490 YZ |
489 | if (lru_gen_enabled()) { |
490 | folio_inc_refs(folio); | |
491 | return; | |
492 | } | |
493 | ||
76580b65 MWO |
494 | if (!folio_test_referenced(folio)) { |
495 | folio_set_referenced(folio); | |
496 | } else if (folio_test_unevictable(folio)) { | |
a1100a74 FW |
497 | /* |
498 | * Unevictable pages are on the "LRU_UNEVICTABLE" list. But, | |
499 | * this list is never rotated or maintained, so marking an | |
914c32e4 | 500 | * unevictable page accessed has no effect. |
a1100a74 | 501 | */ |
76580b65 | 502 | } else if (!folio_test_active(folio)) { |
059285a2 | 503 | /* |
3a44610b | 504 | * If the folio is on the LRU, queue it for activation via |
82ac64d8 | 505 | * cpu_fbatches.activate. Otherwise, assume the folio is in a |
3a44610b | 506 | * folio_batch, mark it active and it'll be moved to the active |
059285a2 MG |
507 | * LRU on the next drain. |
508 | */ | |
76580b65 MWO |
509 | if (folio_test_lru(folio)) |
510 | folio_activate(folio); | |
059285a2 | 511 | else |
76580b65 MWO |
512 | __lru_cache_activate_folio(folio); |
513 | folio_clear_referenced(folio); | |
514 | workingset_activation(folio); | |
1da177e4 | 515 | } |
76580b65 MWO |
516 | if (folio_test_idle(folio)) |
517 | folio_clear_idle(folio); | |
1da177e4 | 518 | } |
76580b65 | 519 | EXPORT_SYMBOL(folio_mark_accessed); |
1da177e4 | 520 | |
f04e9ebb | 521 | /** |
0d31125d MWO |
522 | * folio_add_lru - Add a folio to an LRU list. |
523 | * @folio: The folio to be added to the LRU. | |
2329d375 | 524 | * |
0d31125d | 525 | * Queue the folio for addition to the LRU. The decision on whether |
2329d375 | 526 | * to add the page to the [in]active [file|anon] list is deferred until the |
82ac64d8 | 527 | * folio_batch is drained. This gives a chance for the caller of folio_add_lru() |
0d31125d | 528 | * have the folio added to the active list using folio_mark_accessed(). |
f04e9ebb | 529 | */ |
0d31125d | 530 | void folio_add_lru(struct folio *folio) |
1da177e4 | 531 | { |
70dea534 | 532 | struct folio_batch *fbatch; |
6058eaec | 533 | |
70dea534 MWO |
534 | VM_BUG_ON_FOLIO(folio_test_active(folio) && |
535 | folio_test_unevictable(folio), folio); | |
0d31125d | 536 | VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); |
6058eaec | 537 | |
ec1c86b2 YZ |
538 | /* see the comment in lru_gen_add_folio() */ |
539 | if (lru_gen_enabled() && !folio_test_unevictable(folio) && | |
540 | lru_gen_in_fault() && !(current->flags & PF_MEMALLOC)) | |
541 | folio_set_active(folio); | |
542 | ||
0d31125d | 543 | folio_get(folio); |
82ac64d8 MWO |
544 | local_lock(&cpu_fbatches.lock); |
545 | fbatch = this_cpu_ptr(&cpu_fbatches.lru_add); | |
70dea534 | 546 | folio_batch_add_and_move(fbatch, folio, lru_add_fn); |
82ac64d8 | 547 | local_unlock(&cpu_fbatches.lock); |
1da177e4 | 548 | } |
0d31125d | 549 | EXPORT_SYMBOL(folio_add_lru); |
1da177e4 | 550 | |
00501b53 | 551 | /** |
681ecf63 MWO |
552 | * folio_add_lru_vma() - Add a folio to the appropate LRU list for this VMA. |
553 | * @folio: The folio to be added to the LRU. | |
554 | * @vma: VMA in which the folio is mapped. | |
00501b53 | 555 | * |
681ecf63 MWO |
556 | * If the VMA is mlocked, @folio is added to the unevictable list. |
557 | * Otherwise, it is treated the same way as folio_add_lru(). | |
00501b53 | 558 | */ |
681ecf63 | 559 | void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma) |
00501b53 | 560 | { |
681ecf63 | 561 | VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); |
00501b53 | 562 | |
2fbb0c10 | 563 | if (unlikely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED)) |
681ecf63 | 564 | mlock_new_page(&folio->page); |
2fbb0c10 | 565 | else |
681ecf63 | 566 | folio_add_lru(folio); |
00501b53 JW |
567 | } |
568 | ||
31560180 | 569 | /* |
7a3dbfe8 | 570 | * If the folio cannot be invalidated, it is moved to the |
31560180 MK |
571 | * inactive list to speed up its reclaim. It is moved to the |
572 | * head of the list, rather than the tail, to give the flusher | |
573 | * threads some time to write it out, as this is much more | |
574 | * effective than the single-page writeout from reclaim. | |
278df9f4 | 575 | * |
7a3dbfe8 MWO |
576 | * If the folio isn't mapped and dirty/writeback, the folio |
577 | * could be reclaimed asap using the reclaim flag. | |
278df9f4 | 578 | * |
7a3dbfe8 MWO |
579 | * 1. active, mapped folio -> none |
580 | * 2. active, dirty/writeback folio -> inactive, head, reclaim | |
581 | * 3. inactive, mapped folio -> none | |
582 | * 4. inactive, dirty/writeback folio -> inactive, head, reclaim | |
278df9f4 MK |
583 | * 5. inactive, clean -> inactive, tail |
584 | * 6. Others -> none | |
585 | * | |
7a3dbfe8 MWO |
586 | * In 4, it moves to the head of the inactive list so the folio is |
587 | * written out by flusher threads as this is much more efficient | |
278df9f4 | 588 | * than the single-page writeout from reclaim. |
31560180 | 589 | */ |
7a3dbfe8 | 590 | static void lru_deactivate_file_fn(struct lruvec *lruvec, struct folio *folio) |
31560180 | 591 | { |
7a3dbfe8 MWO |
592 | bool active = folio_test_active(folio); |
593 | long nr_pages = folio_nr_pages(folio); | |
31560180 | 594 | |
7a3dbfe8 | 595 | if (folio_test_unevictable(folio)) |
bad49d9c MK |
596 | return; |
597 | ||
7a3dbfe8 MWO |
598 | /* Some processes are using the folio */ |
599 | if (folio_mapped(folio)) | |
31560180 MK |
600 | return; |
601 | ||
7a3dbfe8 MWO |
602 | lruvec_del_folio(lruvec, folio); |
603 | folio_clear_active(folio); | |
604 | folio_clear_referenced(folio); | |
31560180 | 605 | |
7a3dbfe8 | 606 | if (folio_test_writeback(folio) || folio_test_dirty(folio)) { |
278df9f4 | 607 | /* |
7a3dbfe8 MWO |
608 | * Setting the reclaim flag could race with |
609 | * folio_end_writeback() and confuse readahead. But the | |
610 | * race window is _really_ small and it's not a critical | |
611 | * problem. | |
278df9f4 | 612 | */ |
7a3dbfe8 MWO |
613 | lruvec_add_folio(lruvec, folio); |
614 | folio_set_reclaim(folio); | |
278df9f4 MK |
615 | } else { |
616 | /* | |
7a3dbfe8 MWO |
617 | * The folio's writeback ended while it was in the batch. |
618 | * We move that folio to the tail of the inactive list. | |
278df9f4 | 619 | */ |
7a3dbfe8 | 620 | lruvec_add_folio_tail(lruvec, folio); |
5d91f31f | 621 | __count_vm_events(PGROTATED, nr_pages); |
278df9f4 MK |
622 | } |
623 | ||
21e330fc | 624 | if (active) { |
5d91f31f | 625 | __count_vm_events(PGDEACTIVATE, nr_pages); |
21e330fc SB |
626 | __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, |
627 | nr_pages); | |
628 | } | |
31560180 MK |
629 | } |
630 | ||
85cd7791 | 631 | static void lru_deactivate_fn(struct lruvec *lruvec, struct folio *folio) |
9c276cc6 | 632 | { |
ec1c86b2 | 633 | if (!folio_test_unevictable(folio) && (folio_test_active(folio) || lru_gen_enabled())) { |
85cd7791 | 634 | long nr_pages = folio_nr_pages(folio); |
9c276cc6 | 635 | |
85cd7791 MWO |
636 | lruvec_del_folio(lruvec, folio); |
637 | folio_clear_active(folio); | |
638 | folio_clear_referenced(folio); | |
639 | lruvec_add_folio(lruvec, folio); | |
9c276cc6 | 640 | |
21e330fc SB |
641 | __count_vm_events(PGDEACTIVATE, nr_pages); |
642 | __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, | |
643 | nr_pages); | |
9c276cc6 MK |
644 | } |
645 | } | |
10853a03 | 646 | |
cec394ba | 647 | static void lru_lazyfree_fn(struct lruvec *lruvec, struct folio *folio) |
10853a03 | 648 | { |
cec394ba MWO |
649 | if (folio_test_anon(folio) && folio_test_swapbacked(folio) && |
650 | !folio_test_swapcache(folio) && !folio_test_unevictable(folio)) { | |
651 | long nr_pages = folio_nr_pages(folio); | |
10853a03 | 652 | |
cec394ba MWO |
653 | lruvec_del_folio(lruvec, folio); |
654 | folio_clear_active(folio); | |
655 | folio_clear_referenced(folio); | |
f7ad2a6c | 656 | /* |
cec394ba MWO |
657 | * Lazyfree folios are clean anonymous folios. They have |
658 | * the swapbacked flag cleared, to distinguish them from normal | |
659 | * anonymous folios | |
f7ad2a6c | 660 | */ |
cec394ba MWO |
661 | folio_clear_swapbacked(folio); |
662 | lruvec_add_folio(lruvec, folio); | |
10853a03 | 663 | |
21e330fc SB |
664 | __count_vm_events(PGLAZYFREE, nr_pages); |
665 | __count_memcg_events(lruvec_memcg(lruvec), PGLAZYFREE, | |
666 | nr_pages); | |
10853a03 MK |
667 | } |
668 | } | |
669 | ||
902aaed0 | 670 | /* |
82ac64d8 | 671 | * Drain pages out of the cpu's folio_batch. |
902aaed0 HH |
672 | * Either "cpu" is the current CPU, and preemption has already been |
673 | * disabled; or "cpu" is being hot-unplugged, and is already dead. | |
674 | */ | |
f0cb3c76 | 675 | void lru_add_drain_cpu(int cpu) |
1da177e4 | 676 | { |
a2d33b5d MWO |
677 | struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu); |
678 | struct folio_batch *fbatch = &fbatches->lru_add; | |
1da177e4 | 679 | |
70dea534 MWO |
680 | if (folio_batch_count(fbatch)) |
681 | folio_batch_move_lru(fbatch, lru_add_fn); | |
902aaed0 | 682 | |
c2bc1681 | 683 | fbatch = &per_cpu(lru_rotate.fbatch, cpu); |
7e0cc01e | 684 | /* Disabling interrupts below acts as a compiler barrier. */ |
c2bc1681 | 685 | if (data_race(folio_batch_count(fbatch))) { |
902aaed0 HH |
686 | unsigned long flags; |
687 | ||
688 | /* No harm done if a racing interrupt already did this */ | |
b01b2141 | 689 | local_lock_irqsave(&lru_rotate.lock, flags); |
c2bc1681 | 690 | folio_batch_move_lru(fbatch, lru_move_tail_fn); |
b01b2141 | 691 | local_unlock_irqrestore(&lru_rotate.lock, flags); |
902aaed0 | 692 | } |
31560180 | 693 | |
a2d33b5d | 694 | fbatch = &fbatches->lru_deactivate_file; |
7a3dbfe8 MWO |
695 | if (folio_batch_count(fbatch)) |
696 | folio_batch_move_lru(fbatch, lru_deactivate_file_fn); | |
eb709b0d | 697 | |
a2d33b5d | 698 | fbatch = &fbatches->lru_deactivate; |
85cd7791 MWO |
699 | if (folio_batch_count(fbatch)) |
700 | folio_batch_move_lru(fbatch, lru_deactivate_fn); | |
9c276cc6 | 701 | |
a2d33b5d | 702 | fbatch = &fbatches->lru_lazyfree; |
cec394ba MWO |
703 | if (folio_batch_count(fbatch)) |
704 | folio_batch_move_lru(fbatch, lru_lazyfree_fn); | |
10853a03 | 705 | |
3a44610b | 706 | folio_activate_drain(cpu); |
31560180 MK |
707 | } |
708 | ||
709 | /** | |
7a3dbfe8 | 710 | * deactivate_file_folio() - Deactivate a file folio. |
261b6840 | 711 | * @folio: Folio to deactivate. |
31560180 | 712 | * |
261b6840 MWO |
713 | * This function hints to the VM that @folio is a good reclaim candidate, |
714 | * for example if its invalidation fails due to the folio being dirty | |
31560180 | 715 | * or under writeback. |
261b6840 | 716 | * |
7a3dbfe8 | 717 | * Context: Caller holds a reference on the folio. |
31560180 | 718 | */ |
261b6840 | 719 | void deactivate_file_folio(struct folio *folio) |
31560180 | 720 | { |
7a3dbfe8 | 721 | struct folio_batch *fbatch; |
261b6840 | 722 | |
7a3dbfe8 | 723 | /* Deactivating an unevictable folio will not accelerate reclaim */ |
261b6840 | 724 | if (folio_test_unevictable(folio)) |
821ed6bb MK |
725 | return; |
726 | ||
261b6840 | 727 | folio_get(folio); |
82ac64d8 MWO |
728 | local_lock(&cpu_fbatches.lock); |
729 | fbatch = this_cpu_ptr(&cpu_fbatches.lru_deactivate_file); | |
7a3dbfe8 | 730 | folio_batch_add_and_move(fbatch, folio, lru_deactivate_file_fn); |
82ac64d8 | 731 | local_unlock(&cpu_fbatches.lock); |
80bfed90 AM |
732 | } |
733 | ||
9c276cc6 MK |
734 | /* |
735 | * deactivate_page - deactivate a page | |
736 | * @page: page to deactivate | |
737 | * | |
738 | * deactivate_page() moves @page to the inactive list if @page was on the active | |
739 | * list and was not an unevictable page. This is done to accelerate the reclaim | |
740 | * of @page. | |
741 | */ | |
742 | void deactivate_page(struct page *page) | |
743 | { | |
85cd7791 | 744 | struct folio *folio = page_folio(page); |
9c276cc6 | 745 | |
ec1c86b2 YZ |
746 | if (folio_test_lru(folio) && !folio_test_unevictable(folio) && |
747 | (folio_test_active(folio) || lru_gen_enabled())) { | |
85cd7791 MWO |
748 | struct folio_batch *fbatch; |
749 | ||
750 | folio_get(folio); | |
82ac64d8 MWO |
751 | local_lock(&cpu_fbatches.lock); |
752 | fbatch = this_cpu_ptr(&cpu_fbatches.lru_deactivate); | |
85cd7791 | 753 | folio_batch_add_and_move(fbatch, folio, lru_deactivate_fn); |
82ac64d8 | 754 | local_unlock(&cpu_fbatches.lock); |
9c276cc6 MK |
755 | } |
756 | } | |
757 | ||
10853a03 | 758 | /** |
f7ad2a6c | 759 | * mark_page_lazyfree - make an anon page lazyfree |
10853a03 MK |
760 | * @page: page to deactivate |
761 | * | |
f7ad2a6c SL |
762 | * mark_page_lazyfree() moves @page to the inactive file list. |
763 | * This is done to accelerate the reclaim of @page. | |
10853a03 | 764 | */ |
f7ad2a6c | 765 | void mark_page_lazyfree(struct page *page) |
10853a03 | 766 | { |
cec394ba | 767 | struct folio *folio = page_folio(page); |
10853a03 | 768 | |
cec394ba MWO |
769 | if (folio_test_lru(folio) && folio_test_anon(folio) && |
770 | folio_test_swapbacked(folio) && !folio_test_swapcache(folio) && | |
771 | !folio_test_unevictable(folio)) { | |
772 | struct folio_batch *fbatch; | |
773 | ||
774 | folio_get(folio); | |
82ac64d8 MWO |
775 | local_lock(&cpu_fbatches.lock); |
776 | fbatch = this_cpu_ptr(&cpu_fbatches.lru_lazyfree); | |
cec394ba | 777 | folio_batch_add_and_move(fbatch, folio, lru_lazyfree_fn); |
82ac64d8 | 778 | local_unlock(&cpu_fbatches.lock); |
10853a03 MK |
779 | } |
780 | } | |
781 | ||
80bfed90 AM |
782 | void lru_add_drain(void) |
783 | { | |
82ac64d8 | 784 | local_lock(&cpu_fbatches.lock); |
b01b2141 | 785 | lru_add_drain_cpu(smp_processor_id()); |
82ac64d8 | 786 | local_unlock(&cpu_fbatches.lock); |
adb11e78 | 787 | mlock_page_drain_local(); |
b01b2141 IM |
788 | } |
789 | ||
243418e3 MK |
790 | /* |
791 | * It's called from per-cpu workqueue context in SMP case so | |
792 | * lru_add_drain_cpu and invalidate_bh_lrus_cpu should run on | |
793 | * the same cpu. It shouldn't be a problem in !SMP case since | |
794 | * the core is only one and the locks will disable preemption. | |
795 | */ | |
796 | static void lru_add_and_bh_lrus_drain(void) | |
797 | { | |
82ac64d8 | 798 | local_lock(&cpu_fbatches.lock); |
243418e3 | 799 | lru_add_drain_cpu(smp_processor_id()); |
82ac64d8 | 800 | local_unlock(&cpu_fbatches.lock); |
243418e3 | 801 | invalidate_bh_lrus_cpu(); |
adb11e78 | 802 | mlock_page_drain_local(); |
243418e3 MK |
803 | } |
804 | ||
b01b2141 IM |
805 | void lru_add_drain_cpu_zone(struct zone *zone) |
806 | { | |
82ac64d8 | 807 | local_lock(&cpu_fbatches.lock); |
b01b2141 IM |
808 | lru_add_drain_cpu(smp_processor_id()); |
809 | drain_local_pages(zone); | |
82ac64d8 | 810 | local_unlock(&cpu_fbatches.lock); |
adb11e78 | 811 | mlock_page_drain_local(); |
1da177e4 LT |
812 | } |
813 | ||
6ea183d6 MH |
814 | #ifdef CONFIG_SMP |
815 | ||
816 | static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work); | |
817 | ||
c4028958 | 818 | static void lru_add_drain_per_cpu(struct work_struct *dummy) |
053837fc | 819 | { |
243418e3 | 820 | lru_add_and_bh_lrus_drain(); |
053837fc NP |
821 | } |
822 | ||
4864545a MWO |
823 | static bool cpu_needs_drain(unsigned int cpu) |
824 | { | |
825 | struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu); | |
826 | ||
827 | /* Check these in order of likelihood that they're not zero */ | |
828 | return folio_batch_count(&fbatches->lru_add) || | |
829 | data_race(folio_batch_count(&per_cpu(lru_rotate.fbatch, cpu))) || | |
830 | folio_batch_count(&fbatches->lru_deactivate_file) || | |
831 | folio_batch_count(&fbatches->lru_deactivate) || | |
832 | folio_batch_count(&fbatches->lru_lazyfree) || | |
833 | folio_batch_count(&fbatches->activate) || | |
834 | need_mlock_page_drain(cpu) || | |
835 | has_bh_in_lru(cpu, NULL); | |
836 | } | |
837 | ||
9852a721 MH |
838 | /* |
839 | * Doesn't need any cpu hotplug locking because we do rely on per-cpu | |
840 | * kworkers being shut down before our page_alloc_cpu_dead callback is | |
841 | * executed on the offlined cpu. | |
842 | * Calling this function with cpu hotplug locks held can actually lead | |
843 | * to obscure indirect dependencies via WQ context. | |
844 | */ | |
3db3264d | 845 | static inline void __lru_add_drain_all(bool force_all_cpus) |
053837fc | 846 | { |
6446a513 AD |
847 | /* |
848 | * lru_drain_gen - Global pages generation number | |
849 | * | |
850 | * (A) Definition: global lru_drain_gen = x implies that all generations | |
851 | * 0 < n <= x are already *scheduled* for draining. | |
852 | * | |
853 | * This is an optimization for the highly-contended use case where a | |
854 | * user space workload keeps constantly generating a flow of pages for | |
855 | * each CPU. | |
856 | */ | |
857 | static unsigned int lru_drain_gen; | |
5fbc4616 | 858 | static struct cpumask has_work; |
6446a513 AD |
859 | static DEFINE_MUTEX(lock); |
860 | unsigned cpu, this_gen; | |
5fbc4616 | 861 | |
ce612879 MH |
862 | /* |
863 | * Make sure nobody triggers this path before mm_percpu_wq is fully | |
864 | * initialized. | |
865 | */ | |
866 | if (WARN_ON(!mm_percpu_wq)) | |
867 | return; | |
868 | ||
6446a513 | 869 | /* |
82ac64d8 MWO |
870 | * Guarantee folio_batch counter stores visible by this CPU |
871 | * are visible to other CPUs before loading the current drain | |
872 | * generation. | |
6446a513 AD |
873 | */ |
874 | smp_mb(); | |
875 | ||
876 | /* | |
877 | * (B) Locally cache global LRU draining generation number | |
878 | * | |
879 | * The read barrier ensures that the counter is loaded before the mutex | |
880 | * is taken. It pairs with smp_mb() inside the mutex critical section | |
881 | * at (D). | |
882 | */ | |
883 | this_gen = smp_load_acquire(&lru_drain_gen); | |
eef1a429 | 884 | |
5fbc4616 | 885 | mutex_lock(&lock); |
eef1a429 KK |
886 | |
887 | /* | |
6446a513 AD |
888 | * (C) Exit the draining operation if a newer generation, from another |
889 | * lru_add_drain_all(), was already scheduled for draining. Check (A). | |
eef1a429 | 890 | */ |
d479960e | 891 | if (unlikely(this_gen != lru_drain_gen && !force_all_cpus)) |
eef1a429 KK |
892 | goto done; |
893 | ||
6446a513 AD |
894 | /* |
895 | * (D) Increment global generation number | |
896 | * | |
897 | * Pairs with smp_load_acquire() at (B), outside of the critical | |
82ac64d8 MWO |
898 | * section. Use a full memory barrier to guarantee that the |
899 | * new global drain generation number is stored before loading | |
900 | * folio_batch counters. | |
6446a513 AD |
901 | * |
902 | * This pairing must be done here, before the for_each_online_cpu loop | |
903 | * below which drains the page vectors. | |
904 | * | |
905 | * Let x, y, and z represent some system CPU numbers, where x < y < z. | |
cb152a1a | 906 | * Assume CPU #z is in the middle of the for_each_online_cpu loop |
6446a513 AD |
907 | * below and has already reached CPU #y's per-cpu data. CPU #x comes |
908 | * along, adds some pages to its per-cpu vectors, then calls | |
909 | * lru_add_drain_all(). | |
910 | * | |
911 | * If the paired barrier is done at any later step, e.g. after the | |
912 | * loop, CPU #x will just exit at (C) and miss flushing out all of its | |
913 | * added pages. | |
914 | */ | |
915 | WRITE_ONCE(lru_drain_gen, lru_drain_gen + 1); | |
916 | smp_mb(); | |
eef1a429 | 917 | |
5fbc4616 | 918 | cpumask_clear(&has_work); |
5fbc4616 CM |
919 | for_each_online_cpu(cpu) { |
920 | struct work_struct *work = &per_cpu(lru_add_drain_work, cpu); | |
921 | ||
4864545a | 922 | if (cpu_needs_drain(cpu)) { |
5fbc4616 | 923 | INIT_WORK(work, lru_add_drain_per_cpu); |
ce612879 | 924 | queue_work_on(cpu, mm_percpu_wq, work); |
6446a513 | 925 | __cpumask_set_cpu(cpu, &has_work); |
5fbc4616 CM |
926 | } |
927 | } | |
928 | ||
929 | for_each_cpu(cpu, &has_work) | |
930 | flush_work(&per_cpu(lru_add_drain_work, cpu)); | |
931 | ||
eef1a429 | 932 | done: |
5fbc4616 | 933 | mutex_unlock(&lock); |
053837fc | 934 | } |
d479960e MK |
935 | |
936 | void lru_add_drain_all(void) | |
937 | { | |
938 | __lru_add_drain_all(false); | |
939 | } | |
6ea183d6 MH |
940 | #else |
941 | void lru_add_drain_all(void) | |
942 | { | |
943 | lru_add_drain(); | |
944 | } | |
6446a513 | 945 | #endif /* CONFIG_SMP */ |
053837fc | 946 | |
d479960e MK |
947 | atomic_t lru_disable_count = ATOMIC_INIT(0); |
948 | ||
949 | /* | |
950 | * lru_cache_disable() needs to be called before we start compiling | |
951 | * a list of pages to be migrated using isolate_lru_page(). | |
952 | * It drains pages on LRU cache and then disable on all cpus until | |
953 | * lru_cache_enable is called. | |
954 | * | |
955 | * Must be paired with a call to lru_cache_enable(). | |
956 | */ | |
957 | void lru_cache_disable(void) | |
958 | { | |
959 | atomic_inc(&lru_disable_count); | |
d479960e | 960 | /* |
ff042f4a MT |
961 | * Readers of lru_disable_count are protected by either disabling |
962 | * preemption or rcu_read_lock: | |
963 | * | |
964 | * preempt_disable, local_irq_disable [bh_lru_lock()] | |
965 | * rcu_read_lock [rt_spin_lock CONFIG_PREEMPT_RT] | |
966 | * preempt_disable [local_lock !CONFIG_PREEMPT_RT] | |
967 | * | |
968 | * Since v5.1 kernel, synchronize_rcu() is guaranteed to wait on | |
969 | * preempt_disable() regions of code. So any CPU which sees | |
970 | * lru_disable_count = 0 will have exited the critical | |
971 | * section when synchronize_rcu() returns. | |
d479960e | 972 | */ |
31733463 | 973 | synchronize_rcu_expedited(); |
ff042f4a | 974 | #ifdef CONFIG_SMP |
d479960e MK |
975 | __lru_add_drain_all(true); |
976 | #else | |
243418e3 | 977 | lru_add_and_bh_lrus_drain(); |
d479960e MK |
978 | #endif |
979 | } | |
980 | ||
aabfb572 | 981 | /** |
ea1754a0 | 982 | * release_pages - batched put_page() |
aabfb572 MH |
983 | * @pages: array of pages to release |
984 | * @nr: number of pages | |
1da177e4 | 985 | * |
aabfb572 MH |
986 | * Decrement the reference count on all the pages in @pages. If it |
987 | * fell to zero, remove the page from the LRU and free it. | |
1da177e4 | 988 | */ |
c6f92f9f | 989 | void release_pages(struct page **pages, int nr) |
1da177e4 LT |
990 | { |
991 | int i; | |
cc59850e | 992 | LIST_HEAD(pages_to_free); |
6168d0da | 993 | struct lruvec *lruvec = NULL; |
0de340cb | 994 | unsigned long flags = 0; |
3f649ab7 | 995 | unsigned int lock_batch; |
1da177e4 | 996 | |
1da177e4 | 997 | for (i = 0; i < nr; i++) { |
ab5e653e | 998 | struct folio *folio = page_folio(pages[i]); |
1da177e4 | 999 | |
aabfb572 MH |
1000 | /* |
1001 | * Make sure the IRQ-safe lock-holding time does not get | |
1002 | * excessive with a continuous string of pages from the | |
6168d0da | 1003 | * same lruvec. The lock is held only if lruvec != NULL. |
aabfb572 | 1004 | */ |
6168d0da AS |
1005 | if (lruvec && ++lock_batch == SWAP_CLUSTER_MAX) { |
1006 | unlock_page_lruvec_irqrestore(lruvec, flags); | |
1007 | lruvec = NULL; | |
aabfb572 MH |
1008 | } |
1009 | ||
ab5e653e | 1010 | if (is_huge_zero_page(&folio->page)) |
aa88b68c | 1011 | continue; |
aa88b68c | 1012 | |
ab5e653e | 1013 | if (folio_is_zone_device(folio)) { |
6168d0da AS |
1014 | if (lruvec) { |
1015 | unlock_page_lruvec_irqrestore(lruvec, flags); | |
1016 | lruvec = NULL; | |
df6ad698 | 1017 | } |
ab5e653e | 1018 | if (put_devmap_managed_page(&folio->page)) |
c5d6c45e | 1019 | continue; |
ab5e653e MWO |
1020 | if (folio_put_testzero(folio)) |
1021 | free_zone_device_page(&folio->page); | |
43fbdeb3 | 1022 | continue; |
df6ad698 JG |
1023 | } |
1024 | ||
ab5e653e | 1025 | if (!folio_put_testzero(folio)) |
1da177e4 LT |
1026 | continue; |
1027 | ||
ab5e653e | 1028 | if (folio_test_large(folio)) { |
6168d0da AS |
1029 | if (lruvec) { |
1030 | unlock_page_lruvec_irqrestore(lruvec, flags); | |
1031 | lruvec = NULL; | |
ddc58f27 | 1032 | } |
5ef82fe7 | 1033 | __folio_put_large(folio); |
ddc58f27 KS |
1034 | continue; |
1035 | } | |
1036 | ||
ab5e653e | 1037 | if (folio_test_lru(folio)) { |
2a5e4e34 AD |
1038 | struct lruvec *prev_lruvec = lruvec; |
1039 | ||
0de340cb | 1040 | lruvec = folio_lruvec_relock_irqsave(folio, lruvec, |
2a5e4e34 AD |
1041 | &flags); |
1042 | if (prev_lruvec != lruvec) | |
aabfb572 | 1043 | lock_batch = 0; |
fa9add64 | 1044 | |
ab5e653e MWO |
1045 | lruvec_del_folio(lruvec, folio); |
1046 | __folio_clear_lru_flags(folio); | |
46453a6e NP |
1047 | } |
1048 | ||
b109b870 HD |
1049 | /* |
1050 | * In rare cases, when truncation or holepunching raced with | |
1051 | * munlock after VM_LOCKED was cleared, Mlocked may still be | |
1052 | * found set here. This does not indicate a problem, unless | |
1053 | * "unevictable_pgs_cleared" appears worryingly large. | |
1054 | */ | |
ab5e653e MWO |
1055 | if (unlikely(folio_test_mlocked(folio))) { |
1056 | __folio_clear_mlocked(folio); | |
1057 | zone_stat_sub_folio(folio, NR_MLOCK); | |
b109b870 HD |
1058 | count_vm_event(UNEVICTABLE_PGCLEARED); |
1059 | } | |
1060 | ||
ab5e653e | 1061 | list_add(&folio->lru, &pages_to_free); |
1da177e4 | 1062 | } |
6168d0da AS |
1063 | if (lruvec) |
1064 | unlock_page_lruvec_irqrestore(lruvec, flags); | |
1da177e4 | 1065 | |
747db954 | 1066 | mem_cgroup_uncharge_list(&pages_to_free); |
2d4894b5 | 1067 | free_unref_page_list(&pages_to_free); |
1da177e4 | 1068 | } |
0be8557b | 1069 | EXPORT_SYMBOL(release_pages); |
1da177e4 LT |
1070 | |
1071 | /* | |
1072 | * The pages which we're about to release may be in the deferred lru-addition | |
1073 | * queues. That would prevent them from really being freed right now. That's | |
1074 | * OK from a correctness point of view but is inefficient - those pages may be | |
1075 | * cache-warm and we want to give them back to the page allocator ASAP. | |
1076 | * | |
70dea534 MWO |
1077 | * So __pagevec_release() will drain those queues here. |
1078 | * folio_batch_move_lru() calls folios_put() directly to avoid | |
1da177e4 LT |
1079 | * mutual recursion. |
1080 | */ | |
1081 | void __pagevec_release(struct pagevec *pvec) | |
1082 | { | |
7f0b5fb9 | 1083 | if (!pvec->percpu_pvec_drained) { |
d9ed0d08 | 1084 | lru_add_drain(); |
7f0b5fb9 | 1085 | pvec->percpu_pvec_drained = true; |
d9ed0d08 | 1086 | } |
c6f92f9f | 1087 | release_pages(pvec->pages, pagevec_count(pvec)); |
1da177e4 LT |
1088 | pagevec_reinit(pvec); |
1089 | } | |
7f285701 SF |
1090 | EXPORT_SYMBOL(__pagevec_release); |
1091 | ||
0cd6144a | 1092 | /** |
1613fac9 MWO |
1093 | * folio_batch_remove_exceptionals() - Prune non-folios from a batch. |
1094 | * @fbatch: The batch to prune | |
0cd6144a | 1095 | * |
1613fac9 MWO |
1096 | * find_get_entries() fills a batch with both folios and shadow/swap/DAX |
1097 | * entries. This function prunes all the non-folio entries from @fbatch | |
1098 | * without leaving holes, so that it can be passed on to folio-only batch | |
1099 | * operations. | |
0cd6144a | 1100 | */ |
1613fac9 | 1101 | void folio_batch_remove_exceptionals(struct folio_batch *fbatch) |
0cd6144a | 1102 | { |
1613fac9 | 1103 | unsigned int i, j; |
0cd6144a | 1104 | |
1613fac9 MWO |
1105 | for (i = 0, j = 0; i < folio_batch_count(fbatch); i++) { |
1106 | struct folio *folio = fbatch->folios[i]; | |
1107 | if (!xa_is_value(folio)) | |
1108 | fbatch->folios[j++] = folio; | |
0cd6144a | 1109 | } |
1613fac9 | 1110 | fbatch->nr = j; |
0cd6144a JW |
1111 | } |
1112 | ||
72b045ae JK |
1113 | unsigned pagevec_lookup_range_tag(struct pagevec *pvec, |
1114 | struct address_space *mapping, pgoff_t *index, pgoff_t end, | |
10bbd235 | 1115 | xa_mark_t tag) |
1da177e4 | 1116 | { |
72b045ae | 1117 | pvec->nr = find_get_pages_range_tag(mapping, index, end, tag, |
67fd707f | 1118 | PAGEVEC_SIZE, pvec->pages); |
1da177e4 LT |
1119 | return pagevec_count(pvec); |
1120 | } | |
72b045ae | 1121 | EXPORT_SYMBOL(pagevec_lookup_range_tag); |
1da177e4 | 1122 | |
1da177e4 LT |
1123 | /* |
1124 | * Perform any setup for the swap system | |
1125 | */ | |
1126 | void __init swap_setup(void) | |
1127 | { | |
ca79b0c2 | 1128 | unsigned long megs = totalram_pages() >> (20 - PAGE_SHIFT); |
e0bf68dd | 1129 | |
1da177e4 LT |
1130 | /* Use a smaller cluster for small-memory machines */ |
1131 | if (megs < 16) | |
1132 | page_cluster = 2; | |
1133 | else | |
1134 | page_cluster = 3; | |
1135 | /* | |
1136 | * Right now other parts of the system means that we | |
1137 | * _really_ don't want to cluster much more | |
1138 | */ | |
1da177e4 | 1139 | } |