| 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | /* |
| 3 | * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds |
| 4 | * |
| 5 | * Swap reorganised 29.12.95, Stephen Tweedie. |
| 6 | * kswapd added: 7.1.96 sct |
| 7 | * Removed kswapd_ctl limits, and swap out as many pages as needed |
| 8 | * to bring the system back to freepages.high: 2.4.97, Rik van Riel. |
| 9 | * Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com). |
| 10 | * Multiqueue VM started 5.8.00, Rik van Riel. |
| 11 | */ |
| 12 | |
| 13 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
| 14 | |
| 15 | #include <linux/mm.h> |
| 16 | #include <linux/sched/mm.h> |
| 17 | #include <linux/module.h> |
| 18 | #include <linux/gfp.h> |
| 19 | #include <linux/kernel_stat.h> |
| 20 | #include <linux/swap.h> |
| 21 | #include <linux/pagemap.h> |
| 22 | #include <linux/init.h> |
| 23 | #include <linux/highmem.h> |
| 24 | #include <linux/vmpressure.h> |
| 25 | #include <linux/vmstat.h> |
| 26 | #include <linux/file.h> |
| 27 | #include <linux/writeback.h> |
| 28 | #include <linux/blkdev.h> |
| 29 | #include <linux/buffer_head.h> /* for buffer_heads_over_limit */ |
| 30 | #include <linux/mm_inline.h> |
| 31 | #include <linux/backing-dev.h> |
| 32 | #include <linux/rmap.h> |
| 33 | #include <linux/topology.h> |
| 34 | #include <linux/cpu.h> |
| 35 | #include <linux/cpuset.h> |
| 36 | #include <linux/compaction.h> |
| 37 | #include <linux/notifier.h> |
| 38 | #include <linux/rwsem.h> |
| 39 | #include <linux/delay.h> |
| 40 | #include <linux/kthread.h> |
| 41 | #include <linux/freezer.h> |
| 42 | #include <linux/memcontrol.h> |
| 43 | #include <linux/migrate.h> |
| 44 | #include <linux/delayacct.h> |
| 45 | #include <linux/sysctl.h> |
| 46 | #include <linux/memory-tiers.h> |
| 47 | #include <linux/oom.h> |
| 48 | #include <linux/pagevec.h> |
| 49 | #include <linux/prefetch.h> |
| 50 | #include <linux/printk.h> |
| 51 | #include <linux/dax.h> |
| 52 | #include <linux/psi.h> |
| 53 | #include <linux/pagewalk.h> |
| 54 | #include <linux/shmem_fs.h> |
| 55 | #include <linux/ctype.h> |
| 56 | #include <linux/debugfs.h> |
| 57 | #include <linux/khugepaged.h> |
| 58 | |
| 59 | #include <asm/tlbflush.h> |
| 60 | #include <asm/div64.h> |
| 61 | |
| 62 | #include <linux/swapops.h> |
| 63 | #include <linux/balloon_compaction.h> |
| 64 | #include <linux/sched/sysctl.h> |
| 65 | |
| 66 | #include "internal.h" |
| 67 | #include "swap.h" |
| 68 | |
| 69 | #define CREATE_TRACE_POINTS |
| 70 | #include <trace/events/vmscan.h> |
| 71 | |
| 72 | struct scan_control { |
| 73 | /* How many pages shrink_list() should reclaim */ |
| 74 | unsigned long nr_to_reclaim; |
| 75 | |
| 76 | /* |
| 77 | * Nodemask of nodes allowed by the caller. If NULL, all nodes |
| 78 | * are scanned. |
| 79 | */ |
| 80 | nodemask_t *nodemask; |
| 81 | |
| 82 | /* |
| 83 | * The memory cgroup that hit its limit and as a result is the |
| 84 | * primary target of this reclaim invocation. |
| 85 | */ |
| 86 | struct mem_cgroup *target_mem_cgroup; |
| 87 | |
| 88 | /* |
| 89 | * Scan pressure balancing between anon and file LRUs |
| 90 | */ |
| 91 | unsigned long anon_cost; |
| 92 | unsigned long file_cost; |
| 93 | |
| 94 | /* Can active folios be deactivated as part of reclaim? */ |
| 95 | #define DEACTIVATE_ANON 1 |
| 96 | #define DEACTIVATE_FILE 2 |
| 97 | unsigned int may_deactivate:2; |
| 98 | unsigned int force_deactivate:1; |
| 99 | unsigned int skipped_deactivate:1; |
| 100 | |
| 101 | /* Writepage batching in laptop mode; RECLAIM_WRITE */ |
| 102 | unsigned int may_writepage:1; |
| 103 | |
| 104 | /* Can mapped folios be reclaimed? */ |
| 105 | unsigned int may_unmap:1; |
| 106 | |
| 107 | /* Can folios be swapped as part of reclaim? */ |
| 108 | unsigned int may_swap:1; |
| 109 | |
| 110 | /* Proactive reclaim invoked by userspace through memory.reclaim */ |
| 111 | unsigned int proactive:1; |
| 112 | |
| 113 | /* |
| 114 | * Cgroup memory below memory.low is protected as long as we |
| 115 | * don't threaten to OOM. If any cgroup is reclaimed at |
| 116 | * reduced force or passed over entirely due to its memory.low |
| 117 | * setting (memcg_low_skipped), and nothing is reclaimed as a |
| 118 | * result, then go back for one more cycle that reclaims the protected |
| 119 | * memory (memcg_low_reclaim) to avert OOM. |
| 120 | */ |
| 121 | unsigned int memcg_low_reclaim:1; |
| 122 | unsigned int memcg_low_skipped:1; |
| 123 | |
| 124 | unsigned int hibernation_mode:1; |
| 125 | |
| 126 | /* One of the zones is ready for compaction */ |
| 127 | unsigned int compaction_ready:1; |
| 128 | |
| 129 | /* There is easily reclaimable cold cache in the current node */ |
| 130 | unsigned int cache_trim_mode:1; |
| 131 | |
| 132 | /* The file folios on the current node are dangerously low */ |
| 133 | unsigned int file_is_tiny:1; |
| 134 | |
| 135 | /* Always discard instead of demoting to lower tier memory */ |
| 136 | unsigned int no_demotion:1; |
| 137 | |
| 138 | #ifdef CONFIG_LRU_GEN |
| 139 | /* help kswapd make better choices among multiple memcgs */ |
| 140 | unsigned int memcgs_need_aging:1; |
| 141 | unsigned long last_reclaimed; |
| 142 | #endif |
| 143 | |
| 144 | /* Allocation order */ |
| 145 | s8 order; |
| 146 | |
| 147 | /* Scan (total_size >> priority) pages at once */ |
| 148 | s8 priority; |
| 149 | |
| 150 | /* The highest zone to isolate folios for reclaim from */ |
| 151 | s8 reclaim_idx; |
| 152 | |
| 153 | /* This context's GFP mask */ |
| 154 | gfp_t gfp_mask; |
| 155 | |
| 156 | /* Incremented by the number of inactive pages that were scanned */ |
| 157 | unsigned long nr_scanned; |
| 158 | |
| 159 | /* Number of pages freed so far during a call to shrink_zones() */ |
| 160 | unsigned long nr_reclaimed; |
| 161 | |
| 162 | struct { |
| 163 | unsigned int dirty; |
| 164 | unsigned int unqueued_dirty; |
| 165 | unsigned int congested; |
| 166 | unsigned int writeback; |
| 167 | unsigned int immediate; |
| 168 | unsigned int file_taken; |
| 169 | unsigned int taken; |
| 170 | } nr; |
| 171 | |
| 172 | /* for recording the reclaimed slab by now */ |
| 173 | struct reclaim_state reclaim_state; |
| 174 | }; |
| 175 | |
| 176 | #ifdef ARCH_HAS_PREFETCHW |
| 177 | #define prefetchw_prev_lru_folio(_folio, _base, _field) \ |
| 178 | do { \ |
| 179 | if ((_folio)->lru.prev != _base) { \ |
| 180 | struct folio *prev; \ |
| 181 | \ |
| 182 | prev = lru_to_folio(&(_folio->lru)); \ |
| 183 | prefetchw(&prev->_field); \ |
| 184 | } \ |
| 185 | } while (0) |
| 186 | #else |
| 187 | #define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0) |
| 188 | #endif |
| 189 | |
| 190 | /* |
| 191 | * From 0 .. 200. Higher means more swappy. |
| 192 | */ |
| 193 | int vm_swappiness = 60; |
| 194 | |
| 195 | static void set_task_reclaim_state(struct task_struct *task, |
| 196 | struct reclaim_state *rs) |
| 197 | { |
| 198 | /* Check for an overwrite */ |
| 199 | WARN_ON_ONCE(rs && task->reclaim_state); |
| 200 | |
| 201 | /* Check for the nulling of an already-nulled member */ |
| 202 | WARN_ON_ONCE(!rs && !task->reclaim_state); |
| 203 | |
| 204 | task->reclaim_state = rs; |
| 205 | } |
| 206 | |
| 207 | LIST_HEAD(shrinker_list); |
| 208 | DECLARE_RWSEM(shrinker_rwsem); |
| 209 | |
| 210 | #ifdef CONFIG_MEMCG |
| 211 | static int shrinker_nr_max; |
| 212 | |
| 213 | /* The shrinker_info is expanded in a batch of BITS_PER_LONG */ |
| 214 | static inline int shrinker_map_size(int nr_items) |
| 215 | { |
| 216 | return (DIV_ROUND_UP(nr_items, BITS_PER_LONG) * sizeof(unsigned long)); |
| 217 | } |
| 218 | |
| 219 | static inline int shrinker_defer_size(int nr_items) |
| 220 | { |
| 221 | return (round_up(nr_items, BITS_PER_LONG) * sizeof(atomic_long_t)); |
| 222 | } |
| 223 | |
| 224 | static struct shrinker_info *shrinker_info_protected(struct mem_cgroup *memcg, |
| 225 | int nid) |
| 226 | { |
| 227 | return rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_info, |
| 228 | lockdep_is_held(&shrinker_rwsem)); |
| 229 | } |
| 230 | |
| 231 | static int expand_one_shrinker_info(struct mem_cgroup *memcg, |
| 232 | int map_size, int defer_size, |
| 233 | int old_map_size, int old_defer_size) |
| 234 | { |
| 235 | struct shrinker_info *new, *old; |
| 236 | struct mem_cgroup_per_node *pn; |
| 237 | int nid; |
| 238 | int size = map_size + defer_size; |
| 239 | |
| 240 | for_each_node(nid) { |
| 241 | pn = memcg->nodeinfo[nid]; |
| 242 | old = shrinker_info_protected(memcg, nid); |
| 243 | /* Not yet online memcg */ |
| 244 | if (!old) |
| 245 | return 0; |
| 246 | |
| 247 | new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid); |
| 248 | if (!new) |
| 249 | return -ENOMEM; |
| 250 | |
| 251 | new->nr_deferred = (atomic_long_t *)(new + 1); |
| 252 | new->map = (void *)new->nr_deferred + defer_size; |
| 253 | |
| 254 | /* map: set all old bits, clear all new bits */ |
| 255 | memset(new->map, (int)0xff, old_map_size); |
| 256 | memset((void *)new->map + old_map_size, 0, map_size - old_map_size); |
| 257 | /* nr_deferred: copy old values, clear all new values */ |
| 258 | memcpy(new->nr_deferred, old->nr_deferred, old_defer_size); |
| 259 | memset((void *)new->nr_deferred + old_defer_size, 0, |
| 260 | defer_size - old_defer_size); |
| 261 | |
| 262 | rcu_assign_pointer(pn->shrinker_info, new); |
| 263 | kvfree_rcu(old, rcu); |
| 264 | } |
| 265 | |
| 266 | return 0; |
| 267 | } |
| 268 | |
| 269 | void free_shrinker_info(struct mem_cgroup *memcg) |
| 270 | { |
| 271 | struct mem_cgroup_per_node *pn; |
| 272 | struct shrinker_info *info; |
| 273 | int nid; |
| 274 | |
| 275 | for_each_node(nid) { |
| 276 | pn = memcg->nodeinfo[nid]; |
| 277 | info = rcu_dereference_protected(pn->shrinker_info, true); |
| 278 | kvfree(info); |
| 279 | rcu_assign_pointer(pn->shrinker_info, NULL); |
| 280 | } |
| 281 | } |
| 282 | |
| 283 | int alloc_shrinker_info(struct mem_cgroup *memcg) |
| 284 | { |
| 285 | struct shrinker_info *info; |
| 286 | int nid, size, ret = 0; |
| 287 | int map_size, defer_size = 0; |
| 288 | |
| 289 | down_write(&shrinker_rwsem); |
| 290 | map_size = shrinker_map_size(shrinker_nr_max); |
| 291 | defer_size = shrinker_defer_size(shrinker_nr_max); |
| 292 | size = map_size + defer_size; |
| 293 | for_each_node(nid) { |
| 294 | info = kvzalloc_node(sizeof(*info) + size, GFP_KERNEL, nid); |
| 295 | if (!info) { |
| 296 | free_shrinker_info(memcg); |
| 297 | ret = -ENOMEM; |
| 298 | break; |
| 299 | } |
| 300 | info->nr_deferred = (atomic_long_t *)(info + 1); |
| 301 | info->map = (void *)info->nr_deferred + defer_size; |
| 302 | rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_info, info); |
| 303 | } |
| 304 | up_write(&shrinker_rwsem); |
| 305 | |
| 306 | return ret; |
| 307 | } |
| 308 | |
| 309 | static inline bool need_expand(int nr_max) |
| 310 | { |
| 311 | return round_up(nr_max, BITS_PER_LONG) > |
| 312 | round_up(shrinker_nr_max, BITS_PER_LONG); |
| 313 | } |
| 314 | |
| 315 | static int expand_shrinker_info(int new_id) |
| 316 | { |
| 317 | int ret = 0; |
| 318 | int new_nr_max = new_id + 1; |
| 319 | int map_size, defer_size = 0; |
| 320 | int old_map_size, old_defer_size = 0; |
| 321 | struct mem_cgroup *memcg; |
| 322 | |
| 323 | if (!need_expand(new_nr_max)) |
| 324 | goto out; |
| 325 | |
| 326 | if (!root_mem_cgroup) |
| 327 | goto out; |
| 328 | |
| 329 | lockdep_assert_held(&shrinker_rwsem); |
| 330 | |
| 331 | map_size = shrinker_map_size(new_nr_max); |
| 332 | defer_size = shrinker_defer_size(new_nr_max); |
| 333 | old_map_size = shrinker_map_size(shrinker_nr_max); |
| 334 | old_defer_size = shrinker_defer_size(shrinker_nr_max); |
| 335 | |
| 336 | memcg = mem_cgroup_iter(NULL, NULL, NULL); |
| 337 | do { |
| 338 | ret = expand_one_shrinker_info(memcg, map_size, defer_size, |
| 339 | old_map_size, old_defer_size); |
| 340 | if (ret) { |
| 341 | mem_cgroup_iter_break(NULL, memcg); |
| 342 | goto out; |
| 343 | } |
| 344 | } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL); |
| 345 | out: |
| 346 | if (!ret) |
| 347 | shrinker_nr_max = new_nr_max; |
| 348 | |
| 349 | return ret; |
| 350 | } |
| 351 | |
| 352 | void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id) |
| 353 | { |
| 354 | if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) { |
| 355 | struct shrinker_info *info; |
| 356 | |
| 357 | rcu_read_lock(); |
| 358 | info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info); |
| 359 | /* Pairs with smp mb in shrink_slab() */ |
| 360 | smp_mb__before_atomic(); |
| 361 | set_bit(shrinker_id, info->map); |
| 362 | rcu_read_unlock(); |
| 363 | } |
| 364 | } |
| 365 | |
| 366 | static DEFINE_IDR(shrinker_idr); |
| 367 | |
| 368 | static int prealloc_memcg_shrinker(struct shrinker *shrinker) |
| 369 | { |
| 370 | int id, ret = -ENOMEM; |
| 371 | |
| 372 | if (mem_cgroup_disabled()) |
| 373 | return -ENOSYS; |
| 374 | |
| 375 | down_write(&shrinker_rwsem); |
| 376 | /* This may call shrinker, so it must use down_read_trylock() */ |
| 377 | id = idr_alloc(&shrinker_idr, shrinker, 0, 0, GFP_KERNEL); |
| 378 | if (id < 0) |
| 379 | goto unlock; |
| 380 | |
| 381 | if (id >= shrinker_nr_max) { |
| 382 | if (expand_shrinker_info(id)) { |
| 383 | idr_remove(&shrinker_idr, id); |
| 384 | goto unlock; |
| 385 | } |
| 386 | } |
| 387 | shrinker->id = id; |
| 388 | ret = 0; |
| 389 | unlock: |
| 390 | up_write(&shrinker_rwsem); |
| 391 | return ret; |
| 392 | } |
| 393 | |
| 394 | static void unregister_memcg_shrinker(struct shrinker *shrinker) |
| 395 | { |
| 396 | int id = shrinker->id; |
| 397 | |
| 398 | BUG_ON(id < 0); |
| 399 | |
| 400 | lockdep_assert_held(&shrinker_rwsem); |
| 401 | |
| 402 | idr_remove(&shrinker_idr, id); |
| 403 | } |
| 404 | |
| 405 | static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker, |
| 406 | struct mem_cgroup *memcg) |
| 407 | { |
| 408 | struct shrinker_info *info; |
| 409 | |
| 410 | info = shrinker_info_protected(memcg, nid); |
| 411 | return atomic_long_xchg(&info->nr_deferred[shrinker->id], 0); |
| 412 | } |
| 413 | |
| 414 | static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker, |
| 415 | struct mem_cgroup *memcg) |
| 416 | { |
| 417 | struct shrinker_info *info; |
| 418 | |
| 419 | info = shrinker_info_protected(memcg, nid); |
| 420 | return atomic_long_add_return(nr, &info->nr_deferred[shrinker->id]); |
| 421 | } |
| 422 | |
| 423 | void reparent_shrinker_deferred(struct mem_cgroup *memcg) |
| 424 | { |
| 425 | int i, nid; |
| 426 | long nr; |
| 427 | struct mem_cgroup *parent; |
| 428 | struct shrinker_info *child_info, *parent_info; |
| 429 | |
| 430 | parent = parent_mem_cgroup(memcg); |
| 431 | if (!parent) |
| 432 | parent = root_mem_cgroup; |
| 433 | |
| 434 | /* Prevent from concurrent shrinker_info expand */ |
| 435 | down_read(&shrinker_rwsem); |
| 436 | for_each_node(nid) { |
| 437 | child_info = shrinker_info_protected(memcg, nid); |
| 438 | parent_info = shrinker_info_protected(parent, nid); |
| 439 | for (i = 0; i < shrinker_nr_max; i++) { |
| 440 | nr = atomic_long_read(&child_info->nr_deferred[i]); |
| 441 | atomic_long_add(nr, &parent_info->nr_deferred[i]); |
| 442 | } |
| 443 | } |
| 444 | up_read(&shrinker_rwsem); |
| 445 | } |
| 446 | |
| 447 | static bool cgroup_reclaim(struct scan_control *sc) |
| 448 | { |
| 449 | return sc->target_mem_cgroup; |
| 450 | } |
| 451 | |
| 452 | /** |
| 453 | * writeback_throttling_sane - is the usual dirty throttling mechanism available? |
| 454 | * @sc: scan_control in question |
| 455 | * |
| 456 | * The normal page dirty throttling mechanism in balance_dirty_pages() is |
| 457 | * completely broken with the legacy memcg and direct stalling in |
| 458 | * shrink_folio_list() is used for throttling instead, which lacks all the |
| 459 | * niceties such as fairness, adaptive pausing, bandwidth proportional |
| 460 | * allocation and configurability. |
| 461 | * |
| 462 | * This function tests whether the vmscan currently in progress can assume |
| 463 | * that the normal dirty throttling mechanism is operational. |
| 464 | */ |
| 465 | static bool writeback_throttling_sane(struct scan_control *sc) |
| 466 | { |
| 467 | if (!cgroup_reclaim(sc)) |
| 468 | return true; |
| 469 | #ifdef CONFIG_CGROUP_WRITEBACK |
| 470 | if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) |
| 471 | return true; |
| 472 | #endif |
| 473 | return false; |
| 474 | } |
| 475 | #else |
| 476 | static int prealloc_memcg_shrinker(struct shrinker *shrinker) |
| 477 | { |
| 478 | return -ENOSYS; |
| 479 | } |
| 480 | |
| 481 | static void unregister_memcg_shrinker(struct shrinker *shrinker) |
| 482 | { |
| 483 | } |
| 484 | |
| 485 | static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker, |
| 486 | struct mem_cgroup *memcg) |
| 487 | { |
| 488 | return 0; |
| 489 | } |
| 490 | |
| 491 | static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker, |
| 492 | struct mem_cgroup *memcg) |
| 493 | { |
| 494 | return 0; |
| 495 | } |
| 496 | |
| 497 | static bool cgroup_reclaim(struct scan_control *sc) |
| 498 | { |
| 499 | return false; |
| 500 | } |
| 501 | |
| 502 | static bool writeback_throttling_sane(struct scan_control *sc) |
| 503 | { |
| 504 | return true; |
| 505 | } |
| 506 | #endif |
| 507 | |
| 508 | static long xchg_nr_deferred(struct shrinker *shrinker, |
| 509 | struct shrink_control *sc) |
| 510 | { |
| 511 | int nid = sc->nid; |
| 512 | |
| 513 | if (!(shrinker->flags & SHRINKER_NUMA_AWARE)) |
| 514 | nid = 0; |
| 515 | |
| 516 | if (sc->memcg && |
| 517 | (shrinker->flags & SHRINKER_MEMCG_AWARE)) |
| 518 | return xchg_nr_deferred_memcg(nid, shrinker, |
| 519 | sc->memcg); |
| 520 | |
| 521 | return atomic_long_xchg(&shrinker->nr_deferred[nid], 0); |
| 522 | } |
| 523 | |
| 524 | |
| 525 | static long add_nr_deferred(long nr, struct shrinker *shrinker, |
| 526 | struct shrink_control *sc) |
| 527 | { |
| 528 | int nid = sc->nid; |
| 529 | |
| 530 | if (!(shrinker->flags & SHRINKER_NUMA_AWARE)) |
| 531 | nid = 0; |
| 532 | |
| 533 | if (sc->memcg && |
| 534 | (shrinker->flags & SHRINKER_MEMCG_AWARE)) |
| 535 | return add_nr_deferred_memcg(nr, nid, shrinker, |
| 536 | sc->memcg); |
| 537 | |
| 538 | return atomic_long_add_return(nr, &shrinker->nr_deferred[nid]); |
| 539 | } |
| 540 | |
| 541 | static bool can_demote(int nid, struct scan_control *sc) |
| 542 | { |
| 543 | if (!numa_demotion_enabled) |
| 544 | return false; |
| 545 | if (sc && sc->no_demotion) |
| 546 | return false; |
| 547 | if (next_demotion_node(nid) == NUMA_NO_NODE) |
| 548 | return false; |
| 549 | |
| 550 | return true; |
| 551 | } |
| 552 | |
| 553 | static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg, |
| 554 | int nid, |
| 555 | struct scan_control *sc) |
| 556 | { |
| 557 | if (memcg == NULL) { |
| 558 | /* |
| 559 | * For non-memcg reclaim, is there |
| 560 | * space in any swap device? |
| 561 | */ |
| 562 | if (get_nr_swap_pages() > 0) |
| 563 | return true; |
| 564 | } else { |
| 565 | /* Is the memcg below its swap limit? */ |
| 566 | if (mem_cgroup_get_nr_swap_pages(memcg) > 0) |
| 567 | return true; |
| 568 | } |
| 569 | |
| 570 | /* |
| 571 | * The page can not be swapped. |
| 572 | * |
| 573 | * Can it be reclaimed from this node via demotion? |
| 574 | */ |
| 575 | return can_demote(nid, sc); |
| 576 | } |
| 577 | |
| 578 | /* |
| 579 | * This misses isolated folios which are not accounted for to save counters. |
| 580 | * As the data only determines if reclaim or compaction continues, it is |
| 581 | * not expected that isolated folios will be a dominating factor. |
| 582 | */ |
| 583 | unsigned long zone_reclaimable_pages(struct zone *zone) |
| 584 | { |
| 585 | unsigned long nr; |
| 586 | |
| 587 | nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) + |
| 588 | zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE); |
| 589 | if (can_reclaim_anon_pages(NULL, zone_to_nid(zone), NULL)) |
| 590 | nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) + |
| 591 | zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON); |
| 592 | |
| 593 | return nr; |
| 594 | } |
| 595 | |
| 596 | /** |
| 597 | * lruvec_lru_size - Returns the number of pages on the given LRU list. |
| 598 | * @lruvec: lru vector |
| 599 | * @lru: lru to use |
| 600 | * @zone_idx: zones to consider (use MAX_NR_ZONES - 1 for the whole LRU list) |
| 601 | */ |
| 602 | static unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, |
| 603 | int zone_idx) |
| 604 | { |
| 605 | unsigned long size = 0; |
| 606 | int zid; |
| 607 | |
| 608 | for (zid = 0; zid <= zone_idx; zid++) { |
| 609 | struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid]; |
| 610 | |
| 611 | if (!managed_zone(zone)) |
| 612 | continue; |
| 613 | |
| 614 | if (!mem_cgroup_disabled()) |
| 615 | size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid); |
| 616 | else |
| 617 | size += zone_page_state(zone, NR_ZONE_LRU_BASE + lru); |
| 618 | } |
| 619 | return size; |
| 620 | } |
| 621 | |
| 622 | /* |
| 623 | * Add a shrinker callback to be called from the vm. |
| 624 | */ |
| 625 | static int __prealloc_shrinker(struct shrinker *shrinker) |
| 626 | { |
| 627 | unsigned int size; |
| 628 | int err; |
| 629 | |
| 630 | if (shrinker->flags & SHRINKER_MEMCG_AWARE) { |
| 631 | err = prealloc_memcg_shrinker(shrinker); |
| 632 | if (err != -ENOSYS) |
| 633 | return err; |
| 634 | |
| 635 | shrinker->flags &= ~SHRINKER_MEMCG_AWARE; |
| 636 | } |
| 637 | |
| 638 | size = sizeof(*shrinker->nr_deferred); |
| 639 | if (shrinker->flags & SHRINKER_NUMA_AWARE) |
| 640 | size *= nr_node_ids; |
| 641 | |
| 642 | shrinker->nr_deferred = kzalloc(size, GFP_KERNEL); |
| 643 | if (!shrinker->nr_deferred) |
| 644 | return -ENOMEM; |
| 645 | |
| 646 | return 0; |
| 647 | } |
| 648 | |
| 649 | #ifdef CONFIG_SHRINKER_DEBUG |
| 650 | int prealloc_shrinker(struct shrinker *shrinker, const char *fmt, ...) |
| 651 | { |
| 652 | va_list ap; |
| 653 | int err; |
| 654 | |
| 655 | va_start(ap, fmt); |
| 656 | shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap); |
| 657 | va_end(ap); |
| 658 | if (!shrinker->name) |
| 659 | return -ENOMEM; |
| 660 | |
| 661 | err = __prealloc_shrinker(shrinker); |
| 662 | if (err) { |
| 663 | kfree_const(shrinker->name); |
| 664 | shrinker->name = NULL; |
| 665 | } |
| 666 | |
| 667 | return err; |
| 668 | } |
| 669 | #else |
| 670 | int prealloc_shrinker(struct shrinker *shrinker, const char *fmt, ...) |
| 671 | { |
| 672 | return __prealloc_shrinker(shrinker); |
| 673 | } |
| 674 | #endif |
| 675 | |
| 676 | void free_prealloced_shrinker(struct shrinker *shrinker) |
| 677 | { |
| 678 | #ifdef CONFIG_SHRINKER_DEBUG |
| 679 | kfree_const(shrinker->name); |
| 680 | shrinker->name = NULL; |
| 681 | #endif |
| 682 | if (shrinker->flags & SHRINKER_MEMCG_AWARE) { |
| 683 | down_write(&shrinker_rwsem); |
| 684 | unregister_memcg_shrinker(shrinker); |
| 685 | up_write(&shrinker_rwsem); |
| 686 | return; |
| 687 | } |
| 688 | |
| 689 | kfree(shrinker->nr_deferred); |
| 690 | shrinker->nr_deferred = NULL; |
| 691 | } |
| 692 | |
| 693 | void register_shrinker_prepared(struct shrinker *shrinker) |
| 694 | { |
| 695 | down_write(&shrinker_rwsem); |
| 696 | list_add_tail(&shrinker->list, &shrinker_list); |
| 697 | shrinker->flags |= SHRINKER_REGISTERED; |
| 698 | shrinker_debugfs_add(shrinker); |
| 699 | up_write(&shrinker_rwsem); |
| 700 | } |
| 701 | |
| 702 | static int __register_shrinker(struct shrinker *shrinker) |
| 703 | { |
| 704 | int err = __prealloc_shrinker(shrinker); |
| 705 | |
| 706 | if (err) |
| 707 | return err; |
| 708 | register_shrinker_prepared(shrinker); |
| 709 | return 0; |
| 710 | } |
| 711 | |
| 712 | #ifdef CONFIG_SHRINKER_DEBUG |
| 713 | int register_shrinker(struct shrinker *shrinker, const char *fmt, ...) |
| 714 | { |
| 715 | va_list ap; |
| 716 | int err; |
| 717 | |
| 718 | va_start(ap, fmt); |
| 719 | shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap); |
| 720 | va_end(ap); |
| 721 | if (!shrinker->name) |
| 722 | return -ENOMEM; |
| 723 | |
| 724 | err = __register_shrinker(shrinker); |
| 725 | if (err) { |
| 726 | kfree_const(shrinker->name); |
| 727 | shrinker->name = NULL; |
| 728 | } |
| 729 | return err; |
| 730 | } |
| 731 | #else |
| 732 | int register_shrinker(struct shrinker *shrinker, const char *fmt, ...) |
| 733 | { |
| 734 | return __register_shrinker(shrinker); |
| 735 | } |
| 736 | #endif |
| 737 | EXPORT_SYMBOL(register_shrinker); |
| 738 | |
| 739 | /* |
| 740 | * Remove one |
| 741 | */ |
| 742 | void unregister_shrinker(struct shrinker *shrinker) |
| 743 | { |
| 744 | if (!(shrinker->flags & SHRINKER_REGISTERED)) |
| 745 | return; |
| 746 | |
| 747 | down_write(&shrinker_rwsem); |
| 748 | list_del(&shrinker->list); |
| 749 | shrinker->flags &= ~SHRINKER_REGISTERED; |
| 750 | if (shrinker->flags & SHRINKER_MEMCG_AWARE) |
| 751 | unregister_memcg_shrinker(shrinker); |
| 752 | shrinker_debugfs_remove(shrinker); |
| 753 | up_write(&shrinker_rwsem); |
| 754 | |
| 755 | kfree(shrinker->nr_deferred); |
| 756 | shrinker->nr_deferred = NULL; |
| 757 | } |
| 758 | EXPORT_SYMBOL(unregister_shrinker); |
| 759 | |
| 760 | /** |
| 761 | * synchronize_shrinkers - Wait for all running shrinkers to complete. |
| 762 | * |
| 763 | * This is equivalent to calling unregister_shrink() and register_shrinker(), |
| 764 | * but atomically and with less overhead. This is useful to guarantee that all |
| 765 | * shrinker invocations have seen an update, before freeing memory, similar to |
| 766 | * rcu. |
| 767 | */ |
| 768 | void synchronize_shrinkers(void) |
| 769 | { |
| 770 | down_write(&shrinker_rwsem); |
| 771 | up_write(&shrinker_rwsem); |
| 772 | } |
| 773 | EXPORT_SYMBOL(synchronize_shrinkers); |
| 774 | |
| 775 | #define SHRINK_BATCH 128 |
| 776 | |
| 777 | static unsigned long do_shrink_slab(struct shrink_control *shrinkctl, |
| 778 | struct shrinker *shrinker, int priority) |
| 779 | { |
| 780 | unsigned long freed = 0; |
| 781 | unsigned long long delta; |
| 782 | long total_scan; |
| 783 | long freeable; |
| 784 | long nr; |
| 785 | long new_nr; |
| 786 | long batch_size = shrinker->batch ? shrinker->batch |
| 787 | : SHRINK_BATCH; |
| 788 | long scanned = 0, next_deferred; |
| 789 | |
| 790 | freeable = shrinker->count_objects(shrinker, shrinkctl); |
| 791 | if (freeable == 0 || freeable == SHRINK_EMPTY) |
| 792 | return freeable; |
| 793 | |
| 794 | /* |
| 795 | * copy the current shrinker scan count into a local variable |
| 796 | * and zero it so that other concurrent shrinker invocations |
| 797 | * don't also do this scanning work. |
| 798 | */ |
| 799 | nr = xchg_nr_deferred(shrinker, shrinkctl); |
| 800 | |
| 801 | if (shrinker->seeks) { |
| 802 | delta = freeable >> priority; |
| 803 | delta *= 4; |
| 804 | do_div(delta, shrinker->seeks); |
| 805 | } else { |
| 806 | /* |
| 807 | * These objects don't require any IO to create. Trim |
| 808 | * them aggressively under memory pressure to keep |
| 809 | * them from causing refetches in the IO caches. |
| 810 | */ |
| 811 | delta = freeable / 2; |
| 812 | } |
| 813 | |
| 814 | total_scan = nr >> priority; |
| 815 | total_scan += delta; |
| 816 | total_scan = min(total_scan, (2 * freeable)); |
| 817 | |
| 818 | trace_mm_shrink_slab_start(shrinker, shrinkctl, nr, |
| 819 | freeable, delta, total_scan, priority); |
| 820 | |
| 821 | /* |
| 822 | * Normally, we should not scan less than batch_size objects in one |
| 823 | * pass to avoid too frequent shrinker calls, but if the slab has less |
| 824 | * than batch_size objects in total and we are really tight on memory, |
| 825 | * we will try to reclaim all available objects, otherwise we can end |
| 826 | * up failing allocations although there are plenty of reclaimable |
| 827 | * objects spread over several slabs with usage less than the |
| 828 | * batch_size. |
| 829 | * |
| 830 | * We detect the "tight on memory" situations by looking at the total |
| 831 | * number of objects we want to scan (total_scan). If it is greater |
| 832 | * than the total number of objects on slab (freeable), we must be |
| 833 | * scanning at high prio and therefore should try to reclaim as much as |
| 834 | * possible. |
| 835 | */ |
| 836 | while (total_scan >= batch_size || |
| 837 | total_scan >= freeable) { |
| 838 | unsigned long ret; |
| 839 | unsigned long nr_to_scan = min(batch_size, total_scan); |
| 840 | |
| 841 | shrinkctl->nr_to_scan = nr_to_scan; |
| 842 | shrinkctl->nr_scanned = nr_to_scan; |
| 843 | ret = shrinker->scan_objects(shrinker, shrinkctl); |
| 844 | if (ret == SHRINK_STOP) |
| 845 | break; |
| 846 | freed += ret; |
| 847 | |
| 848 | count_vm_events(SLABS_SCANNED, shrinkctl->nr_scanned); |
| 849 | total_scan -= shrinkctl->nr_scanned; |
| 850 | scanned += shrinkctl->nr_scanned; |
| 851 | |
| 852 | cond_resched(); |
| 853 | } |
| 854 | |
| 855 | /* |
| 856 | * The deferred work is increased by any new work (delta) that wasn't |
| 857 | * done, decreased by old deferred work that was done now. |
| 858 | * |
| 859 | * And it is capped to two times of the freeable items. |
| 860 | */ |
| 861 | next_deferred = max_t(long, (nr + delta - scanned), 0); |
| 862 | next_deferred = min(next_deferred, (2 * freeable)); |
| 863 | |
| 864 | /* |
| 865 | * move the unused scan count back into the shrinker in a |
| 866 | * manner that handles concurrent updates. |
| 867 | */ |
| 868 | new_nr = add_nr_deferred(next_deferred, shrinker, shrinkctl); |
| 869 | |
| 870 | trace_mm_shrink_slab_end(shrinker, shrinkctl->nid, freed, nr, new_nr, total_scan); |
| 871 | return freed; |
| 872 | } |
| 873 | |
| 874 | #ifdef CONFIG_MEMCG |
| 875 | static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, |
| 876 | struct mem_cgroup *memcg, int priority) |
| 877 | { |
| 878 | struct shrinker_info *info; |
| 879 | unsigned long ret, freed = 0; |
| 880 | int i; |
| 881 | |
| 882 | if (!mem_cgroup_online(memcg)) |
| 883 | return 0; |
| 884 | |
| 885 | if (!down_read_trylock(&shrinker_rwsem)) |
| 886 | return 0; |
| 887 | |
| 888 | info = shrinker_info_protected(memcg, nid); |
| 889 | if (unlikely(!info)) |
| 890 | goto unlock; |
| 891 | |
| 892 | for_each_set_bit(i, info->map, shrinker_nr_max) { |
| 893 | struct shrink_control sc = { |
| 894 | .gfp_mask = gfp_mask, |
| 895 | .nid = nid, |
| 896 | .memcg = memcg, |
| 897 | }; |
| 898 | struct shrinker *shrinker; |
| 899 | |
| 900 | shrinker = idr_find(&shrinker_idr, i); |
| 901 | if (unlikely(!shrinker || !(shrinker->flags & SHRINKER_REGISTERED))) { |
| 902 | if (!shrinker) |
| 903 | clear_bit(i, info->map); |
| 904 | continue; |
| 905 | } |
| 906 | |
| 907 | /* Call non-slab shrinkers even though kmem is disabled */ |
| 908 | if (!memcg_kmem_enabled() && |
| 909 | !(shrinker->flags & SHRINKER_NONSLAB)) |
| 910 | continue; |
| 911 | |
| 912 | ret = do_shrink_slab(&sc, shrinker, priority); |
| 913 | if (ret == SHRINK_EMPTY) { |
| 914 | clear_bit(i, info->map); |
| 915 | /* |
| 916 | * After the shrinker reported that it had no objects to |
| 917 | * free, but before we cleared the corresponding bit in |
| 918 | * the memcg shrinker map, a new object might have been |
| 919 | * added. To make sure, we have the bit set in this |
| 920 | * case, we invoke the shrinker one more time and reset |
| 921 | * the bit if it reports that it is not empty anymore. |
| 922 | * The memory barrier here pairs with the barrier in |
| 923 | * set_shrinker_bit(): |
| 924 | * |
| 925 | * list_lru_add() shrink_slab_memcg() |
| 926 | * list_add_tail() clear_bit() |
| 927 | * <MB> <MB> |
| 928 | * set_bit() do_shrink_slab() |
| 929 | */ |
| 930 | smp_mb__after_atomic(); |
| 931 | ret = do_shrink_slab(&sc, shrinker, priority); |
| 932 | if (ret == SHRINK_EMPTY) |
| 933 | ret = 0; |
| 934 | else |
| 935 | set_shrinker_bit(memcg, nid, i); |
| 936 | } |
| 937 | freed += ret; |
| 938 | |
| 939 | if (rwsem_is_contended(&shrinker_rwsem)) { |
| 940 | freed = freed ? : 1; |
| 941 | break; |
| 942 | } |
| 943 | } |
| 944 | unlock: |
| 945 | up_read(&shrinker_rwsem); |
| 946 | return freed; |
| 947 | } |
| 948 | #else /* CONFIG_MEMCG */ |
| 949 | static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, |
| 950 | struct mem_cgroup *memcg, int priority) |
| 951 | { |
| 952 | return 0; |
| 953 | } |
| 954 | #endif /* CONFIG_MEMCG */ |
| 955 | |
| 956 | /** |
| 957 | * shrink_slab - shrink slab caches |
| 958 | * @gfp_mask: allocation context |
| 959 | * @nid: node whose slab caches to target |
| 960 | * @memcg: memory cgroup whose slab caches to target |
| 961 | * @priority: the reclaim priority |
| 962 | * |
| 963 | * Call the shrink functions to age shrinkable caches. |
| 964 | * |
| 965 | * @nid is passed along to shrinkers with SHRINKER_NUMA_AWARE set, |
| 966 | * unaware shrinkers will receive a node id of 0 instead. |
| 967 | * |
| 968 | * @memcg specifies the memory cgroup to target. Unaware shrinkers |
| 969 | * are called only if it is the root cgroup. |
| 970 | * |
| 971 | * @priority is sc->priority, we take the number of objects and >> by priority |
| 972 | * in order to get the scan target. |
| 973 | * |
| 974 | * Returns the number of reclaimed slab objects. |
| 975 | */ |
| 976 | static unsigned long shrink_slab(gfp_t gfp_mask, int nid, |
| 977 | struct mem_cgroup *memcg, |
| 978 | int priority) |
| 979 | { |
| 980 | unsigned long ret, freed = 0; |
| 981 | struct shrinker *shrinker; |
| 982 | |
| 983 | /* |
| 984 | * The root memcg might be allocated even though memcg is disabled |
| 985 | * via "cgroup_disable=memory" boot parameter. This could make |
| 986 | * mem_cgroup_is_root() return false, then just run memcg slab |
| 987 | * shrink, but skip global shrink. This may result in premature |
| 988 | * oom. |
| 989 | */ |
| 990 | if (!mem_cgroup_disabled() && !mem_cgroup_is_root(memcg)) |
| 991 | return shrink_slab_memcg(gfp_mask, nid, memcg, priority); |
| 992 | |
| 993 | if (!down_read_trylock(&shrinker_rwsem)) |
| 994 | goto out; |
| 995 | |
| 996 | list_for_each_entry(shrinker, &shrinker_list, list) { |
| 997 | struct shrink_control sc = { |
| 998 | .gfp_mask = gfp_mask, |
| 999 | .nid = nid, |
| 1000 | .memcg = memcg, |
| 1001 | }; |
| 1002 | |
| 1003 | ret = do_shrink_slab(&sc, shrinker, priority); |
| 1004 | if (ret == SHRINK_EMPTY) |
| 1005 | ret = 0; |
| 1006 | freed += ret; |
| 1007 | /* |
| 1008 | * Bail out if someone want to register a new shrinker to |
| 1009 | * prevent the registration from being stalled for long periods |
| 1010 | * by parallel ongoing shrinking. |
| 1011 | */ |
| 1012 | if (rwsem_is_contended(&shrinker_rwsem)) { |
| 1013 | freed = freed ? : 1; |
| 1014 | break; |
| 1015 | } |
| 1016 | } |
| 1017 | |
| 1018 | up_read(&shrinker_rwsem); |
| 1019 | out: |
| 1020 | cond_resched(); |
| 1021 | return freed; |
| 1022 | } |
| 1023 | |
| 1024 | static unsigned long drop_slab_node(int nid) |
| 1025 | { |
| 1026 | unsigned long freed = 0; |
| 1027 | struct mem_cgroup *memcg = NULL; |
| 1028 | |
| 1029 | memcg = mem_cgroup_iter(NULL, NULL, NULL); |
| 1030 | do { |
| 1031 | freed += shrink_slab(GFP_KERNEL, nid, memcg, 0); |
| 1032 | } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL); |
| 1033 | |
| 1034 | return freed; |
| 1035 | } |
| 1036 | |
| 1037 | void drop_slab(void) |
| 1038 | { |
| 1039 | int nid; |
| 1040 | int shift = 0; |
| 1041 | unsigned long freed; |
| 1042 | |
| 1043 | do { |
| 1044 | freed = 0; |
| 1045 | for_each_online_node(nid) { |
| 1046 | if (fatal_signal_pending(current)) |
| 1047 | return; |
| 1048 | |
| 1049 | freed += drop_slab_node(nid); |
| 1050 | } |
| 1051 | } while ((freed >> shift++) > 1); |
| 1052 | } |
| 1053 | |
| 1054 | static int reclaimer_offset(void) |
| 1055 | { |
| 1056 | BUILD_BUG_ON(PGSTEAL_DIRECT - PGSTEAL_KSWAPD != |
| 1057 | PGDEMOTE_DIRECT - PGDEMOTE_KSWAPD); |
| 1058 | BUILD_BUG_ON(PGSTEAL_DIRECT - PGSTEAL_KSWAPD != |
| 1059 | PGSCAN_DIRECT - PGSCAN_KSWAPD); |
| 1060 | BUILD_BUG_ON(PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD != |
| 1061 | PGDEMOTE_KHUGEPAGED - PGDEMOTE_KSWAPD); |
| 1062 | BUILD_BUG_ON(PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD != |
| 1063 | PGSCAN_KHUGEPAGED - PGSCAN_KSWAPD); |
| 1064 | |
| 1065 | if (current_is_kswapd()) |
| 1066 | return 0; |
| 1067 | if (current_is_khugepaged()) |
| 1068 | return PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD; |
| 1069 | return PGSTEAL_DIRECT - PGSTEAL_KSWAPD; |
| 1070 | } |
| 1071 | |
| 1072 | static inline int is_page_cache_freeable(struct folio *folio) |
| 1073 | { |
| 1074 | /* |
| 1075 | * A freeable page cache folio is referenced only by the caller |
| 1076 | * that isolated the folio, the page cache and optional filesystem |
| 1077 | * private data at folio->private. |
| 1078 | */ |
| 1079 | return folio_ref_count(folio) - folio_test_private(folio) == |
| 1080 | 1 + folio_nr_pages(folio); |
| 1081 | } |
| 1082 | |
| 1083 | /* |
| 1084 | * We detected a synchronous write error writing a folio out. Probably |
| 1085 | * -ENOSPC. We need to propagate that into the address_space for a subsequent |
| 1086 | * fsync(), msync() or close(). |
| 1087 | * |
| 1088 | * The tricky part is that after writepage we cannot touch the mapping: nothing |
| 1089 | * prevents it from being freed up. But we have a ref on the folio and once |
| 1090 | * that folio is locked, the mapping is pinned. |
| 1091 | * |
| 1092 | * We're allowed to run sleeping folio_lock() here because we know the caller has |
| 1093 | * __GFP_FS. |
| 1094 | */ |
| 1095 | static void handle_write_error(struct address_space *mapping, |
| 1096 | struct folio *folio, int error) |
| 1097 | { |
| 1098 | folio_lock(folio); |
| 1099 | if (folio_mapping(folio) == mapping) |
| 1100 | mapping_set_error(mapping, error); |
| 1101 | folio_unlock(folio); |
| 1102 | } |
| 1103 | |
| 1104 | static bool skip_throttle_noprogress(pg_data_t *pgdat) |
| 1105 | { |
| 1106 | int reclaimable = 0, write_pending = 0; |
| 1107 | int i; |
| 1108 | |
| 1109 | /* |
| 1110 | * If kswapd is disabled, reschedule if necessary but do not |
| 1111 | * throttle as the system is likely near OOM. |
| 1112 | */ |
| 1113 | if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES) |
| 1114 | return true; |
| 1115 | |
| 1116 | /* |
| 1117 | * If there are a lot of dirty/writeback folios then do not |
| 1118 | * throttle as throttling will occur when the folios cycle |
| 1119 | * towards the end of the LRU if still under writeback. |
| 1120 | */ |
| 1121 | for (i = 0; i < MAX_NR_ZONES; i++) { |
| 1122 | struct zone *zone = pgdat->node_zones + i; |
| 1123 | |
| 1124 | if (!managed_zone(zone)) |
| 1125 | continue; |
| 1126 | |
| 1127 | reclaimable += zone_reclaimable_pages(zone); |
| 1128 | write_pending += zone_page_state_snapshot(zone, |
| 1129 | NR_ZONE_WRITE_PENDING); |
| 1130 | } |
| 1131 | if (2 * write_pending <= reclaimable) |
| 1132 | return true; |
| 1133 | |
| 1134 | return false; |
| 1135 | } |
| 1136 | |
| 1137 | void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason) |
| 1138 | { |
| 1139 | wait_queue_head_t *wqh = &pgdat->reclaim_wait[reason]; |
| 1140 | long timeout, ret; |
| 1141 | DEFINE_WAIT(wait); |
| 1142 | |
| 1143 | /* |
| 1144 | * Do not throttle IO workers, kthreads other than kswapd or |
| 1145 | * workqueues. They may be required for reclaim to make |
| 1146 | * forward progress (e.g. journalling workqueues or kthreads). |
| 1147 | */ |
| 1148 | if (!current_is_kswapd() && |
| 1149 | current->flags & (PF_IO_WORKER|PF_KTHREAD)) { |
| 1150 | cond_resched(); |
| 1151 | return; |
| 1152 | } |
| 1153 | |
| 1154 | /* |
| 1155 | * These figures are pulled out of thin air. |
| 1156 | * VMSCAN_THROTTLE_ISOLATED is a transient condition based on too many |
| 1157 | * parallel reclaimers which is a short-lived event so the timeout is |
| 1158 | * short. Failing to make progress or waiting on writeback are |
| 1159 | * potentially long-lived events so use a longer timeout. This is shaky |
| 1160 | * logic as a failure to make progress could be due to anything from |
| 1161 | * writeback to a slow device to excessive referenced folios at the tail |
| 1162 | * of the inactive LRU. |
| 1163 | */ |
| 1164 | switch(reason) { |
| 1165 | case VMSCAN_THROTTLE_WRITEBACK: |
| 1166 | timeout = HZ/10; |
| 1167 | |
| 1168 | if (atomic_inc_return(&pgdat->nr_writeback_throttled) == 1) { |
| 1169 | WRITE_ONCE(pgdat->nr_reclaim_start, |
| 1170 | node_page_state(pgdat, NR_THROTTLED_WRITTEN)); |
| 1171 | } |
| 1172 | |
| 1173 | break; |
| 1174 | case VMSCAN_THROTTLE_CONGESTED: |
| 1175 | fallthrough; |
| 1176 | case VMSCAN_THROTTLE_NOPROGRESS: |
| 1177 | if (skip_throttle_noprogress(pgdat)) { |
| 1178 | cond_resched(); |
| 1179 | return; |
| 1180 | } |
| 1181 | |
| 1182 | timeout = 1; |
| 1183 | |
| 1184 | break; |
| 1185 | case VMSCAN_THROTTLE_ISOLATED: |
| 1186 | timeout = HZ/50; |
| 1187 | break; |
| 1188 | default: |
| 1189 | WARN_ON_ONCE(1); |
| 1190 | timeout = HZ; |
| 1191 | break; |
| 1192 | } |
| 1193 | |
| 1194 | prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE); |
| 1195 | ret = schedule_timeout(timeout); |
| 1196 | finish_wait(wqh, &wait); |
| 1197 | |
| 1198 | if (reason == VMSCAN_THROTTLE_WRITEBACK) |
| 1199 | atomic_dec(&pgdat->nr_writeback_throttled); |
| 1200 | |
| 1201 | trace_mm_vmscan_throttled(pgdat->node_id, jiffies_to_usecs(timeout), |
| 1202 | jiffies_to_usecs(timeout - ret), |
| 1203 | reason); |
| 1204 | } |
| 1205 | |
| 1206 | /* |
| 1207 | * Account for folios written if tasks are throttled waiting on dirty |
| 1208 | * folios to clean. If enough folios have been cleaned since throttling |
| 1209 | * started then wakeup the throttled tasks. |
| 1210 | */ |
| 1211 | void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio, |
| 1212 | int nr_throttled) |
| 1213 | { |
| 1214 | unsigned long nr_written; |
| 1215 | |
| 1216 | node_stat_add_folio(folio, NR_THROTTLED_WRITTEN); |
| 1217 | |
| 1218 | /* |
| 1219 | * This is an inaccurate read as the per-cpu deltas may not |
| 1220 | * be synchronised. However, given that the system is |
| 1221 | * writeback throttled, it is not worth taking the penalty |
| 1222 | * of getting an accurate count. At worst, the throttle |
| 1223 | * timeout guarantees forward progress. |
| 1224 | */ |
| 1225 | nr_written = node_page_state(pgdat, NR_THROTTLED_WRITTEN) - |
| 1226 | READ_ONCE(pgdat->nr_reclaim_start); |
| 1227 | |
| 1228 | if (nr_written > SWAP_CLUSTER_MAX * nr_throttled) |
| 1229 | wake_up(&pgdat->reclaim_wait[VMSCAN_THROTTLE_WRITEBACK]); |
| 1230 | } |
| 1231 | |
| 1232 | /* possible outcome of pageout() */ |
| 1233 | typedef enum { |
| 1234 | /* failed to write folio out, folio is locked */ |
| 1235 | PAGE_KEEP, |
| 1236 | /* move folio to the active list, folio is locked */ |
| 1237 | PAGE_ACTIVATE, |
| 1238 | /* folio has been sent to the disk successfully, folio is unlocked */ |
| 1239 | PAGE_SUCCESS, |
| 1240 | /* folio is clean and locked */ |
| 1241 | PAGE_CLEAN, |
| 1242 | } pageout_t; |
| 1243 | |
| 1244 | /* |
| 1245 | * pageout is called by shrink_folio_list() for each dirty folio. |
| 1246 | * Calls ->writepage(). |
| 1247 | */ |
| 1248 | static pageout_t pageout(struct folio *folio, struct address_space *mapping, |
| 1249 | struct swap_iocb **plug) |
| 1250 | { |
| 1251 | /* |
| 1252 | * If the folio is dirty, only perform writeback if that write |
| 1253 | * will be non-blocking. To prevent this allocation from being |
| 1254 | * stalled by pagecache activity. But note that there may be |
| 1255 | * stalls if we need to run get_block(). We could test |
| 1256 | * PagePrivate for that. |
| 1257 | * |
| 1258 | * If this process is currently in __generic_file_write_iter() against |
| 1259 | * this folio's queue, we can perform writeback even if that |
| 1260 | * will block. |
| 1261 | * |
| 1262 | * If the folio is swapcache, write it back even if that would |
| 1263 | * block, for some throttling. This happens by accident, because |
| 1264 | * swap_backing_dev_info is bust: it doesn't reflect the |
| 1265 | * congestion state of the swapdevs. Easy to fix, if needed. |
| 1266 | */ |
| 1267 | if (!is_page_cache_freeable(folio)) |
| 1268 | return PAGE_KEEP; |
| 1269 | if (!mapping) { |
| 1270 | /* |
| 1271 | * Some data journaling orphaned folios can have |
| 1272 | * folio->mapping == NULL while being dirty with clean buffers. |
| 1273 | */ |
| 1274 | if (folio_test_private(folio)) { |
| 1275 | if (try_to_free_buffers(folio)) { |
| 1276 | folio_clear_dirty(folio); |
| 1277 | pr_info("%s: orphaned folio\n", __func__); |
| 1278 | return PAGE_CLEAN; |
| 1279 | } |
| 1280 | } |
| 1281 | return PAGE_KEEP; |
| 1282 | } |
| 1283 | if (mapping->a_ops->writepage == NULL) |
| 1284 | return PAGE_ACTIVATE; |
| 1285 | |
| 1286 | if (folio_clear_dirty_for_io(folio)) { |
| 1287 | int res; |
| 1288 | struct writeback_control wbc = { |
| 1289 | .sync_mode = WB_SYNC_NONE, |
| 1290 | .nr_to_write = SWAP_CLUSTER_MAX, |
| 1291 | .range_start = 0, |
| 1292 | .range_end = LLONG_MAX, |
| 1293 | .for_reclaim = 1, |
| 1294 | .swap_plug = plug, |
| 1295 | }; |
| 1296 | |
| 1297 | folio_set_reclaim(folio); |
| 1298 | res = mapping->a_ops->writepage(&folio->page, &wbc); |
| 1299 | if (res < 0) |
| 1300 | handle_write_error(mapping, folio, res); |
| 1301 | if (res == AOP_WRITEPAGE_ACTIVATE) { |
| 1302 | folio_clear_reclaim(folio); |
| 1303 | return PAGE_ACTIVATE; |
| 1304 | } |
| 1305 | |
| 1306 | if (!folio_test_writeback(folio)) { |
| 1307 | /* synchronous write or broken a_ops? */ |
| 1308 | folio_clear_reclaim(folio); |
| 1309 | } |
| 1310 | trace_mm_vmscan_write_folio(folio); |
| 1311 | node_stat_add_folio(folio, NR_VMSCAN_WRITE); |
| 1312 | return PAGE_SUCCESS; |
| 1313 | } |
| 1314 | |
| 1315 | return PAGE_CLEAN; |
| 1316 | } |
| 1317 | |
| 1318 | /* |
| 1319 | * Same as remove_mapping, but if the folio is removed from the mapping, it |
| 1320 | * gets returned with a refcount of 0. |
| 1321 | */ |
| 1322 | static int __remove_mapping(struct address_space *mapping, struct folio *folio, |
| 1323 | bool reclaimed, struct mem_cgroup *target_memcg) |
| 1324 | { |
| 1325 | int refcount; |
| 1326 | void *shadow = NULL; |
| 1327 | |
| 1328 | BUG_ON(!folio_test_locked(folio)); |
| 1329 | BUG_ON(mapping != folio_mapping(folio)); |
| 1330 | |
| 1331 | if (!folio_test_swapcache(folio)) |
| 1332 | spin_lock(&mapping->host->i_lock); |
| 1333 | xa_lock_irq(&mapping->i_pages); |
| 1334 | /* |
| 1335 | * The non racy check for a busy folio. |
| 1336 | * |
| 1337 | * Must be careful with the order of the tests. When someone has |
| 1338 | * a ref to the folio, it may be possible that they dirty it then |
| 1339 | * drop the reference. So if the dirty flag is tested before the |
| 1340 | * refcount here, then the following race may occur: |
| 1341 | * |
| 1342 | * get_user_pages(&page); |
| 1343 | * [user mapping goes away] |
| 1344 | * write_to(page); |
| 1345 | * !folio_test_dirty(folio) [good] |
| 1346 | * folio_set_dirty(folio); |
| 1347 | * folio_put(folio); |
| 1348 | * !refcount(folio) [good, discard it] |
| 1349 | * |
| 1350 | * [oops, our write_to data is lost] |
| 1351 | * |
| 1352 | * Reversing the order of the tests ensures such a situation cannot |
| 1353 | * escape unnoticed. The smp_rmb is needed to ensure the folio->flags |
| 1354 | * load is not satisfied before that of folio->_refcount. |
| 1355 | * |
| 1356 | * Note that if the dirty flag is always set via folio_mark_dirty, |
| 1357 | * and thus under the i_pages lock, then this ordering is not required. |
| 1358 | */ |
| 1359 | refcount = 1 + folio_nr_pages(folio); |
| 1360 | if (!folio_ref_freeze(folio, refcount)) |
| 1361 | goto cannot_free; |
| 1362 | /* note: atomic_cmpxchg in folio_ref_freeze provides the smp_rmb */ |
| 1363 | if (unlikely(folio_test_dirty(folio))) { |
| 1364 | folio_ref_unfreeze(folio, refcount); |
| 1365 | goto cannot_free; |
| 1366 | } |
| 1367 | |
| 1368 | if (folio_test_swapcache(folio)) { |
| 1369 | swp_entry_t swap = folio_swap_entry(folio); |
| 1370 | |
| 1371 | if (reclaimed && !mapping_exiting(mapping)) |
| 1372 | shadow = workingset_eviction(folio, target_memcg); |
| 1373 | __delete_from_swap_cache(folio, swap, shadow); |
| 1374 | mem_cgroup_swapout(folio, swap); |
| 1375 | xa_unlock_irq(&mapping->i_pages); |
| 1376 | put_swap_folio(folio, swap); |
| 1377 | } else { |
| 1378 | void (*free_folio)(struct folio *); |
| 1379 | |
| 1380 | free_folio = mapping->a_ops->free_folio; |
| 1381 | /* |
| 1382 | * Remember a shadow entry for reclaimed file cache in |
| 1383 | * order to detect refaults, thus thrashing, later on. |
| 1384 | * |
| 1385 | * But don't store shadows in an address space that is |
| 1386 | * already exiting. This is not just an optimization, |
| 1387 | * inode reclaim needs to empty out the radix tree or |
| 1388 | * the nodes are lost. Don't plant shadows behind its |
| 1389 | * back. |
| 1390 | * |
| 1391 | * We also don't store shadows for DAX mappings because the |
| 1392 | * only page cache folios found in these are zero pages |
| 1393 | * covering holes, and because we don't want to mix DAX |
| 1394 | * exceptional entries and shadow exceptional entries in the |
| 1395 | * same address_space. |
| 1396 | */ |
| 1397 | if (reclaimed && folio_is_file_lru(folio) && |
| 1398 | !mapping_exiting(mapping) && !dax_mapping(mapping)) |
| 1399 | shadow = workingset_eviction(folio, target_memcg); |
| 1400 | __filemap_remove_folio(folio, shadow); |
| 1401 | xa_unlock_irq(&mapping->i_pages); |
| 1402 | if (mapping_shrinkable(mapping)) |
| 1403 | inode_add_lru(mapping->host); |
| 1404 | spin_unlock(&mapping->host->i_lock); |
| 1405 | |
| 1406 | if (free_folio) |
| 1407 | free_folio(folio); |
| 1408 | } |
| 1409 | |
| 1410 | return 1; |
| 1411 | |
| 1412 | cannot_free: |
| 1413 | xa_unlock_irq(&mapping->i_pages); |
| 1414 | if (!folio_test_swapcache(folio)) |
| 1415 | spin_unlock(&mapping->host->i_lock); |
| 1416 | return 0; |
| 1417 | } |
| 1418 | |
| 1419 | /** |
| 1420 | * remove_mapping() - Attempt to remove a folio from its mapping. |
| 1421 | * @mapping: The address space. |
| 1422 | * @folio: The folio to remove. |
| 1423 | * |
| 1424 | * If the folio is dirty, under writeback or if someone else has a ref |
| 1425 | * on it, removal will fail. |
| 1426 | * Return: The number of pages removed from the mapping. 0 if the folio |
| 1427 | * could not be removed. |
| 1428 | * Context: The caller should have a single refcount on the folio and |
| 1429 | * hold its lock. |
| 1430 | */ |
| 1431 | long remove_mapping(struct address_space *mapping, struct folio *folio) |
| 1432 | { |
| 1433 | if (__remove_mapping(mapping, folio, false, NULL)) { |
| 1434 | /* |
| 1435 | * Unfreezing the refcount with 1 effectively |
| 1436 | * drops the pagecache ref for us without requiring another |
| 1437 | * atomic operation. |
| 1438 | */ |
| 1439 | folio_ref_unfreeze(folio, 1); |
| 1440 | return folio_nr_pages(folio); |
| 1441 | } |
| 1442 | return 0; |
| 1443 | } |
| 1444 | |
| 1445 | /** |
| 1446 | * folio_putback_lru - Put previously isolated folio onto appropriate LRU list. |
| 1447 | * @folio: Folio to be returned to an LRU list. |
| 1448 | * |
| 1449 | * Add previously isolated @folio to appropriate LRU list. |
| 1450 | * The folio may still be unevictable for other reasons. |
| 1451 | * |
| 1452 | * Context: lru_lock must not be held, interrupts must be enabled. |
| 1453 | */ |
| 1454 | void folio_putback_lru(struct folio *folio) |
| 1455 | { |
| 1456 | folio_add_lru(folio); |
| 1457 | folio_put(folio); /* drop ref from isolate */ |
| 1458 | } |
| 1459 | |
| 1460 | enum folio_references { |
| 1461 | FOLIOREF_RECLAIM, |
| 1462 | FOLIOREF_RECLAIM_CLEAN, |
| 1463 | FOLIOREF_KEEP, |
| 1464 | FOLIOREF_ACTIVATE, |
| 1465 | }; |
| 1466 | |
| 1467 | static enum folio_references folio_check_references(struct folio *folio, |
| 1468 | struct scan_control *sc) |
| 1469 | { |
| 1470 | int referenced_ptes, referenced_folio; |
| 1471 | unsigned long vm_flags; |
| 1472 | |
| 1473 | referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup, |
| 1474 | &vm_flags); |
| 1475 | referenced_folio = folio_test_clear_referenced(folio); |
| 1476 | |
| 1477 | /* |
| 1478 | * The supposedly reclaimable folio was found to be in a VM_LOCKED vma. |
| 1479 | * Let the folio, now marked Mlocked, be moved to the unevictable list. |
| 1480 | */ |
| 1481 | if (vm_flags & VM_LOCKED) |
| 1482 | return FOLIOREF_ACTIVATE; |
| 1483 | |
| 1484 | /* rmap lock contention: rotate */ |
| 1485 | if (referenced_ptes == -1) |
| 1486 | return FOLIOREF_KEEP; |
| 1487 | |
| 1488 | if (referenced_ptes) { |
| 1489 | /* |
| 1490 | * All mapped folios start out with page table |
| 1491 | * references from the instantiating fault, so we need |
| 1492 | * to look twice if a mapped file/anon folio is used more |
| 1493 | * than once. |
| 1494 | * |
| 1495 | * Mark it and spare it for another trip around the |
| 1496 | * inactive list. Another page table reference will |
| 1497 | * lead to its activation. |
| 1498 | * |
| 1499 | * Note: the mark is set for activated folios as well |
| 1500 | * so that recently deactivated but used folios are |
| 1501 | * quickly recovered. |
| 1502 | */ |
| 1503 | folio_set_referenced(folio); |
| 1504 | |
| 1505 | if (referenced_folio || referenced_ptes > 1) |
| 1506 | return FOLIOREF_ACTIVATE; |
| 1507 | |
| 1508 | /* |
| 1509 | * Activate file-backed executable folios after first usage. |
| 1510 | */ |
| 1511 | if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) |
| 1512 | return FOLIOREF_ACTIVATE; |
| 1513 | |
| 1514 | return FOLIOREF_KEEP; |
| 1515 | } |
| 1516 | |
| 1517 | /* Reclaim if clean, defer dirty folios to writeback */ |
| 1518 | if (referenced_folio && folio_is_file_lru(folio)) |
| 1519 | return FOLIOREF_RECLAIM_CLEAN; |
| 1520 | |
| 1521 | return FOLIOREF_RECLAIM; |
| 1522 | } |
| 1523 | |
| 1524 | /* Check if a folio is dirty or under writeback */ |
| 1525 | static void folio_check_dirty_writeback(struct folio *folio, |
| 1526 | bool *dirty, bool *writeback) |
| 1527 | { |
| 1528 | struct address_space *mapping; |
| 1529 | |
| 1530 | /* |
| 1531 | * Anonymous folios are not handled by flushers and must be written |
| 1532 | * from reclaim context. Do not stall reclaim based on them. |
| 1533 | * MADV_FREE anonymous folios are put into inactive file list too. |
| 1534 | * They could be mistakenly treated as file lru. So further anon |
| 1535 | * test is needed. |
| 1536 | */ |
| 1537 | if (!folio_is_file_lru(folio) || |
| 1538 | (folio_test_anon(folio) && !folio_test_swapbacked(folio))) { |
| 1539 | *dirty = false; |
| 1540 | *writeback = false; |
| 1541 | return; |
| 1542 | } |
| 1543 | |
| 1544 | /* By default assume that the folio flags are accurate */ |
| 1545 | *dirty = folio_test_dirty(folio); |
| 1546 | *writeback = folio_test_writeback(folio); |
| 1547 | |
| 1548 | /* Verify dirty/writeback state if the filesystem supports it */ |
| 1549 | if (!folio_test_private(folio)) |
| 1550 | return; |
| 1551 | |
| 1552 | mapping = folio_mapping(folio); |
| 1553 | if (mapping && mapping->a_ops->is_dirty_writeback) |
| 1554 | mapping->a_ops->is_dirty_writeback(folio, dirty, writeback); |
| 1555 | } |
| 1556 | |
| 1557 | static struct page *alloc_demote_page(struct page *page, unsigned long private) |
| 1558 | { |
| 1559 | struct page *target_page; |
| 1560 | nodemask_t *allowed_mask; |
| 1561 | struct migration_target_control *mtc; |
| 1562 | |
| 1563 | mtc = (struct migration_target_control *)private; |
| 1564 | |
| 1565 | allowed_mask = mtc->nmask; |
| 1566 | /* |
| 1567 | * make sure we allocate from the target node first also trying to |
| 1568 | * demote or reclaim pages from the target node via kswapd if we are |
| 1569 | * low on free memory on target node. If we don't do this and if |
| 1570 | * we have free memory on the slower(lower) memtier, we would start |
| 1571 | * allocating pages from slower(lower) memory tiers without even forcing |
| 1572 | * a demotion of cold pages from the target memtier. This can result |
| 1573 | * in the kernel placing hot pages in slower(lower) memory tiers. |
| 1574 | */ |
| 1575 | mtc->nmask = NULL; |
| 1576 | mtc->gfp_mask |= __GFP_THISNODE; |
| 1577 | target_page = alloc_migration_target(page, (unsigned long)mtc); |
| 1578 | if (target_page) |
| 1579 | return target_page; |
| 1580 | |
| 1581 | mtc->gfp_mask &= ~__GFP_THISNODE; |
| 1582 | mtc->nmask = allowed_mask; |
| 1583 | |
| 1584 | return alloc_migration_target(page, (unsigned long)mtc); |
| 1585 | } |
| 1586 | |
| 1587 | /* |
| 1588 | * Take folios on @demote_folios and attempt to demote them to another node. |
| 1589 | * Folios which are not demoted are left on @demote_folios. |
| 1590 | */ |
| 1591 | static unsigned int demote_folio_list(struct list_head *demote_folios, |
| 1592 | struct pglist_data *pgdat) |
| 1593 | { |
| 1594 | int target_nid = next_demotion_node(pgdat->node_id); |
| 1595 | unsigned int nr_succeeded; |
| 1596 | nodemask_t allowed_mask; |
| 1597 | |
| 1598 | struct migration_target_control mtc = { |
| 1599 | /* |
| 1600 | * Allocate from 'node', or fail quickly and quietly. |
| 1601 | * When this happens, 'page' will likely just be discarded |
| 1602 | * instead of migrated. |
| 1603 | */ |
| 1604 | .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOWARN | |
| 1605 | __GFP_NOMEMALLOC | GFP_NOWAIT, |
| 1606 | .nid = target_nid, |
| 1607 | .nmask = &allowed_mask |
| 1608 | }; |
| 1609 | |
| 1610 | if (list_empty(demote_folios)) |
| 1611 | return 0; |
| 1612 | |
| 1613 | if (target_nid == NUMA_NO_NODE) |
| 1614 | return 0; |
| 1615 | |
| 1616 | node_get_allowed_targets(pgdat, &allowed_mask); |
| 1617 | |
| 1618 | /* Demotion ignores all cpuset and mempolicy settings */ |
| 1619 | migrate_pages(demote_folios, alloc_demote_page, NULL, |
| 1620 | (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION, |
| 1621 | &nr_succeeded); |
| 1622 | |
| 1623 | __count_vm_events(PGDEMOTE_KSWAPD + reclaimer_offset(), nr_succeeded); |
| 1624 | |
| 1625 | return nr_succeeded; |
| 1626 | } |
| 1627 | |
| 1628 | static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask) |
| 1629 | { |
| 1630 | if (gfp_mask & __GFP_FS) |
| 1631 | return true; |
| 1632 | if (!folio_test_swapcache(folio) || !(gfp_mask & __GFP_IO)) |
| 1633 | return false; |
| 1634 | /* |
| 1635 | * We can "enter_fs" for swap-cache with only __GFP_IO |
| 1636 | * providing this isn't SWP_FS_OPS. |
| 1637 | * ->flags can be updated non-atomicially (scan_swap_map_slots), |
| 1638 | * but that will never affect SWP_FS_OPS, so the data_race |
| 1639 | * is safe. |
| 1640 | */ |
| 1641 | return !data_race(folio_swap_flags(folio) & SWP_FS_OPS); |
| 1642 | } |
| 1643 | |
| 1644 | /* |
| 1645 | * shrink_folio_list() returns the number of reclaimed pages |
| 1646 | */ |
| 1647 | static unsigned int shrink_folio_list(struct list_head *folio_list, |
| 1648 | struct pglist_data *pgdat, struct scan_control *sc, |
| 1649 | struct reclaim_stat *stat, bool ignore_references) |
| 1650 | { |
| 1651 | LIST_HEAD(ret_folios); |
| 1652 | LIST_HEAD(free_folios); |
| 1653 | LIST_HEAD(demote_folios); |
| 1654 | unsigned int nr_reclaimed = 0; |
| 1655 | unsigned int pgactivate = 0; |
| 1656 | bool do_demote_pass; |
| 1657 | struct swap_iocb *plug = NULL; |
| 1658 | |
| 1659 | memset(stat, 0, sizeof(*stat)); |
| 1660 | cond_resched(); |
| 1661 | do_demote_pass = can_demote(pgdat->node_id, sc); |
| 1662 | |
| 1663 | retry: |
| 1664 | while (!list_empty(folio_list)) { |
| 1665 | struct address_space *mapping; |
| 1666 | struct folio *folio; |
| 1667 | enum folio_references references = FOLIOREF_RECLAIM; |
| 1668 | bool dirty, writeback; |
| 1669 | unsigned int nr_pages; |
| 1670 | |
| 1671 | cond_resched(); |
| 1672 | |
| 1673 | folio = lru_to_folio(folio_list); |
| 1674 | list_del(&folio->lru); |
| 1675 | |
| 1676 | if (!folio_trylock(folio)) |
| 1677 | goto keep; |
| 1678 | |
| 1679 | VM_BUG_ON_FOLIO(folio_test_active(folio), folio); |
| 1680 | |
| 1681 | nr_pages = folio_nr_pages(folio); |
| 1682 | |
| 1683 | /* Account the number of base pages */ |
| 1684 | sc->nr_scanned += nr_pages; |
| 1685 | |
| 1686 | if (unlikely(!folio_evictable(folio))) |
| 1687 | goto activate_locked; |
| 1688 | |
| 1689 | if (!sc->may_unmap && folio_mapped(folio)) |
| 1690 | goto keep_locked; |
| 1691 | |
| 1692 | /* folio_update_gen() tried to promote this page? */ |
| 1693 | if (lru_gen_enabled() && !ignore_references && |
| 1694 | folio_mapped(folio) && folio_test_referenced(folio)) |
| 1695 | goto keep_locked; |
| 1696 | |
| 1697 | /* |
| 1698 | * The number of dirty pages determines if a node is marked |
| 1699 | * reclaim_congested. kswapd will stall and start writing |
| 1700 | * folios if the tail of the LRU is all dirty unqueued folios. |
| 1701 | */ |
| 1702 | folio_check_dirty_writeback(folio, &dirty, &writeback); |
| 1703 | if (dirty || writeback) |
| 1704 | stat->nr_dirty += nr_pages; |
| 1705 | |
| 1706 | if (dirty && !writeback) |
| 1707 | stat->nr_unqueued_dirty += nr_pages; |
| 1708 | |
| 1709 | /* |
| 1710 | * Treat this folio as congested if folios are cycling |
| 1711 | * through the LRU so quickly that the folios marked |
| 1712 | * for immediate reclaim are making it to the end of |
| 1713 | * the LRU a second time. |
| 1714 | */ |
| 1715 | if (writeback && folio_test_reclaim(folio)) |
| 1716 | stat->nr_congested += nr_pages; |
| 1717 | |
| 1718 | /* |
| 1719 | * If a folio at the tail of the LRU is under writeback, there |
| 1720 | * are three cases to consider. |
| 1721 | * |
| 1722 | * 1) If reclaim is encountering an excessive number |
| 1723 | * of folios under writeback and this folio has both |
| 1724 | * the writeback and reclaim flags set, then it |
| 1725 | * indicates that folios are being queued for I/O but |
| 1726 | * are being recycled through the LRU before the I/O |
| 1727 | * can complete. Waiting on the folio itself risks an |
| 1728 | * indefinite stall if it is impossible to writeback |
| 1729 | * the folio due to I/O error or disconnected storage |
| 1730 | * so instead note that the LRU is being scanned too |
| 1731 | * quickly and the caller can stall after the folio |
| 1732 | * list has been processed. |
| 1733 | * |
| 1734 | * 2) Global or new memcg reclaim encounters a folio that is |
| 1735 | * not marked for immediate reclaim, or the caller does not |
| 1736 | * have __GFP_FS (or __GFP_IO if it's simply going to swap, |
| 1737 | * not to fs). In this case mark the folio for immediate |
| 1738 | * reclaim and continue scanning. |
| 1739 | * |
| 1740 | * Require may_enter_fs() because we would wait on fs, which |
| 1741 | * may not have submitted I/O yet. And the loop driver might |
| 1742 | * enter reclaim, and deadlock if it waits on a folio for |
| 1743 | * which it is needed to do the write (loop masks off |
| 1744 | * __GFP_IO|__GFP_FS for this reason); but more thought |
| 1745 | * would probably show more reasons. |
| 1746 | * |
| 1747 | * 3) Legacy memcg encounters a folio that already has the |
| 1748 | * reclaim flag set. memcg does not have any dirty folio |
| 1749 | * throttling so we could easily OOM just because too many |
| 1750 | * folios are in writeback and there is nothing else to |
| 1751 | * reclaim. Wait for the writeback to complete. |
| 1752 | * |
| 1753 | * In cases 1) and 2) we activate the folios to get them out of |
| 1754 | * the way while we continue scanning for clean folios on the |
| 1755 | * inactive list and refilling from the active list. The |
| 1756 | * observation here is that waiting for disk writes is more |
| 1757 | * expensive than potentially causing reloads down the line. |
| 1758 | * Since they're marked for immediate reclaim, they won't put |
| 1759 | * memory pressure on the cache working set any longer than it |
| 1760 | * takes to write them to disk. |
| 1761 | */ |
| 1762 | if (folio_test_writeback(folio)) { |
| 1763 | /* Case 1 above */ |
| 1764 | if (current_is_kswapd() && |
| 1765 | folio_test_reclaim(folio) && |
| 1766 | test_bit(PGDAT_WRITEBACK, &pgdat->flags)) { |
| 1767 | stat->nr_immediate += nr_pages; |
| 1768 | goto activate_locked; |
| 1769 | |
| 1770 | /* Case 2 above */ |
| 1771 | } else if (writeback_throttling_sane(sc) || |
| 1772 | !folio_test_reclaim(folio) || |
| 1773 | !may_enter_fs(folio, sc->gfp_mask)) { |
| 1774 | /* |
| 1775 | * This is slightly racy - |
| 1776 | * folio_end_writeback() might have |
| 1777 | * just cleared the reclaim flag, then |
| 1778 | * setting the reclaim flag here ends up |
| 1779 | * interpreted as the readahead flag - but |
| 1780 | * that does not matter enough to care. |
| 1781 | * What we do want is for this folio to |
| 1782 | * have the reclaim flag set next time |
| 1783 | * memcg reclaim reaches the tests above, |
| 1784 | * so it will then wait for writeback to |
| 1785 | * avoid OOM; and it's also appropriate |
| 1786 | * in global reclaim. |
| 1787 | */ |
| 1788 | folio_set_reclaim(folio); |
| 1789 | stat->nr_writeback += nr_pages; |
| 1790 | goto activate_locked; |
| 1791 | |
| 1792 | /* Case 3 above */ |
| 1793 | } else { |
| 1794 | folio_unlock(folio); |
| 1795 | folio_wait_writeback(folio); |
| 1796 | /* then go back and try same folio again */ |
| 1797 | list_add_tail(&folio->lru, folio_list); |
| 1798 | continue; |
| 1799 | } |
| 1800 | } |
| 1801 | |
| 1802 | if (!ignore_references) |
| 1803 | references = folio_check_references(folio, sc); |
| 1804 | |
| 1805 | switch (references) { |
| 1806 | case FOLIOREF_ACTIVATE: |
| 1807 | goto activate_locked; |
| 1808 | case FOLIOREF_KEEP: |
| 1809 | stat->nr_ref_keep += nr_pages; |
| 1810 | goto keep_locked; |
| 1811 | case FOLIOREF_RECLAIM: |
| 1812 | case FOLIOREF_RECLAIM_CLEAN: |
| 1813 | ; /* try to reclaim the folio below */ |
| 1814 | } |
| 1815 | |
| 1816 | /* |
| 1817 | * Before reclaiming the folio, try to relocate |
| 1818 | * its contents to another node. |
| 1819 | */ |
| 1820 | if (do_demote_pass && |
| 1821 | (thp_migration_supported() || !folio_test_large(folio))) { |
| 1822 | list_add(&folio->lru, &demote_folios); |
| 1823 | folio_unlock(folio); |
| 1824 | continue; |
| 1825 | } |
| 1826 | |
| 1827 | /* |
| 1828 | * Anonymous process memory has backing store? |
| 1829 | * Try to allocate it some swap space here. |
| 1830 | * Lazyfree folio could be freed directly |
| 1831 | */ |
| 1832 | if (folio_test_anon(folio) && folio_test_swapbacked(folio)) { |
| 1833 | if (!folio_test_swapcache(folio)) { |
| 1834 | if (!(sc->gfp_mask & __GFP_IO)) |
| 1835 | goto keep_locked; |
| 1836 | if (folio_maybe_dma_pinned(folio)) |
| 1837 | goto keep_locked; |
| 1838 | if (folio_test_large(folio)) { |
| 1839 | /* cannot split folio, skip it */ |
| 1840 | if (!can_split_folio(folio, NULL)) |
| 1841 | goto activate_locked; |
| 1842 | /* |
| 1843 | * Split folios without a PMD map right |
| 1844 | * away. Chances are some or all of the |
| 1845 | * tail pages can be freed without IO. |
| 1846 | */ |
| 1847 | if (!folio_entire_mapcount(folio) && |
| 1848 | split_folio_to_list(folio, |
| 1849 | folio_list)) |
| 1850 | goto activate_locked; |
| 1851 | } |
| 1852 | if (!add_to_swap(folio)) { |
| 1853 | if (!folio_test_large(folio)) |
| 1854 | goto activate_locked_split; |
| 1855 | /* Fallback to swap normal pages */ |
| 1856 | if (split_folio_to_list(folio, |
| 1857 | folio_list)) |
| 1858 | goto activate_locked; |
| 1859 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| 1860 | count_vm_event(THP_SWPOUT_FALLBACK); |
| 1861 | #endif |
| 1862 | if (!add_to_swap(folio)) |
| 1863 | goto activate_locked_split; |
| 1864 | } |
| 1865 | } |
| 1866 | } else if (folio_test_swapbacked(folio) && |
| 1867 | folio_test_large(folio)) { |
| 1868 | /* Split shmem folio */ |
| 1869 | if (split_folio_to_list(folio, folio_list)) |
| 1870 | goto keep_locked; |
| 1871 | } |
| 1872 | |
| 1873 | /* |
| 1874 | * If the folio was split above, the tail pages will make |
| 1875 | * their own pass through this function and be accounted |
| 1876 | * then. |
| 1877 | */ |
| 1878 | if ((nr_pages > 1) && !folio_test_large(folio)) { |
| 1879 | sc->nr_scanned -= (nr_pages - 1); |
| 1880 | nr_pages = 1; |
| 1881 | } |
| 1882 | |
| 1883 | /* |
| 1884 | * The folio is mapped into the page tables of one or more |
| 1885 | * processes. Try to unmap it here. |
| 1886 | */ |
| 1887 | if (folio_mapped(folio)) { |
| 1888 | enum ttu_flags flags = TTU_BATCH_FLUSH; |
| 1889 | bool was_swapbacked = folio_test_swapbacked(folio); |
| 1890 | |
| 1891 | if (folio_test_pmd_mappable(folio)) |
| 1892 | flags |= TTU_SPLIT_HUGE_PMD; |
| 1893 | |
| 1894 | try_to_unmap(folio, flags); |
| 1895 | if (folio_mapped(folio)) { |
| 1896 | stat->nr_unmap_fail += nr_pages; |
| 1897 | if (!was_swapbacked && |
| 1898 | folio_test_swapbacked(folio)) |
| 1899 | stat->nr_lazyfree_fail += nr_pages; |
| 1900 | goto activate_locked; |
| 1901 | } |
| 1902 | } |
| 1903 | |
| 1904 | mapping = folio_mapping(folio); |
| 1905 | if (folio_test_dirty(folio)) { |
| 1906 | /* |
| 1907 | * Only kswapd can writeback filesystem folios |
| 1908 | * to avoid risk of stack overflow. But avoid |
| 1909 | * injecting inefficient single-folio I/O into |
| 1910 | * flusher writeback as much as possible: only |
| 1911 | * write folios when we've encountered many |
| 1912 | * dirty folios, and when we've already scanned |
| 1913 | * the rest of the LRU for clean folios and see |
| 1914 | * the same dirty folios again (with the reclaim |
| 1915 | * flag set). |
| 1916 | */ |
| 1917 | if (folio_is_file_lru(folio) && |
| 1918 | (!current_is_kswapd() || |
| 1919 | !folio_test_reclaim(folio) || |
| 1920 | !test_bit(PGDAT_DIRTY, &pgdat->flags))) { |
| 1921 | /* |
| 1922 | * Immediately reclaim when written back. |
| 1923 | * Similar in principle to deactivate_page() |
| 1924 | * except we already have the folio isolated |
| 1925 | * and know it's dirty |
| 1926 | */ |
| 1927 | node_stat_mod_folio(folio, NR_VMSCAN_IMMEDIATE, |
| 1928 | nr_pages); |
| 1929 | folio_set_reclaim(folio); |
| 1930 | |
| 1931 | goto activate_locked; |
| 1932 | } |
| 1933 | |
| 1934 | if (references == FOLIOREF_RECLAIM_CLEAN) |
| 1935 | goto keep_locked; |
| 1936 | if (!may_enter_fs(folio, sc->gfp_mask)) |
| 1937 | goto keep_locked; |
| 1938 | if (!sc->may_writepage) |
| 1939 | goto keep_locked; |
| 1940 | |
| 1941 | /* |
| 1942 | * Folio is dirty. Flush the TLB if a writable entry |
| 1943 | * potentially exists to avoid CPU writes after I/O |
| 1944 | * starts and then write it out here. |
| 1945 | */ |
| 1946 | try_to_unmap_flush_dirty(); |
| 1947 | switch (pageout(folio, mapping, &plug)) { |
| 1948 | case PAGE_KEEP: |
| 1949 | goto keep_locked; |
| 1950 | case PAGE_ACTIVATE: |
| 1951 | goto activate_locked; |
| 1952 | case PAGE_SUCCESS: |
| 1953 | stat->nr_pageout += nr_pages; |
| 1954 | |
| 1955 | if (folio_test_writeback(folio)) |
| 1956 | goto keep; |
| 1957 | if (folio_test_dirty(folio)) |
| 1958 | goto keep; |
| 1959 | |
| 1960 | /* |
| 1961 | * A synchronous write - probably a ramdisk. Go |
| 1962 | * ahead and try to reclaim the folio. |
| 1963 | */ |
| 1964 | if (!folio_trylock(folio)) |
| 1965 | goto keep; |
| 1966 | if (folio_test_dirty(folio) || |
| 1967 | folio_test_writeback(folio)) |
| 1968 | goto keep_locked; |
| 1969 | mapping = folio_mapping(folio); |
| 1970 | fallthrough; |
| 1971 | case PAGE_CLEAN: |
| 1972 | ; /* try to free the folio below */ |
| 1973 | } |
| 1974 | } |
| 1975 | |
| 1976 | /* |
| 1977 | * If the folio has buffers, try to free the buffer |
| 1978 | * mappings associated with this folio. If we succeed |
| 1979 | * we try to free the folio as well. |
| 1980 | * |
| 1981 | * We do this even if the folio is dirty. |
| 1982 | * filemap_release_folio() does not perform I/O, but it |
| 1983 | * is possible for a folio to have the dirty flag set, |
| 1984 | * but it is actually clean (all its buffers are clean). |
| 1985 | * This happens if the buffers were written out directly, |
| 1986 | * with submit_bh(). ext3 will do this, as well as |
| 1987 | * the blockdev mapping. filemap_release_folio() will |
| 1988 | * discover that cleanness and will drop the buffers |
| 1989 | * and mark the folio clean - it can be freed. |
| 1990 | * |
| 1991 | * Rarely, folios can have buffers and no ->mapping. |
| 1992 | * These are the folios which were not successfully |
| 1993 | * invalidated in truncate_cleanup_folio(). We try to |
| 1994 | * drop those buffers here and if that worked, and the |
| 1995 | * folio is no longer mapped into process address space |
| 1996 | * (refcount == 1) it can be freed. Otherwise, leave |
| 1997 | * the folio on the LRU so it is swappable. |
| 1998 | */ |
| 1999 | if (folio_has_private(folio)) { |
| 2000 | if (!filemap_release_folio(folio, sc->gfp_mask)) |
| 2001 | goto activate_locked; |
| 2002 | if (!mapping && folio_ref_count(folio) == 1) { |
| 2003 | folio_unlock(folio); |
| 2004 | if (folio_put_testzero(folio)) |
| 2005 | goto free_it; |
| 2006 | else { |
| 2007 | /* |
| 2008 | * rare race with speculative reference. |
| 2009 | * the speculative reference will free |
| 2010 | * this folio shortly, so we may |
| 2011 | * increment nr_reclaimed here (and |
| 2012 | * leave it off the LRU). |
| 2013 | */ |
| 2014 | nr_reclaimed += nr_pages; |
| 2015 | continue; |
| 2016 | } |
| 2017 | } |
| 2018 | } |
| 2019 | |
| 2020 | if (folio_test_anon(folio) && !folio_test_swapbacked(folio)) { |
| 2021 | /* follow __remove_mapping for reference */ |
| 2022 | if (!folio_ref_freeze(folio, 1)) |
| 2023 | goto keep_locked; |
| 2024 | /* |
| 2025 | * The folio has only one reference left, which is |
| 2026 | * from the isolation. After the caller puts the |
| 2027 | * folio back on the lru and drops the reference, the |
| 2028 | * folio will be freed anyway. It doesn't matter |
| 2029 | * which lru it goes on. So we don't bother checking |
| 2030 | * the dirty flag here. |
| 2031 | */ |
| 2032 | count_vm_events(PGLAZYFREED, nr_pages); |
| 2033 | count_memcg_folio_events(folio, PGLAZYFREED, nr_pages); |
| 2034 | } else if (!mapping || !__remove_mapping(mapping, folio, true, |
| 2035 | sc->target_mem_cgroup)) |
| 2036 | goto keep_locked; |
| 2037 | |
| 2038 | folio_unlock(folio); |
| 2039 | free_it: |
| 2040 | /* |
| 2041 | * Folio may get swapped out as a whole, need to account |
| 2042 | * all pages in it. |
| 2043 | */ |
| 2044 | nr_reclaimed += nr_pages; |
| 2045 | |
| 2046 | /* |
| 2047 | * Is there need to periodically free_folio_list? It would |
| 2048 | * appear not as the counts should be low |
| 2049 | */ |
| 2050 | if (unlikely(folio_test_large(folio))) |
| 2051 | destroy_large_folio(folio); |
| 2052 | else |
| 2053 | list_add(&folio->lru, &free_folios); |
| 2054 | continue; |
| 2055 | |
| 2056 | activate_locked_split: |
| 2057 | /* |
| 2058 | * The tail pages that are failed to add into swap cache |
| 2059 | * reach here. Fixup nr_scanned and nr_pages. |
| 2060 | */ |
| 2061 | if (nr_pages > 1) { |
| 2062 | sc->nr_scanned -= (nr_pages - 1); |
| 2063 | nr_pages = 1; |
| 2064 | } |
| 2065 | activate_locked: |
| 2066 | /* Not a candidate for swapping, so reclaim swap space. */ |
| 2067 | if (folio_test_swapcache(folio) && |
| 2068 | (mem_cgroup_swap_full(folio) || folio_test_mlocked(folio))) |
| 2069 | folio_free_swap(folio); |
| 2070 | VM_BUG_ON_FOLIO(folio_test_active(folio), folio); |
| 2071 | if (!folio_test_mlocked(folio)) { |
| 2072 | int type = folio_is_file_lru(folio); |
| 2073 | folio_set_active(folio); |
| 2074 | stat->nr_activate[type] += nr_pages; |
| 2075 | count_memcg_folio_events(folio, PGACTIVATE, nr_pages); |
| 2076 | } |
| 2077 | keep_locked: |
| 2078 | folio_unlock(folio); |
| 2079 | keep: |
| 2080 | list_add(&folio->lru, &ret_folios); |
| 2081 | VM_BUG_ON_FOLIO(folio_test_lru(folio) || |
| 2082 | folio_test_unevictable(folio), folio); |
| 2083 | } |
| 2084 | /* 'folio_list' is always empty here */ |
| 2085 | |
| 2086 | /* Migrate folios selected for demotion */ |
| 2087 | nr_reclaimed += demote_folio_list(&demote_folios, pgdat); |
| 2088 | /* Folios that could not be demoted are still in @demote_folios */ |
| 2089 | if (!list_empty(&demote_folios)) { |
| 2090 | /* Folios which weren't demoted go back on @folio_list */ |
| 2091 | list_splice_init(&demote_folios, folio_list); |
| 2092 | |
| 2093 | /* |
| 2094 | * goto retry to reclaim the undemoted folios in folio_list if |
| 2095 | * desired. |
| 2096 | * |
| 2097 | * Reclaiming directly from top tier nodes is not often desired |
| 2098 | * due to it breaking the LRU ordering: in general memory |
| 2099 | * should be reclaimed from lower tier nodes and demoted from |
| 2100 | * top tier nodes. |
| 2101 | * |
| 2102 | * However, disabling reclaim from top tier nodes entirely |
| 2103 | * would cause ooms in edge scenarios where lower tier memory |
| 2104 | * is unreclaimable for whatever reason, eg memory being |
| 2105 | * mlocked or too hot to reclaim. We can disable reclaim |
| 2106 | * from top tier nodes in proactive reclaim though as that is |
| 2107 | * not real memory pressure. |
| 2108 | */ |
| 2109 | if (!sc->proactive) { |
| 2110 | do_demote_pass = false; |
| 2111 | goto retry; |
| 2112 | } |
| 2113 | } |
| 2114 | |
| 2115 | pgactivate = stat->nr_activate[0] + stat->nr_activate[1]; |
| 2116 | |
| 2117 | mem_cgroup_uncharge_list(&free_folios); |
| 2118 | try_to_unmap_flush(); |
| 2119 | free_unref_page_list(&free_folios); |
| 2120 | |
| 2121 | list_splice(&ret_folios, folio_list); |
| 2122 | count_vm_events(PGACTIVATE, pgactivate); |
| 2123 | |
| 2124 | if (plug) |
| 2125 | swap_write_unplug(plug); |
| 2126 | return nr_reclaimed; |
| 2127 | } |
| 2128 | |
| 2129 | unsigned int reclaim_clean_pages_from_list(struct zone *zone, |
| 2130 | struct list_head *folio_list) |
| 2131 | { |
| 2132 | struct scan_control sc = { |
| 2133 | .gfp_mask = GFP_KERNEL, |
| 2134 | .may_unmap = 1, |
| 2135 | }; |
| 2136 | struct reclaim_stat stat; |
| 2137 | unsigned int nr_reclaimed; |
| 2138 | struct folio *folio, *next; |
| 2139 | LIST_HEAD(clean_folios); |
| 2140 | unsigned int noreclaim_flag; |
| 2141 | |
| 2142 | list_for_each_entry_safe(folio, next, folio_list, lru) { |
| 2143 | if (!folio_test_hugetlb(folio) && folio_is_file_lru(folio) && |
| 2144 | !folio_test_dirty(folio) && !__folio_test_movable(folio) && |
| 2145 | !folio_test_unevictable(folio)) { |
| 2146 | folio_clear_active(folio); |
| 2147 | list_move(&folio->lru, &clean_folios); |
| 2148 | } |
| 2149 | } |
| 2150 | |
| 2151 | /* |
| 2152 | * We should be safe here since we are only dealing with file pages and |
| 2153 | * we are not kswapd and therefore cannot write dirty file pages. But |
| 2154 | * call memalloc_noreclaim_save() anyway, just in case these conditions |
| 2155 | * change in the future. |
| 2156 | */ |
| 2157 | noreclaim_flag = memalloc_noreclaim_save(); |
| 2158 | nr_reclaimed = shrink_folio_list(&clean_folios, zone->zone_pgdat, &sc, |
| 2159 | &stat, true); |
| 2160 | memalloc_noreclaim_restore(noreclaim_flag); |
| 2161 | |
| 2162 | list_splice(&clean_folios, folio_list); |
| 2163 | mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, |
| 2164 | -(long)nr_reclaimed); |
| 2165 | /* |
| 2166 | * Since lazyfree pages are isolated from file LRU from the beginning, |
| 2167 | * they will rotate back to anonymous LRU in the end if it failed to |
| 2168 | * discard so isolated count will be mismatched. |
| 2169 | * Compensate the isolated count for both LRU lists. |
| 2170 | */ |
| 2171 | mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON, |
| 2172 | stat.nr_lazyfree_fail); |
| 2173 | mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, |
| 2174 | -(long)stat.nr_lazyfree_fail); |
| 2175 | return nr_reclaimed; |
| 2176 | } |
| 2177 | |
| 2178 | /* |
| 2179 | * Update LRU sizes after isolating pages. The LRU size updates must |
| 2180 | * be complete before mem_cgroup_update_lru_size due to a sanity check. |
| 2181 | */ |
| 2182 | static __always_inline void update_lru_sizes(struct lruvec *lruvec, |
| 2183 | enum lru_list lru, unsigned long *nr_zone_taken) |
| 2184 | { |
| 2185 | int zid; |
| 2186 | |
| 2187 | for (zid = 0; zid < MAX_NR_ZONES; zid++) { |
| 2188 | if (!nr_zone_taken[zid]) |
| 2189 | continue; |
| 2190 | |
| 2191 | update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]); |
| 2192 | } |
| 2193 | |
| 2194 | } |
| 2195 | |
| 2196 | /* |
| 2197 | * Isolating page from the lruvec to fill in @dst list by nr_to_scan times. |
| 2198 | * |
| 2199 | * lruvec->lru_lock is heavily contended. Some of the functions that |
| 2200 | * shrink the lists perform better by taking out a batch of pages |
| 2201 | * and working on them outside the LRU lock. |
| 2202 | * |
| 2203 | * For pagecache intensive workloads, this function is the hottest |
| 2204 | * spot in the kernel (apart from copy_*_user functions). |
| 2205 | * |
| 2206 | * Lru_lock must be held before calling this function. |
| 2207 | * |
| 2208 | * @nr_to_scan: The number of eligible pages to look through on the list. |
| 2209 | * @lruvec: The LRU vector to pull pages from. |
| 2210 | * @dst: The temp list to put pages on to. |
| 2211 | * @nr_scanned: The number of pages that were scanned. |
| 2212 | * @sc: The scan_control struct for this reclaim session |
| 2213 | * @lru: LRU list id for isolating |
| 2214 | * |
| 2215 | * returns how many pages were moved onto *@dst. |
| 2216 | */ |
| 2217 | static unsigned long isolate_lru_folios(unsigned long nr_to_scan, |
| 2218 | struct lruvec *lruvec, struct list_head *dst, |
| 2219 | unsigned long *nr_scanned, struct scan_control *sc, |
| 2220 | enum lru_list lru) |
| 2221 | { |
| 2222 | struct list_head *src = &lruvec->lists[lru]; |
| 2223 | unsigned long nr_taken = 0; |
| 2224 | unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 }; |
| 2225 | unsigned long nr_skipped[MAX_NR_ZONES] = { 0, }; |
| 2226 | unsigned long skipped = 0; |
| 2227 | unsigned long scan, total_scan, nr_pages; |
| 2228 | LIST_HEAD(folios_skipped); |
| 2229 | |
| 2230 | total_scan = 0; |
| 2231 | scan = 0; |
| 2232 | while (scan < nr_to_scan && !list_empty(src)) { |
| 2233 | struct list_head *move_to = src; |
| 2234 | struct folio *folio; |
| 2235 | |
| 2236 | folio = lru_to_folio(src); |
| 2237 | prefetchw_prev_lru_folio(folio, src, flags); |
| 2238 | |
| 2239 | nr_pages = folio_nr_pages(folio); |
| 2240 | total_scan += nr_pages; |
| 2241 | |
| 2242 | if (folio_zonenum(folio) > sc->reclaim_idx) { |
| 2243 | nr_skipped[folio_zonenum(folio)] += nr_pages; |
| 2244 | move_to = &folios_skipped; |
| 2245 | goto move; |
| 2246 | } |
| 2247 | |
| 2248 | /* |
| 2249 | * Do not count skipped folios because that makes the function |
| 2250 | * return with no isolated folios if the LRU mostly contains |
| 2251 | * ineligible folios. This causes the VM to not reclaim any |
| 2252 | * folios, triggering a premature OOM. |
| 2253 | * Account all pages in a folio. |
| 2254 | */ |
| 2255 | scan += nr_pages; |
| 2256 | |
| 2257 | if (!folio_test_lru(folio)) |
| 2258 | goto move; |
| 2259 | if (!sc->may_unmap && folio_mapped(folio)) |
| 2260 | goto move; |
| 2261 | |
| 2262 | /* |
| 2263 | * Be careful not to clear the lru flag until after we're |
| 2264 | * sure the folio is not being freed elsewhere -- the |
| 2265 | * folio release code relies on it. |
| 2266 | */ |
| 2267 | if (unlikely(!folio_try_get(folio))) |
| 2268 | goto move; |
| 2269 | |
| 2270 | if (!folio_test_clear_lru(folio)) { |
| 2271 | /* Another thread is already isolating this folio */ |
| 2272 | folio_put(folio); |
| 2273 | goto move; |
| 2274 | } |
| 2275 | |
| 2276 | nr_taken += nr_pages; |
| 2277 | nr_zone_taken[folio_zonenum(folio)] += nr_pages; |
| 2278 | move_to = dst; |
| 2279 | move: |
| 2280 | list_move(&folio->lru, move_to); |
| 2281 | } |
| 2282 | |
| 2283 | /* |
| 2284 | * Splice any skipped folios to the start of the LRU list. Note that |
| 2285 | * this disrupts the LRU order when reclaiming for lower zones but |
| 2286 | * we cannot splice to the tail. If we did then the SWAP_CLUSTER_MAX |
| 2287 | * scanning would soon rescan the same folios to skip and waste lots |
| 2288 | * of cpu cycles. |
| 2289 | */ |
| 2290 | if (!list_empty(&folios_skipped)) { |
| 2291 | int zid; |
| 2292 | |
| 2293 | list_splice(&folios_skipped, src); |
| 2294 | for (zid = 0; zid < MAX_NR_ZONES; zid++) { |
| 2295 | if (!nr_skipped[zid]) |
| 2296 | continue; |
| 2297 | |
| 2298 | __count_zid_vm_events(PGSCAN_SKIP, zid, nr_skipped[zid]); |
| 2299 | skipped += nr_skipped[zid]; |
| 2300 | } |
| 2301 | } |
| 2302 | *nr_scanned = total_scan; |
| 2303 | trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan, |
| 2304 | total_scan, skipped, nr_taken, |
| 2305 | sc->may_unmap ? 0 : ISOLATE_UNMAPPED, lru); |
| 2306 | update_lru_sizes(lruvec, lru, nr_zone_taken); |
| 2307 | return nr_taken; |
| 2308 | } |
| 2309 | |
| 2310 | /** |
| 2311 | * folio_isolate_lru() - Try to isolate a folio from its LRU list. |
| 2312 | * @folio: Folio to isolate from its LRU list. |
| 2313 | * |
| 2314 | * Isolate a @folio from an LRU list and adjust the vmstat statistic |
| 2315 | * corresponding to whatever LRU list the folio was on. |
| 2316 | * |
| 2317 | * The folio will have its LRU flag cleared. If it was found on the |
| 2318 | * active list, it will have the Active flag set. If it was found on the |
| 2319 | * unevictable list, it will have the Unevictable flag set. These flags |
| 2320 | * may need to be cleared by the caller before letting the page go. |
| 2321 | * |
| 2322 | * Context: |
| 2323 | * |
| 2324 | * (1) Must be called with an elevated refcount on the folio. This is a |
| 2325 | * fundamental difference from isolate_lru_folios() (which is called |
| 2326 | * without a stable reference). |
| 2327 | * (2) The lru_lock must not be held. |
| 2328 | * (3) Interrupts must be enabled. |
| 2329 | * |
| 2330 | * Return: 0 if the folio was removed from an LRU list. |
| 2331 | * -EBUSY if the folio was not on an LRU list. |
| 2332 | */ |
| 2333 | int folio_isolate_lru(struct folio *folio) |
| 2334 | { |
| 2335 | int ret = -EBUSY; |
| 2336 | |
| 2337 | VM_BUG_ON_FOLIO(!folio_ref_count(folio), folio); |
| 2338 | |
| 2339 | if (folio_test_clear_lru(folio)) { |
| 2340 | struct lruvec *lruvec; |
| 2341 | |
| 2342 | folio_get(folio); |
| 2343 | lruvec = folio_lruvec_lock_irq(folio); |
| 2344 | lruvec_del_folio(lruvec, folio); |
| 2345 | unlock_page_lruvec_irq(lruvec); |
| 2346 | ret = 0; |
| 2347 | } |
| 2348 | |
| 2349 | return ret; |
| 2350 | } |
| 2351 | |
| 2352 | /* |
| 2353 | * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and |
| 2354 | * then get rescheduled. When there are massive number of tasks doing page |
| 2355 | * allocation, such sleeping direct reclaimers may keep piling up on each CPU, |
| 2356 | * the LRU list will go small and be scanned faster than necessary, leading to |
| 2357 | * unnecessary swapping, thrashing and OOM. |
| 2358 | */ |
| 2359 | static int too_many_isolated(struct pglist_data *pgdat, int file, |
| 2360 | struct scan_control *sc) |
| 2361 | { |
| 2362 | unsigned long inactive, isolated; |
| 2363 | bool too_many; |
| 2364 | |
| 2365 | if (current_is_kswapd()) |
| 2366 | return 0; |
| 2367 | |
| 2368 | if (!writeback_throttling_sane(sc)) |
| 2369 | return 0; |
| 2370 | |
| 2371 | if (file) { |
| 2372 | inactive = node_page_state(pgdat, NR_INACTIVE_FILE); |
| 2373 | isolated = node_page_state(pgdat, NR_ISOLATED_FILE); |
| 2374 | } else { |
| 2375 | inactive = node_page_state(pgdat, NR_INACTIVE_ANON); |
| 2376 | isolated = node_page_state(pgdat, NR_ISOLATED_ANON); |
| 2377 | } |
| 2378 | |
| 2379 | /* |
| 2380 | * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they |
| 2381 | * won't get blocked by normal direct-reclaimers, forming a circular |
| 2382 | * deadlock. |
| 2383 | */ |
| 2384 | if ((sc->gfp_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) |
| 2385 | inactive >>= 3; |
| 2386 | |
| 2387 | too_many = isolated > inactive; |
| 2388 | |
| 2389 | /* Wake up tasks throttled due to too_many_isolated. */ |
| 2390 | if (!too_many) |
| 2391 | wake_throttle_isolated(pgdat); |
| 2392 | |
| 2393 | return too_many; |
| 2394 | } |
| 2395 | |
| 2396 | /* |
| 2397 | * move_folios_to_lru() moves folios from private @list to appropriate LRU list. |
| 2398 | * On return, @list is reused as a list of folios to be freed by the caller. |
| 2399 | * |
| 2400 | * Returns the number of pages moved to the given lruvec. |
| 2401 | */ |
| 2402 | static unsigned int move_folios_to_lru(struct lruvec *lruvec, |
| 2403 | struct list_head *list) |
| 2404 | { |
| 2405 | int nr_pages, nr_moved = 0; |
| 2406 | LIST_HEAD(folios_to_free); |
| 2407 | |
| 2408 | while (!list_empty(list)) { |
| 2409 | struct folio *folio = lru_to_folio(list); |
| 2410 | |
| 2411 | VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); |
| 2412 | list_del(&folio->lru); |
| 2413 | if (unlikely(!folio_evictable(folio))) { |
| 2414 | spin_unlock_irq(&lruvec->lru_lock); |
| 2415 | folio_putback_lru(folio); |
| 2416 | spin_lock_irq(&lruvec->lru_lock); |
| 2417 | continue; |
| 2418 | } |
| 2419 | |
| 2420 | /* |
| 2421 | * The folio_set_lru needs to be kept here for list integrity. |
| 2422 | * Otherwise: |
| 2423 | * #0 move_folios_to_lru #1 release_pages |
| 2424 | * if (!folio_put_testzero()) |
| 2425 | * if (folio_put_testzero()) |
| 2426 | * !lru //skip lru_lock |
| 2427 | * folio_set_lru() |
| 2428 | * list_add(&folio->lru,) |
| 2429 | * list_add(&folio->lru,) |
| 2430 | */ |
| 2431 | folio_set_lru(folio); |
| 2432 | |
| 2433 | if (unlikely(folio_put_testzero(folio))) { |
| 2434 | __folio_clear_lru_flags(folio); |
| 2435 | |
| 2436 | if (unlikely(folio_test_large(folio))) { |
| 2437 | spin_unlock_irq(&lruvec->lru_lock); |
| 2438 | destroy_large_folio(folio); |
| 2439 | spin_lock_irq(&lruvec->lru_lock); |
| 2440 | } else |
| 2441 | list_add(&folio->lru, &folios_to_free); |
| 2442 | |
| 2443 | continue; |
| 2444 | } |
| 2445 | |
| 2446 | /* |
| 2447 | * All pages were isolated from the same lruvec (and isolation |
| 2448 | * inhibits memcg migration). |
| 2449 | */ |
| 2450 | VM_BUG_ON_FOLIO(!folio_matches_lruvec(folio, lruvec), folio); |
| 2451 | lruvec_add_folio(lruvec, folio); |
| 2452 | nr_pages = folio_nr_pages(folio); |
| 2453 | nr_moved += nr_pages; |
| 2454 | if (folio_test_active(folio)) |
| 2455 | workingset_age_nonresident(lruvec, nr_pages); |
| 2456 | } |
| 2457 | |
| 2458 | /* |
| 2459 | * To save our caller's stack, now use input list for pages to free. |
| 2460 | */ |
| 2461 | list_splice(&folios_to_free, list); |
| 2462 | |
| 2463 | return nr_moved; |
| 2464 | } |
| 2465 | |
| 2466 | /* |
| 2467 | * If a kernel thread (such as nfsd for loop-back mounts) services a backing |
| 2468 | * device by writing to the page cache it sets PF_LOCAL_THROTTLE. In this case |
| 2469 | * we should not throttle. Otherwise it is safe to do so. |
| 2470 | */ |
| 2471 | static int current_may_throttle(void) |
| 2472 | { |
| 2473 | return !(current->flags & PF_LOCAL_THROTTLE); |
| 2474 | } |
| 2475 | |
| 2476 | /* |
| 2477 | * shrink_inactive_list() is a helper for shrink_node(). It returns the number |
| 2478 | * of reclaimed pages |
| 2479 | */ |
| 2480 | static unsigned long shrink_inactive_list(unsigned long nr_to_scan, |
| 2481 | struct lruvec *lruvec, struct scan_control *sc, |
| 2482 | enum lru_list lru) |
| 2483 | { |
| 2484 | LIST_HEAD(folio_list); |
| 2485 | unsigned long nr_scanned; |
| 2486 | unsigned int nr_reclaimed = 0; |
| 2487 | unsigned long nr_taken; |
| 2488 | struct reclaim_stat stat; |
| 2489 | bool file = is_file_lru(lru); |
| 2490 | enum vm_event_item item; |
| 2491 | struct pglist_data *pgdat = lruvec_pgdat(lruvec); |
| 2492 | bool stalled = false; |
| 2493 | |
| 2494 | while (unlikely(too_many_isolated(pgdat, file, sc))) { |
| 2495 | if (stalled) |
| 2496 | return 0; |
| 2497 | |
| 2498 | /* wait a bit for the reclaimer. */ |
| 2499 | stalled = true; |
| 2500 | reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED); |
| 2501 | |
| 2502 | /* We are about to die and free our memory. Return now. */ |
| 2503 | if (fatal_signal_pending(current)) |
| 2504 | return SWAP_CLUSTER_MAX; |
| 2505 | } |
| 2506 | |
| 2507 | lru_add_drain(); |
| 2508 | |
| 2509 | spin_lock_irq(&lruvec->lru_lock); |
| 2510 | |
| 2511 | nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &folio_list, |
| 2512 | &nr_scanned, sc, lru); |
| 2513 | |
| 2514 | __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); |
| 2515 | item = PGSCAN_KSWAPD + reclaimer_offset(); |
| 2516 | if (!cgroup_reclaim(sc)) |
| 2517 | __count_vm_events(item, nr_scanned); |
| 2518 | __count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned); |
| 2519 | __count_vm_events(PGSCAN_ANON + file, nr_scanned); |
| 2520 | |
| 2521 | spin_unlock_irq(&lruvec->lru_lock); |
| 2522 | |
| 2523 | if (nr_taken == 0) |
| 2524 | return 0; |
| 2525 | |
| 2526 | nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false); |
| 2527 | |
| 2528 | spin_lock_irq(&lruvec->lru_lock); |
| 2529 | move_folios_to_lru(lruvec, &folio_list); |
| 2530 | |
| 2531 | __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); |
| 2532 | item = PGSTEAL_KSWAPD + reclaimer_offset(); |
| 2533 | if (!cgroup_reclaim(sc)) |
| 2534 | __count_vm_events(item, nr_reclaimed); |
| 2535 | __count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed); |
| 2536 | __count_vm_events(PGSTEAL_ANON + file, nr_reclaimed); |
| 2537 | spin_unlock_irq(&lruvec->lru_lock); |
| 2538 | |
| 2539 | lru_note_cost(lruvec, file, stat.nr_pageout, nr_scanned - nr_reclaimed); |
| 2540 | mem_cgroup_uncharge_list(&folio_list); |
| 2541 | free_unref_page_list(&folio_list); |
| 2542 | |
| 2543 | /* |
| 2544 | * If dirty folios are scanned that are not queued for IO, it |
| 2545 | * implies that flushers are not doing their job. This can |
| 2546 | * happen when memory pressure pushes dirty folios to the end of |
| 2547 | * the LRU before the dirty limits are breached and the dirty |
| 2548 | * data has expired. It can also happen when the proportion of |
| 2549 | * dirty folios grows not through writes but through memory |
| 2550 | * pressure reclaiming all the clean cache. And in some cases, |
| 2551 | * the flushers simply cannot keep up with the allocation |
| 2552 | * rate. Nudge the flusher threads in case they are asleep. |
| 2553 | */ |
| 2554 | if (stat.nr_unqueued_dirty == nr_taken) { |
| 2555 | wakeup_flusher_threads(WB_REASON_VMSCAN); |
| 2556 | /* |
| 2557 | * For cgroupv1 dirty throttling is achieved by waking up |
| 2558 | * the kernel flusher here and later waiting on folios |
| 2559 | * which are in writeback to finish (see shrink_folio_list()). |
| 2560 | * |
| 2561 | * Flusher may not be able to issue writeback quickly |
| 2562 | * enough for cgroupv1 writeback throttling to work |
| 2563 | * on a large system. |
| 2564 | */ |
| 2565 | if (!writeback_throttling_sane(sc)) |
| 2566 | reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK); |
| 2567 | } |
| 2568 | |
| 2569 | sc->nr.dirty += stat.nr_dirty; |
| 2570 | sc->nr.congested += stat.nr_congested; |
| 2571 | sc->nr.unqueued_dirty += stat.nr_unqueued_dirty; |
| 2572 | sc->nr.writeback += stat.nr_writeback; |
| 2573 | sc->nr.immediate += stat.nr_immediate; |
| 2574 | sc->nr.taken += nr_taken; |
| 2575 | if (file) |
| 2576 | sc->nr.file_taken += nr_taken; |
| 2577 | |
| 2578 | trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id, |
| 2579 | nr_scanned, nr_reclaimed, &stat, sc->priority, file); |
| 2580 | return nr_reclaimed; |
| 2581 | } |
| 2582 | |
| 2583 | /* |
| 2584 | * shrink_active_list() moves folios from the active LRU to the inactive LRU. |
| 2585 | * |
| 2586 | * We move them the other way if the folio is referenced by one or more |
| 2587 | * processes. |
| 2588 | * |
| 2589 | * If the folios are mostly unmapped, the processing is fast and it is |
| 2590 | * appropriate to hold lru_lock across the whole operation. But if |
| 2591 | * the folios are mapped, the processing is slow (folio_referenced()), so |
| 2592 | * we should drop lru_lock around each folio. It's impossible to balance |
| 2593 | * this, so instead we remove the folios from the LRU while processing them. |
| 2594 | * It is safe to rely on the active flag against the non-LRU folios in here |
| 2595 | * because nobody will play with that bit on a non-LRU folio. |
| 2596 | * |
| 2597 | * The downside is that we have to touch folio->_refcount against each folio. |
| 2598 | * But we had to alter folio->flags anyway. |
| 2599 | */ |
| 2600 | static void shrink_active_list(unsigned long nr_to_scan, |
| 2601 | struct lruvec *lruvec, |
| 2602 | struct scan_control *sc, |
| 2603 | enum lru_list lru) |
| 2604 | { |
| 2605 | unsigned long nr_taken; |
| 2606 | unsigned long nr_scanned; |
| 2607 | unsigned long vm_flags; |
| 2608 | LIST_HEAD(l_hold); /* The folios which were snipped off */ |
| 2609 | LIST_HEAD(l_active); |
| 2610 | LIST_HEAD(l_inactive); |
| 2611 | unsigned nr_deactivate, nr_activate; |
| 2612 | unsigned nr_rotated = 0; |
| 2613 | int file = is_file_lru(lru); |
| 2614 | struct pglist_data *pgdat = lruvec_pgdat(lruvec); |
| 2615 | |
| 2616 | lru_add_drain(); |
| 2617 | |
| 2618 | spin_lock_irq(&lruvec->lru_lock); |
| 2619 | |
| 2620 | nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &l_hold, |
| 2621 | &nr_scanned, sc, lru); |
| 2622 | |
| 2623 | __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); |
| 2624 | |
| 2625 | if (!cgroup_reclaim(sc)) |
| 2626 | __count_vm_events(PGREFILL, nr_scanned); |
| 2627 | __count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned); |
| 2628 | |
| 2629 | spin_unlock_irq(&lruvec->lru_lock); |
| 2630 | |
| 2631 | while (!list_empty(&l_hold)) { |
| 2632 | struct folio *folio; |
| 2633 | |
| 2634 | cond_resched(); |
| 2635 | folio = lru_to_folio(&l_hold); |
| 2636 | list_del(&folio->lru); |
| 2637 | |
| 2638 | if (unlikely(!folio_evictable(folio))) { |
| 2639 | folio_putback_lru(folio); |
| 2640 | continue; |
| 2641 | } |
| 2642 | |
| 2643 | if (unlikely(buffer_heads_over_limit)) { |
| 2644 | if (folio_test_private(folio) && folio_trylock(folio)) { |
| 2645 | if (folio_test_private(folio)) |
| 2646 | filemap_release_folio(folio, 0); |
| 2647 | folio_unlock(folio); |
| 2648 | } |
| 2649 | } |
| 2650 | |
| 2651 | /* Referenced or rmap lock contention: rotate */ |
| 2652 | if (folio_referenced(folio, 0, sc->target_mem_cgroup, |
| 2653 | &vm_flags) != 0) { |
| 2654 | /* |
| 2655 | * Identify referenced, file-backed active folios and |
| 2656 | * give them one more trip around the active list. So |
| 2657 | * that executable code get better chances to stay in |
| 2658 | * memory under moderate memory pressure. Anon folios |
| 2659 | * are not likely to be evicted by use-once streaming |
| 2660 | * IO, plus JVM can create lots of anon VM_EXEC folios, |
| 2661 | * so we ignore them here. |
| 2662 | */ |
| 2663 | if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) { |
| 2664 | nr_rotated += folio_nr_pages(folio); |
| 2665 | list_add(&folio->lru, &l_active); |
| 2666 | continue; |
| 2667 | } |
| 2668 | } |
| 2669 | |
| 2670 | folio_clear_active(folio); /* we are de-activating */ |
| 2671 | folio_set_workingset(folio); |
| 2672 | list_add(&folio->lru, &l_inactive); |
| 2673 | } |
| 2674 | |
| 2675 | /* |
| 2676 | * Move folios back to the lru list. |
| 2677 | */ |
| 2678 | spin_lock_irq(&lruvec->lru_lock); |
| 2679 | |
| 2680 | nr_activate = move_folios_to_lru(lruvec, &l_active); |
| 2681 | nr_deactivate = move_folios_to_lru(lruvec, &l_inactive); |
| 2682 | /* Keep all free folios in l_active list */ |
| 2683 | list_splice(&l_inactive, &l_active); |
| 2684 | |
| 2685 | __count_vm_events(PGDEACTIVATE, nr_deactivate); |
| 2686 | __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate); |
| 2687 | |
| 2688 | __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); |
| 2689 | spin_unlock_irq(&lruvec->lru_lock); |
| 2690 | |
| 2691 | if (nr_rotated) |
| 2692 | lru_note_cost(lruvec, file, 0, nr_rotated); |
| 2693 | mem_cgroup_uncharge_list(&l_active); |
| 2694 | free_unref_page_list(&l_active); |
| 2695 | trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate, |
| 2696 | nr_deactivate, nr_rotated, sc->priority, file); |
| 2697 | } |
| 2698 | |
| 2699 | static unsigned int reclaim_folio_list(struct list_head *folio_list, |
| 2700 | struct pglist_data *pgdat) |
| 2701 | { |
| 2702 | struct reclaim_stat dummy_stat; |
| 2703 | unsigned int nr_reclaimed; |
| 2704 | struct folio *folio; |
| 2705 | struct scan_control sc = { |
| 2706 | .gfp_mask = GFP_KERNEL, |
| 2707 | .may_writepage = 1, |
| 2708 | .may_unmap = 1, |
| 2709 | .may_swap = 1, |
| 2710 | .no_demotion = 1, |
| 2711 | }; |
| 2712 | |
| 2713 | nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &dummy_stat, false); |
| 2714 | while (!list_empty(folio_list)) { |
| 2715 | folio = lru_to_folio(folio_list); |
| 2716 | list_del(&folio->lru); |
| 2717 | folio_putback_lru(folio); |
| 2718 | } |
| 2719 | |
| 2720 | return nr_reclaimed; |
| 2721 | } |
| 2722 | |
| 2723 | unsigned long reclaim_pages(struct list_head *folio_list) |
| 2724 | { |
| 2725 | int nid; |
| 2726 | unsigned int nr_reclaimed = 0; |
| 2727 | LIST_HEAD(node_folio_list); |
| 2728 | unsigned int noreclaim_flag; |
| 2729 | |
| 2730 | if (list_empty(folio_list)) |
| 2731 | return nr_reclaimed; |
| 2732 | |
| 2733 | noreclaim_flag = memalloc_noreclaim_save(); |
| 2734 | |
| 2735 | nid = folio_nid(lru_to_folio(folio_list)); |
| 2736 | do { |
| 2737 | struct folio *folio = lru_to_folio(folio_list); |
| 2738 | |
| 2739 | if (nid == folio_nid(folio)) { |
| 2740 | folio_clear_active(folio); |
| 2741 | list_move(&folio->lru, &node_folio_list); |
| 2742 | continue; |
| 2743 | } |
| 2744 | |
| 2745 | nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); |
| 2746 | nid = folio_nid(lru_to_folio(folio_list)); |
| 2747 | } while (!list_empty(folio_list)); |
| 2748 | |
| 2749 | nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); |
| 2750 | |
| 2751 | memalloc_noreclaim_restore(noreclaim_flag); |
| 2752 | |
| 2753 | return nr_reclaimed; |
| 2754 | } |
| 2755 | |
| 2756 | static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan, |
| 2757 | struct lruvec *lruvec, struct scan_control *sc) |
| 2758 | { |
| 2759 | if (is_active_lru(lru)) { |
| 2760 | if (sc->may_deactivate & (1 << is_file_lru(lru))) |
| 2761 | shrink_active_list(nr_to_scan, lruvec, sc, lru); |
| 2762 | else |
| 2763 | sc->skipped_deactivate = 1; |
| 2764 | return 0; |
| 2765 | } |
| 2766 | |
| 2767 | return shrink_inactive_list(nr_to_scan, lruvec, sc, lru); |
| 2768 | } |
| 2769 | |
| 2770 | /* |
| 2771 | * The inactive anon list should be small enough that the VM never has |
| 2772 | * to do too much work. |
| 2773 | * |
| 2774 | * The inactive file list should be small enough to leave most memory |
| 2775 | * to the established workingset on the scan-resistant active list, |
| 2776 | * but large enough to avoid thrashing the aggregate readahead window. |
| 2777 | * |
| 2778 | * Both inactive lists should also be large enough that each inactive |
| 2779 | * folio has a chance to be referenced again before it is reclaimed. |
| 2780 | * |
| 2781 | * If that fails and refaulting is observed, the inactive list grows. |
| 2782 | * |
| 2783 | * The inactive_ratio is the target ratio of ACTIVE to INACTIVE folios |
| 2784 | * on this LRU, maintained by the pageout code. An inactive_ratio |
| 2785 | * of 3 means 3:1 or 25% of the folios are kept on the inactive list. |
| 2786 | * |
| 2787 | * total target max |
| 2788 | * memory ratio inactive |
| 2789 | * ------------------------------------- |
| 2790 | * 10MB 1 5MB |
| 2791 | * 100MB 1 50MB |
| 2792 | * 1GB 3 250MB |
| 2793 | * 10GB 10 0.9GB |
| 2794 | * 100GB 31 3GB |
| 2795 | * 1TB 101 10GB |
| 2796 | * 10TB 320 32GB |
| 2797 | */ |
| 2798 | static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru) |
| 2799 | { |
| 2800 | enum lru_list active_lru = inactive_lru + LRU_ACTIVE; |
| 2801 | unsigned long inactive, active; |
| 2802 | unsigned long inactive_ratio; |
| 2803 | unsigned long gb; |
| 2804 | |
| 2805 | inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru); |
| 2806 | active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru); |
| 2807 | |
| 2808 | gb = (inactive + active) >> (30 - PAGE_SHIFT); |
| 2809 | if (gb) |
| 2810 | inactive_ratio = int_sqrt(10 * gb); |
| 2811 | else |
| 2812 | inactive_ratio = 1; |
| 2813 | |
| 2814 | return inactive * inactive_ratio < active; |
| 2815 | } |
| 2816 | |
| 2817 | enum scan_balance { |
| 2818 | SCAN_EQUAL, |
| 2819 | SCAN_FRACT, |
| 2820 | SCAN_ANON, |
| 2821 | SCAN_FILE, |
| 2822 | }; |
| 2823 | |
| 2824 | static void prepare_scan_count(pg_data_t *pgdat, struct scan_control *sc) |
| 2825 | { |
| 2826 | unsigned long file; |
| 2827 | struct lruvec *target_lruvec; |
| 2828 | |
| 2829 | if (lru_gen_enabled()) |
| 2830 | return; |
| 2831 | |
| 2832 | target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); |
| 2833 | |
| 2834 | /* |
| 2835 | * Flush the memory cgroup stats, so that we read accurate per-memcg |
| 2836 | * lruvec stats for heuristics. |
| 2837 | */ |
| 2838 | mem_cgroup_flush_stats(); |
| 2839 | |
| 2840 | /* |
| 2841 | * Determine the scan balance between anon and file LRUs. |
| 2842 | */ |
| 2843 | spin_lock_irq(&target_lruvec->lru_lock); |
| 2844 | sc->anon_cost = target_lruvec->anon_cost; |
| 2845 | sc->file_cost = target_lruvec->file_cost; |
| 2846 | spin_unlock_irq(&target_lruvec->lru_lock); |
| 2847 | |
| 2848 | /* |
| 2849 | * Target desirable inactive:active list ratios for the anon |
| 2850 | * and file LRU lists. |
| 2851 | */ |
| 2852 | if (!sc->force_deactivate) { |
| 2853 | unsigned long refaults; |
| 2854 | |
| 2855 | /* |
| 2856 | * When refaults are being observed, it means a new |
| 2857 | * workingset is being established. Deactivate to get |
| 2858 | * rid of any stale active pages quickly. |
| 2859 | */ |
| 2860 | refaults = lruvec_page_state(target_lruvec, |
| 2861 | WORKINGSET_ACTIVATE_ANON); |
| 2862 | if (refaults != target_lruvec->refaults[WORKINGSET_ANON] || |
| 2863 | inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) |
| 2864 | sc->may_deactivate |= DEACTIVATE_ANON; |
| 2865 | else |
| 2866 | sc->may_deactivate &= ~DEACTIVATE_ANON; |
| 2867 | |
| 2868 | refaults = lruvec_page_state(target_lruvec, |
| 2869 | WORKINGSET_ACTIVATE_FILE); |
| 2870 | if (refaults != target_lruvec->refaults[WORKINGSET_FILE] || |
| 2871 | inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) |
| 2872 | sc->may_deactivate |= DEACTIVATE_FILE; |
| 2873 | else |
| 2874 | sc->may_deactivate &= ~DEACTIVATE_FILE; |
| 2875 | } else |
| 2876 | sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; |
| 2877 | |
| 2878 | /* |
| 2879 | * If we have plenty of inactive file pages that aren't |
| 2880 | * thrashing, try to reclaim those first before touching |
| 2881 | * anonymous pages. |
| 2882 | */ |
| 2883 | file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); |
| 2884 | if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE)) |
| 2885 | sc->cache_trim_mode = 1; |
| 2886 | else |
| 2887 | sc->cache_trim_mode = 0; |
| 2888 | |
| 2889 | /* |
| 2890 | * Prevent the reclaimer from falling into the cache trap: as |
| 2891 | * cache pages start out inactive, every cache fault will tip |
| 2892 | * the scan balance towards the file LRU. And as the file LRU |
| 2893 | * shrinks, so does the window for rotation from references. |
| 2894 | * This means we have a runaway feedback loop where a tiny |
| 2895 | * thrashing file LRU becomes infinitely more attractive than |
| 2896 | * anon pages. Try to detect this based on file LRU size. |
| 2897 | */ |
| 2898 | if (!cgroup_reclaim(sc)) { |
| 2899 | unsigned long total_high_wmark = 0; |
| 2900 | unsigned long free, anon; |
| 2901 | int z; |
| 2902 | |
| 2903 | free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); |
| 2904 | file = node_page_state(pgdat, NR_ACTIVE_FILE) + |
| 2905 | node_page_state(pgdat, NR_INACTIVE_FILE); |
| 2906 | |
| 2907 | for (z = 0; z < MAX_NR_ZONES; z++) { |
| 2908 | struct zone *zone = &pgdat->node_zones[z]; |
| 2909 | |
| 2910 | if (!managed_zone(zone)) |
| 2911 | continue; |
| 2912 | |
| 2913 | total_high_wmark += high_wmark_pages(zone); |
| 2914 | } |
| 2915 | |
| 2916 | /* |
| 2917 | * Consider anon: if that's low too, this isn't a |
| 2918 | * runaway file reclaim problem, but rather just |
| 2919 | * extreme pressure. Reclaim as per usual then. |
| 2920 | */ |
| 2921 | anon = node_page_state(pgdat, NR_INACTIVE_ANON); |
| 2922 | |
| 2923 | sc->file_is_tiny = |
| 2924 | file + free <= total_high_wmark && |
| 2925 | !(sc->may_deactivate & DEACTIVATE_ANON) && |
| 2926 | anon >> sc->priority; |
| 2927 | } |
| 2928 | } |
| 2929 | |
| 2930 | /* |
| 2931 | * Determine how aggressively the anon and file LRU lists should be |
| 2932 | * scanned. |
| 2933 | * |
| 2934 | * nr[0] = anon inactive folios to scan; nr[1] = anon active folios to scan |
| 2935 | * nr[2] = file inactive folios to scan; nr[3] = file active folios to scan |
| 2936 | */ |
| 2937 | static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, |
| 2938 | unsigned long *nr) |
| 2939 | { |
| 2940 | struct pglist_data *pgdat = lruvec_pgdat(lruvec); |
| 2941 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 2942 | unsigned long anon_cost, file_cost, total_cost; |
| 2943 | int swappiness = mem_cgroup_swappiness(memcg); |
| 2944 | u64 fraction[ANON_AND_FILE]; |
| 2945 | u64 denominator = 0; /* gcc */ |
| 2946 | enum scan_balance scan_balance; |
| 2947 | unsigned long ap, fp; |
| 2948 | enum lru_list lru; |
| 2949 | |
| 2950 | /* If we have no swap space, do not bother scanning anon folios. */ |
| 2951 | if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) { |
| 2952 | scan_balance = SCAN_FILE; |
| 2953 | goto out; |
| 2954 | } |
| 2955 | |
| 2956 | /* |
| 2957 | * Global reclaim will swap to prevent OOM even with no |
| 2958 | * swappiness, but memcg users want to use this knob to |
| 2959 | * disable swapping for individual groups completely when |
| 2960 | * using the memory controller's swap limit feature would be |
| 2961 | * too expensive. |
| 2962 | */ |
| 2963 | if (cgroup_reclaim(sc) && !swappiness) { |
| 2964 | scan_balance = SCAN_FILE; |
| 2965 | goto out; |
| 2966 | } |
| 2967 | |
| 2968 | /* |
| 2969 | * Do not apply any pressure balancing cleverness when the |
| 2970 | * system is close to OOM, scan both anon and file equally |
| 2971 | * (unless the swappiness setting disagrees with swapping). |
| 2972 | */ |
| 2973 | if (!sc->priority && swappiness) { |
| 2974 | scan_balance = SCAN_EQUAL; |
| 2975 | goto out; |
| 2976 | } |
| 2977 | |
| 2978 | /* |
| 2979 | * If the system is almost out of file pages, force-scan anon. |
| 2980 | */ |
| 2981 | if (sc->file_is_tiny) { |
| 2982 | scan_balance = SCAN_ANON; |
| 2983 | goto out; |
| 2984 | } |
| 2985 | |
| 2986 | /* |
| 2987 | * If there is enough inactive page cache, we do not reclaim |
| 2988 | * anything from the anonymous working right now. |
| 2989 | */ |
| 2990 | if (sc->cache_trim_mode) { |
| 2991 | scan_balance = SCAN_FILE; |
| 2992 | goto out; |
| 2993 | } |
| 2994 | |
| 2995 | scan_balance = SCAN_FRACT; |
| 2996 | /* |
| 2997 | * Calculate the pressure balance between anon and file pages. |
| 2998 | * |
| 2999 | * The amount of pressure we put on each LRU is inversely |
| 3000 | * proportional to the cost of reclaiming each list, as |
| 3001 | * determined by the share of pages that are refaulting, times |
| 3002 | * the relative IO cost of bringing back a swapped out |
| 3003 | * anonymous page vs reloading a filesystem page (swappiness). |
| 3004 | * |
| 3005 | * Although we limit that influence to ensure no list gets |
| 3006 | * left behind completely: at least a third of the pressure is |
| 3007 | * applied, before swappiness. |
| 3008 | * |
| 3009 | * With swappiness at 100, anon and file have equal IO cost. |
| 3010 | */ |
| 3011 | total_cost = sc->anon_cost + sc->file_cost; |
| 3012 | anon_cost = total_cost + sc->anon_cost; |
| 3013 | file_cost = total_cost + sc->file_cost; |
| 3014 | total_cost = anon_cost + file_cost; |
| 3015 | |
| 3016 | ap = swappiness * (total_cost + 1); |
| 3017 | ap /= anon_cost + 1; |
| 3018 | |
| 3019 | fp = (200 - swappiness) * (total_cost + 1); |
| 3020 | fp /= file_cost + 1; |
| 3021 | |
| 3022 | fraction[0] = ap; |
| 3023 | fraction[1] = fp; |
| 3024 | denominator = ap + fp; |
| 3025 | out: |
| 3026 | for_each_evictable_lru(lru) { |
| 3027 | int file = is_file_lru(lru); |
| 3028 | unsigned long lruvec_size; |
| 3029 | unsigned long low, min; |
| 3030 | unsigned long scan; |
| 3031 | |
| 3032 | lruvec_size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx); |
| 3033 | mem_cgroup_protection(sc->target_mem_cgroup, memcg, |
| 3034 | &min, &low); |
| 3035 | |
| 3036 | if (min || low) { |
| 3037 | /* |
| 3038 | * Scale a cgroup's reclaim pressure by proportioning |
| 3039 | * its current usage to its memory.low or memory.min |
| 3040 | * setting. |
| 3041 | * |
| 3042 | * This is important, as otherwise scanning aggression |
| 3043 | * becomes extremely binary -- from nothing as we |
| 3044 | * approach the memory protection threshold, to totally |
| 3045 | * nominal as we exceed it. This results in requiring |
| 3046 | * setting extremely liberal protection thresholds. It |
| 3047 | * also means we simply get no protection at all if we |
| 3048 | * set it too low, which is not ideal. |
| 3049 | * |
| 3050 | * If there is any protection in place, we reduce scan |
| 3051 | * pressure by how much of the total memory used is |
| 3052 | * within protection thresholds. |
| 3053 | * |
| 3054 | * There is one special case: in the first reclaim pass, |
| 3055 | * we skip over all groups that are within their low |
| 3056 | * protection. If that fails to reclaim enough pages to |
| 3057 | * satisfy the reclaim goal, we come back and override |
| 3058 | * the best-effort low protection. However, we still |
| 3059 | * ideally want to honor how well-behaved groups are in |
| 3060 | * that case instead of simply punishing them all |
| 3061 | * equally. As such, we reclaim them based on how much |
| 3062 | * memory they are using, reducing the scan pressure |
| 3063 | * again by how much of the total memory used is under |
| 3064 | * hard protection. |
| 3065 | */ |
| 3066 | unsigned long cgroup_size = mem_cgroup_size(memcg); |
| 3067 | unsigned long protection; |
| 3068 | |
| 3069 | /* memory.low scaling, make sure we retry before OOM */ |
| 3070 | if (!sc->memcg_low_reclaim && low > min) { |
| 3071 | protection = low; |
| 3072 | sc->memcg_low_skipped = 1; |
| 3073 | } else { |
| 3074 | protection = min; |
| 3075 | } |
| 3076 | |
| 3077 | /* Avoid TOCTOU with earlier protection check */ |
| 3078 | cgroup_size = max(cgroup_size, protection); |
| 3079 | |
| 3080 | scan = lruvec_size - lruvec_size * protection / |
| 3081 | (cgroup_size + 1); |
| 3082 | |
| 3083 | /* |
| 3084 | * Minimally target SWAP_CLUSTER_MAX pages to keep |
| 3085 | * reclaim moving forwards, avoiding decrementing |
| 3086 | * sc->priority further than desirable. |
| 3087 | */ |
| 3088 | scan = max(scan, SWAP_CLUSTER_MAX); |
| 3089 | } else { |
| 3090 | scan = lruvec_size; |
| 3091 | } |
| 3092 | |
| 3093 | scan >>= sc->priority; |
| 3094 | |
| 3095 | /* |
| 3096 | * If the cgroup's already been deleted, make sure to |
| 3097 | * scrape out the remaining cache. |
| 3098 | */ |
| 3099 | if (!scan && !mem_cgroup_online(memcg)) |
| 3100 | scan = min(lruvec_size, SWAP_CLUSTER_MAX); |
| 3101 | |
| 3102 | switch (scan_balance) { |
| 3103 | case SCAN_EQUAL: |
| 3104 | /* Scan lists relative to size */ |
| 3105 | break; |
| 3106 | case SCAN_FRACT: |
| 3107 | /* |
| 3108 | * Scan types proportional to swappiness and |
| 3109 | * their relative recent reclaim efficiency. |
| 3110 | * Make sure we don't miss the last page on |
| 3111 | * the offlined memory cgroups because of a |
| 3112 | * round-off error. |
| 3113 | */ |
| 3114 | scan = mem_cgroup_online(memcg) ? |
| 3115 | div64_u64(scan * fraction[file], denominator) : |
| 3116 | DIV64_U64_ROUND_UP(scan * fraction[file], |
| 3117 | denominator); |
| 3118 | break; |
| 3119 | case SCAN_FILE: |
| 3120 | case SCAN_ANON: |
| 3121 | /* Scan one type exclusively */ |
| 3122 | if ((scan_balance == SCAN_FILE) != file) |
| 3123 | scan = 0; |
| 3124 | break; |
| 3125 | default: |
| 3126 | /* Look ma, no brain */ |
| 3127 | BUG(); |
| 3128 | } |
| 3129 | |
| 3130 | nr[lru] = scan; |
| 3131 | } |
| 3132 | } |
| 3133 | |
| 3134 | /* |
| 3135 | * Anonymous LRU management is a waste if there is |
| 3136 | * ultimately no way to reclaim the memory. |
| 3137 | */ |
| 3138 | static bool can_age_anon_pages(struct pglist_data *pgdat, |
| 3139 | struct scan_control *sc) |
| 3140 | { |
| 3141 | /* Aging the anon LRU is valuable if swap is present: */ |
| 3142 | if (total_swap_pages > 0) |
| 3143 | return true; |
| 3144 | |
| 3145 | /* Also valuable if anon pages can be demoted: */ |
| 3146 | return can_demote(pgdat->node_id, sc); |
| 3147 | } |
| 3148 | |
| 3149 | #ifdef CONFIG_LRU_GEN |
| 3150 | |
| 3151 | #ifdef CONFIG_LRU_GEN_ENABLED |
| 3152 | DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS); |
| 3153 | #define get_cap(cap) static_branch_likely(&lru_gen_caps[cap]) |
| 3154 | #else |
| 3155 | DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS); |
| 3156 | #define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap]) |
| 3157 | #endif |
| 3158 | |
| 3159 | /****************************************************************************** |
| 3160 | * shorthand helpers |
| 3161 | ******************************************************************************/ |
| 3162 | |
| 3163 | #define LRU_REFS_FLAGS (BIT(PG_referenced) | BIT(PG_workingset)) |
| 3164 | |
| 3165 | #define DEFINE_MAX_SEQ(lruvec) \ |
| 3166 | unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq) |
| 3167 | |
| 3168 | #define DEFINE_MIN_SEQ(lruvec) \ |
| 3169 | unsigned long min_seq[ANON_AND_FILE] = { \ |
| 3170 | READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \ |
| 3171 | READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \ |
| 3172 | } |
| 3173 | |
| 3174 | #define for_each_gen_type_zone(gen, type, zone) \ |
| 3175 | for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \ |
| 3176 | for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \ |
| 3177 | for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++) |
| 3178 | |
| 3179 | static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid) |
| 3180 | { |
| 3181 | struct pglist_data *pgdat = NODE_DATA(nid); |
| 3182 | |
| 3183 | #ifdef CONFIG_MEMCG |
| 3184 | if (memcg) { |
| 3185 | struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec; |
| 3186 | |
| 3187 | /* see the comment in mem_cgroup_lruvec() */ |
| 3188 | if (!lruvec->pgdat) |
| 3189 | lruvec->pgdat = pgdat; |
| 3190 | |
| 3191 | return lruvec; |
| 3192 | } |
| 3193 | #endif |
| 3194 | VM_WARN_ON_ONCE(!mem_cgroup_disabled()); |
| 3195 | |
| 3196 | return &pgdat->__lruvec; |
| 3197 | } |
| 3198 | |
| 3199 | static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) |
| 3200 | { |
| 3201 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 3202 | struct pglist_data *pgdat = lruvec_pgdat(lruvec); |
| 3203 | |
| 3204 | if (!can_demote(pgdat->node_id, sc) && |
| 3205 | mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) |
| 3206 | return 0; |
| 3207 | |
| 3208 | return mem_cgroup_swappiness(memcg); |
| 3209 | } |
| 3210 | |
| 3211 | static int get_nr_gens(struct lruvec *lruvec, int type) |
| 3212 | { |
| 3213 | return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1; |
| 3214 | } |
| 3215 | |
| 3216 | static bool __maybe_unused seq_is_valid(struct lruvec *lruvec) |
| 3217 | { |
| 3218 | /* see the comment on lru_gen_struct */ |
| 3219 | return get_nr_gens(lruvec, LRU_GEN_FILE) >= MIN_NR_GENS && |
| 3220 | get_nr_gens(lruvec, LRU_GEN_FILE) <= get_nr_gens(lruvec, LRU_GEN_ANON) && |
| 3221 | get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS; |
| 3222 | } |
| 3223 | |
| 3224 | /****************************************************************************** |
| 3225 | * mm_struct list |
| 3226 | ******************************************************************************/ |
| 3227 | |
| 3228 | static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) |
| 3229 | { |
| 3230 | static struct lru_gen_mm_list mm_list = { |
| 3231 | .fifo = LIST_HEAD_INIT(mm_list.fifo), |
| 3232 | .lock = __SPIN_LOCK_UNLOCKED(mm_list.lock), |
| 3233 | }; |
| 3234 | |
| 3235 | #ifdef CONFIG_MEMCG |
| 3236 | if (memcg) |
| 3237 | return &memcg->mm_list; |
| 3238 | #endif |
| 3239 | VM_WARN_ON_ONCE(!mem_cgroup_disabled()); |
| 3240 | |
| 3241 | return &mm_list; |
| 3242 | } |
| 3243 | |
| 3244 | void lru_gen_add_mm(struct mm_struct *mm) |
| 3245 | { |
| 3246 | int nid; |
| 3247 | struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); |
| 3248 | struct lru_gen_mm_list *mm_list = get_mm_list(memcg); |
| 3249 | |
| 3250 | VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list)); |
| 3251 | #ifdef CONFIG_MEMCG |
| 3252 | VM_WARN_ON_ONCE(mm->lru_gen.memcg); |
| 3253 | mm->lru_gen.memcg = memcg; |
| 3254 | #endif |
| 3255 | spin_lock(&mm_list->lock); |
| 3256 | |
| 3257 | for_each_node_state(nid, N_MEMORY) { |
| 3258 | struct lruvec *lruvec = get_lruvec(memcg, nid); |
| 3259 | |
| 3260 | /* the first addition since the last iteration */ |
| 3261 | if (lruvec->mm_state.tail == &mm_list->fifo) |
| 3262 | lruvec->mm_state.tail = &mm->lru_gen.list; |
| 3263 | } |
| 3264 | |
| 3265 | list_add_tail(&mm->lru_gen.list, &mm_list->fifo); |
| 3266 | |
| 3267 | spin_unlock(&mm_list->lock); |
| 3268 | } |
| 3269 | |
| 3270 | void lru_gen_del_mm(struct mm_struct *mm) |
| 3271 | { |
| 3272 | int nid; |
| 3273 | struct lru_gen_mm_list *mm_list; |
| 3274 | struct mem_cgroup *memcg = NULL; |
| 3275 | |
| 3276 | if (list_empty(&mm->lru_gen.list)) |
| 3277 | return; |
| 3278 | |
| 3279 | #ifdef CONFIG_MEMCG |
| 3280 | memcg = mm->lru_gen.memcg; |
| 3281 | #endif |
| 3282 | mm_list = get_mm_list(memcg); |
| 3283 | |
| 3284 | spin_lock(&mm_list->lock); |
| 3285 | |
| 3286 | for_each_node(nid) { |
| 3287 | struct lruvec *lruvec = get_lruvec(memcg, nid); |
| 3288 | |
| 3289 | /* where the last iteration ended (exclusive) */ |
| 3290 | if (lruvec->mm_state.tail == &mm->lru_gen.list) |
| 3291 | lruvec->mm_state.tail = lruvec->mm_state.tail->next; |
| 3292 | |
| 3293 | /* where the current iteration continues (inclusive) */ |
| 3294 | if (lruvec->mm_state.head != &mm->lru_gen.list) |
| 3295 | continue; |
| 3296 | |
| 3297 | lruvec->mm_state.head = lruvec->mm_state.head->next; |
| 3298 | /* the deletion ends the current iteration */ |
| 3299 | if (lruvec->mm_state.head == &mm_list->fifo) |
| 3300 | WRITE_ONCE(lruvec->mm_state.seq, lruvec->mm_state.seq + 1); |
| 3301 | } |
| 3302 | |
| 3303 | list_del_init(&mm->lru_gen.list); |
| 3304 | |
| 3305 | spin_unlock(&mm_list->lock); |
| 3306 | |
| 3307 | #ifdef CONFIG_MEMCG |
| 3308 | mem_cgroup_put(mm->lru_gen.memcg); |
| 3309 | mm->lru_gen.memcg = NULL; |
| 3310 | #endif |
| 3311 | } |
| 3312 | |
| 3313 | #ifdef CONFIG_MEMCG |
| 3314 | void lru_gen_migrate_mm(struct mm_struct *mm) |
| 3315 | { |
| 3316 | struct mem_cgroup *memcg; |
| 3317 | struct task_struct *task = rcu_dereference_protected(mm->owner, true); |
| 3318 | |
| 3319 | VM_WARN_ON_ONCE(task->mm != mm); |
| 3320 | lockdep_assert_held(&task->alloc_lock); |
| 3321 | |
| 3322 | /* for mm_update_next_owner() */ |
| 3323 | if (mem_cgroup_disabled()) |
| 3324 | return; |
| 3325 | |
| 3326 | rcu_read_lock(); |
| 3327 | memcg = mem_cgroup_from_task(task); |
| 3328 | rcu_read_unlock(); |
| 3329 | if (memcg == mm->lru_gen.memcg) |
| 3330 | return; |
| 3331 | |
| 3332 | VM_WARN_ON_ONCE(!mm->lru_gen.memcg); |
| 3333 | VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list)); |
| 3334 | |
| 3335 | lru_gen_del_mm(mm); |
| 3336 | lru_gen_add_mm(mm); |
| 3337 | } |
| 3338 | #endif |
| 3339 | |
| 3340 | /* |
| 3341 | * Bloom filters with m=1<<15, k=2 and the false positive rates of ~1/5 when |
| 3342 | * n=10,000 and ~1/2 when n=20,000, where, conventionally, m is the number of |
| 3343 | * bits in a bitmap, k is the number of hash functions and n is the number of |
| 3344 | * inserted items. |
| 3345 | * |
| 3346 | * Page table walkers use one of the two filters to reduce their search space. |
| 3347 | * To get rid of non-leaf entries that no longer have enough leaf entries, the |
| 3348 | * aging uses the double-buffering technique to flip to the other filter each |
| 3349 | * time it produces a new generation. For non-leaf entries that have enough |
| 3350 | * leaf entries, the aging carries them over to the next generation in |
| 3351 | * walk_pmd_range(); the eviction also report them when walking the rmap |
| 3352 | * in lru_gen_look_around(). |
| 3353 | * |
| 3354 | * For future optimizations: |
| 3355 | * 1. It's not necessary to keep both filters all the time. The spare one can be |
| 3356 | * freed after the RCU grace period and reallocated if needed again. |
| 3357 | * 2. And when reallocating, it's worth scaling its size according to the number |
| 3358 | * of inserted entries in the other filter, to reduce the memory overhead on |
| 3359 | * small systems and false positives on large systems. |
| 3360 | * 3. Jenkins' hash function is an alternative to Knuth's. |
| 3361 | */ |
| 3362 | #define BLOOM_FILTER_SHIFT 15 |
| 3363 | |
| 3364 | static inline int filter_gen_from_seq(unsigned long seq) |
| 3365 | { |
| 3366 | return seq % NR_BLOOM_FILTERS; |
| 3367 | } |
| 3368 | |
| 3369 | static void get_item_key(void *item, int *key) |
| 3370 | { |
| 3371 | u32 hash = hash_ptr(item, BLOOM_FILTER_SHIFT * 2); |
| 3372 | |
| 3373 | BUILD_BUG_ON(BLOOM_FILTER_SHIFT * 2 > BITS_PER_TYPE(u32)); |
| 3374 | |
| 3375 | key[0] = hash & (BIT(BLOOM_FILTER_SHIFT) - 1); |
| 3376 | key[1] = hash >> BLOOM_FILTER_SHIFT; |
| 3377 | } |
| 3378 | |
| 3379 | static void reset_bloom_filter(struct lruvec *lruvec, unsigned long seq) |
| 3380 | { |
| 3381 | unsigned long *filter; |
| 3382 | int gen = filter_gen_from_seq(seq); |
| 3383 | |
| 3384 | filter = lruvec->mm_state.filters[gen]; |
| 3385 | if (filter) { |
| 3386 | bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT)); |
| 3387 | return; |
| 3388 | } |
| 3389 | |
| 3390 | filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), |
| 3391 | __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); |
| 3392 | WRITE_ONCE(lruvec->mm_state.filters[gen], filter); |
| 3393 | } |
| 3394 | |
| 3395 | static void update_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) |
| 3396 | { |
| 3397 | int key[2]; |
| 3398 | unsigned long *filter; |
| 3399 | int gen = filter_gen_from_seq(seq); |
| 3400 | |
| 3401 | filter = READ_ONCE(lruvec->mm_state.filters[gen]); |
| 3402 | if (!filter) |
| 3403 | return; |
| 3404 | |
| 3405 | get_item_key(item, key); |
| 3406 | |
| 3407 | if (!test_bit(key[0], filter)) |
| 3408 | set_bit(key[0], filter); |
| 3409 | if (!test_bit(key[1], filter)) |
| 3410 | set_bit(key[1], filter); |
| 3411 | } |
| 3412 | |
| 3413 | static bool test_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) |
| 3414 | { |
| 3415 | int key[2]; |
| 3416 | unsigned long *filter; |
| 3417 | int gen = filter_gen_from_seq(seq); |
| 3418 | |
| 3419 | filter = READ_ONCE(lruvec->mm_state.filters[gen]); |
| 3420 | if (!filter) |
| 3421 | return true; |
| 3422 | |
| 3423 | get_item_key(item, key); |
| 3424 | |
| 3425 | return test_bit(key[0], filter) && test_bit(key[1], filter); |
| 3426 | } |
| 3427 | |
| 3428 | static void reset_mm_stats(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, bool last) |
| 3429 | { |
| 3430 | int i; |
| 3431 | int hist; |
| 3432 | |
| 3433 | lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock); |
| 3434 | |
| 3435 | if (walk) { |
| 3436 | hist = lru_hist_from_seq(walk->max_seq); |
| 3437 | |
| 3438 | for (i = 0; i < NR_MM_STATS; i++) { |
| 3439 | WRITE_ONCE(lruvec->mm_state.stats[hist][i], |
| 3440 | lruvec->mm_state.stats[hist][i] + walk->mm_stats[i]); |
| 3441 | walk->mm_stats[i] = 0; |
| 3442 | } |
| 3443 | } |
| 3444 | |
| 3445 | if (NR_HIST_GENS > 1 && last) { |
| 3446 | hist = lru_hist_from_seq(lruvec->mm_state.seq + 1); |
| 3447 | |
| 3448 | for (i = 0; i < NR_MM_STATS; i++) |
| 3449 | WRITE_ONCE(lruvec->mm_state.stats[hist][i], 0); |
| 3450 | } |
| 3451 | } |
| 3452 | |
| 3453 | static bool should_skip_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) |
| 3454 | { |
| 3455 | int type; |
| 3456 | unsigned long size = 0; |
| 3457 | struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); |
| 3458 | int key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap); |
| 3459 | |
| 3460 | if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap)) |
| 3461 | return true; |
| 3462 | |
| 3463 | clear_bit(key, &mm->lru_gen.bitmap); |
| 3464 | |
| 3465 | for (type = !walk->can_swap; type < ANON_AND_FILE; type++) { |
| 3466 | size += type ? get_mm_counter(mm, MM_FILEPAGES) : |
| 3467 | get_mm_counter(mm, MM_ANONPAGES) + |
| 3468 | get_mm_counter(mm, MM_SHMEMPAGES); |
| 3469 | } |
| 3470 | |
| 3471 | if (size < MIN_LRU_BATCH) |
| 3472 | return true; |
| 3473 | |
| 3474 | return !mmget_not_zero(mm); |
| 3475 | } |
| 3476 | |
| 3477 | static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, |
| 3478 | struct mm_struct **iter) |
| 3479 | { |
| 3480 | bool first = false; |
| 3481 | bool last = true; |
| 3482 | struct mm_struct *mm = NULL; |
| 3483 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 3484 | struct lru_gen_mm_list *mm_list = get_mm_list(memcg); |
| 3485 | struct lru_gen_mm_state *mm_state = &lruvec->mm_state; |
| 3486 | |
| 3487 | /* |
| 3488 | * There are four interesting cases for this page table walker: |
| 3489 | * 1. It tries to start a new iteration of mm_list with a stale max_seq; |
| 3490 | * there is nothing left to do. |
| 3491 | * 2. It's the first of the current generation, and it needs to reset |
| 3492 | * the Bloom filter for the next generation. |
| 3493 | * 3. It reaches the end of mm_list, and it needs to increment |
| 3494 | * mm_state->seq; the iteration is done. |
| 3495 | * 4. It's the last of the current generation, and it needs to reset the |
| 3496 | * mm stats counters for the next generation. |
| 3497 | */ |
| 3498 | spin_lock(&mm_list->lock); |
| 3499 | |
| 3500 | VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq); |
| 3501 | VM_WARN_ON_ONCE(*iter && mm_state->seq > walk->max_seq); |
| 3502 | VM_WARN_ON_ONCE(*iter && !mm_state->nr_walkers); |
| 3503 | |
| 3504 | if (walk->max_seq <= mm_state->seq) { |
| 3505 | if (!*iter) |
| 3506 | last = false; |
| 3507 | goto done; |
| 3508 | } |
| 3509 | |
| 3510 | if (!mm_state->nr_walkers) { |
| 3511 | VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); |
| 3512 | |
| 3513 | mm_state->head = mm_list->fifo.next; |
| 3514 | first = true; |
| 3515 | } |
| 3516 | |
| 3517 | while (!mm && mm_state->head != &mm_list->fifo) { |
| 3518 | mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list); |
| 3519 | |
| 3520 | mm_state->head = mm_state->head->next; |
| 3521 | |
| 3522 | /* force scan for those added after the last iteration */ |
| 3523 | if (!mm_state->tail || mm_state->tail == &mm->lru_gen.list) { |
| 3524 | mm_state->tail = mm_state->head; |
| 3525 | walk->force_scan = true; |
| 3526 | } |
| 3527 | |
| 3528 | if (should_skip_mm(mm, walk)) |
| 3529 | mm = NULL; |
| 3530 | } |
| 3531 | |
| 3532 | if (mm_state->head == &mm_list->fifo) |
| 3533 | WRITE_ONCE(mm_state->seq, mm_state->seq + 1); |
| 3534 | done: |
| 3535 | if (*iter && !mm) |
| 3536 | mm_state->nr_walkers--; |
| 3537 | if (!*iter && mm) |
| 3538 | mm_state->nr_walkers++; |
| 3539 | |
| 3540 | if (mm_state->nr_walkers) |
| 3541 | last = false; |
| 3542 | |
| 3543 | if (*iter || last) |
| 3544 | reset_mm_stats(lruvec, walk, last); |
| 3545 | |
| 3546 | spin_unlock(&mm_list->lock); |
| 3547 | |
| 3548 | if (mm && first) |
| 3549 | reset_bloom_filter(lruvec, walk->max_seq + 1); |
| 3550 | |
| 3551 | if (*iter) |
| 3552 | mmput_async(*iter); |
| 3553 | |
| 3554 | *iter = mm; |
| 3555 | |
| 3556 | return last; |
| 3557 | } |
| 3558 | |
| 3559 | static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) |
| 3560 | { |
| 3561 | bool success = false; |
| 3562 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 3563 | struct lru_gen_mm_list *mm_list = get_mm_list(memcg); |
| 3564 | struct lru_gen_mm_state *mm_state = &lruvec->mm_state; |
| 3565 | |
| 3566 | spin_lock(&mm_list->lock); |
| 3567 | |
| 3568 | VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq); |
| 3569 | |
| 3570 | if (max_seq > mm_state->seq && !mm_state->nr_walkers) { |
| 3571 | VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); |
| 3572 | |
| 3573 | WRITE_ONCE(mm_state->seq, mm_state->seq + 1); |
| 3574 | reset_mm_stats(lruvec, NULL, true); |
| 3575 | success = true; |
| 3576 | } |
| 3577 | |
| 3578 | spin_unlock(&mm_list->lock); |
| 3579 | |
| 3580 | return success; |
| 3581 | } |
| 3582 | |
| 3583 | /****************************************************************************** |
| 3584 | * refault feedback loop |
| 3585 | ******************************************************************************/ |
| 3586 | |
| 3587 | /* |
| 3588 | * A feedback loop based on Proportional-Integral-Derivative (PID) controller. |
| 3589 | * |
| 3590 | * The P term is refaulted/(evicted+protected) from a tier in the generation |
| 3591 | * currently being evicted; the I term is the exponential moving average of the |
| 3592 | * P term over the generations previously evicted, using the smoothing factor |
| 3593 | * 1/2; the D term isn't supported. |
| 3594 | * |
| 3595 | * The setpoint (SP) is always the first tier of one type; the process variable |
| 3596 | * (PV) is either any tier of the other type or any other tier of the same |
| 3597 | * type. |
| 3598 | * |
| 3599 | * The error is the difference between the SP and the PV; the correction is to |
| 3600 | * turn off protection when SP>PV or turn on protection when SP<PV. |
| 3601 | * |
| 3602 | * For future optimizations: |
| 3603 | * 1. The D term may discount the other two terms over time so that long-lived |
| 3604 | * generations can resist stale information. |
| 3605 | */ |
| 3606 | struct ctrl_pos { |
| 3607 | unsigned long refaulted; |
| 3608 | unsigned long total; |
| 3609 | int gain; |
| 3610 | }; |
| 3611 | |
| 3612 | static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain, |
| 3613 | struct ctrl_pos *pos) |
| 3614 | { |
| 3615 | struct lru_gen_struct *lrugen = &lruvec->lrugen; |
| 3616 | int hist = lru_hist_from_seq(lrugen->min_seq[type]); |
| 3617 | |
| 3618 | pos->refaulted = lrugen->avg_refaulted[type][tier] + |
| 3619 | atomic_long_read(&lrugen->refaulted[hist][type][tier]); |
| 3620 | pos->total = lrugen->avg_total[type][tier] + |
| 3621 | atomic_long_read(&lrugen->evicted[hist][type][tier]); |
| 3622 | if (tier) |
| 3623 | pos->total += lrugen->protected[hist][type][tier - 1]; |
| 3624 | pos->gain = gain; |
| 3625 | } |
| 3626 | |
| 3627 | static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover) |
| 3628 | { |
| 3629 | int hist, tier; |
| 3630 | struct lru_gen_struct *lrugen = &lruvec->lrugen; |
| 3631 | bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1; |
| 3632 | unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1; |
| 3633 | |
| 3634 | lockdep_assert_held(&lruvec->lru_lock); |
| 3635 | |
| 3636 | if (!carryover && !clear) |
| 3637 | return; |
| 3638 | |
| 3639 | hist = lru_hist_from_seq(seq); |
| 3640 | |
| 3641 | for (tier = 0; tier < MAX_NR_TIERS; tier++) { |
| 3642 | if (carryover) { |
| 3643 | unsigned long sum; |
| 3644 | |
| 3645 | sum = lrugen->avg_refaulted[type][tier] + |
| 3646 | atomic_long_read(&lrugen->refaulted[hist][type][tier]); |
| 3647 | WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2); |
| 3648 | |
| 3649 | sum = lrugen->avg_total[type][tier] + |
| 3650 | atomic_long_read(&lrugen->evicted[hist][type][tier]); |
| 3651 | if (tier) |
| 3652 | sum += lrugen->protected[hist][type][tier - 1]; |
| 3653 | WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2); |
| 3654 | } |
| 3655 | |
| 3656 | if (clear) { |
| 3657 | atomic_long_set(&lrugen->refaulted[hist][type][tier], 0); |
| 3658 | atomic_long_set(&lrugen->evicted[hist][type][tier], 0); |
| 3659 | if (tier) |
| 3660 | WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0); |
| 3661 | } |
| 3662 | } |
| 3663 | } |
| 3664 | |
| 3665 | static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) |
| 3666 | { |
| 3667 | /* |
| 3668 | * Return true if the PV has a limited number of refaults or a lower |
| 3669 | * refaulted/total than the SP. |
| 3670 | */ |
| 3671 | return pv->refaulted < MIN_LRU_BATCH || |
| 3672 | pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <= |
| 3673 | (sp->refaulted + 1) * pv->total * pv->gain; |
| 3674 | } |
| 3675 | |
| 3676 | /****************************************************************************** |
| 3677 | * the aging |
| 3678 | ******************************************************************************/ |
| 3679 | |
| 3680 | /* promote pages accessed through page tables */ |
| 3681 | static int folio_update_gen(struct folio *folio, int gen) |
| 3682 | { |
| 3683 | unsigned long new_flags, old_flags = READ_ONCE(folio->flags); |
| 3684 | |
| 3685 | VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); |
| 3686 | VM_WARN_ON_ONCE(!rcu_read_lock_held()); |
| 3687 | |
| 3688 | do { |
| 3689 | /* lru_gen_del_folio() has isolated this page? */ |
| 3690 | if (!(old_flags & LRU_GEN_MASK)) { |
| 3691 | /* for shrink_folio_list() */ |
| 3692 | new_flags = old_flags | BIT(PG_referenced); |
| 3693 | continue; |
| 3694 | } |
| 3695 | |
| 3696 | new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); |
| 3697 | new_flags |= (gen + 1UL) << LRU_GEN_PGOFF; |
| 3698 | } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); |
| 3699 | |
| 3700 | return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; |
| 3701 | } |
| 3702 | |
| 3703 | /* protect pages accessed multiple times through file descriptors */ |
| 3704 | static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming) |
| 3705 | { |
| 3706 | int type = folio_is_file_lru(folio); |
| 3707 | struct lru_gen_struct *lrugen = &lruvec->lrugen; |
| 3708 | int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); |
| 3709 | unsigned long new_flags, old_flags = READ_ONCE(folio->flags); |
| 3710 | |
| 3711 | VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); |
| 3712 | |
| 3713 | do { |
| 3714 | new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; |
| 3715 | /* folio_update_gen() has promoted this page? */ |
| 3716 | if (new_gen >= 0 && new_gen != old_gen) |
| 3717 | return new_gen; |
| 3718 | |
| 3719 | new_gen = (old_gen + 1) % MAX_NR_GENS; |
| 3720 | |
| 3721 | new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); |
| 3722 | new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; |
| 3723 | /* for folio_end_writeback() */ |
| 3724 | if (reclaiming) |
| 3725 | new_flags |= BIT(PG_reclaim); |
| 3726 | } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); |
| 3727 | |
| 3728 | lru_gen_update_size(lruvec, folio, old_gen, new_gen); |
| 3729 | |
| 3730 | return new_gen; |
| 3731 | } |
| 3732 | |
| 3733 | static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio, |
| 3734 | int old_gen, int new_gen) |
| 3735 | { |
| 3736 | int type = folio_is_file_lru(folio); |
| 3737 | int zone = folio_zonenum(folio); |
| 3738 | int delta = folio_nr_pages(folio); |
| 3739 | |
| 3740 | VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS); |
| 3741 | VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS); |
| 3742 | |
| 3743 | walk->batched++; |
| 3744 | |
| 3745 | walk->nr_pages[old_gen][type][zone] -= delta; |
| 3746 | walk->nr_pages[new_gen][type][zone] += delta; |
| 3747 | } |
| 3748 | |
| 3749 | static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk) |
| 3750 | { |
| 3751 | int gen, type, zone; |
| 3752 | struct lru_gen_struct *lrugen = &lruvec->lrugen; |
| 3753 | |
| 3754 | walk->batched = 0; |
| 3755 | |
| 3756 | for_each_gen_type_zone(gen, type, zone) { |
| 3757 | enum lru_list lru = type * LRU_INACTIVE_FILE; |
| 3758 | int delta = walk->nr_pages[gen][type][zone]; |
| 3759 | |
| 3760 | if (!delta) |
| 3761 | continue; |
| 3762 | |
| 3763 | walk->nr_pages[gen][type][zone] = 0; |
| 3764 | WRITE_ONCE(lrugen->nr_pages[gen][type][zone], |
| 3765 | lrugen->nr_pages[gen][type][zone] + delta); |
| 3766 | |
| 3767 | if (lru_gen_is_active(lruvec, gen)) |
| 3768 | lru += LRU_ACTIVE; |
| 3769 | __update_lru_size(lruvec, lru, zone, delta); |
| 3770 | } |
| 3771 | } |
| 3772 | |
| 3773 | static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args) |
| 3774 | { |
| 3775 | struct address_space *mapping; |
| 3776 | struct vm_area_struct *vma = args->vma; |
| 3777 | struct lru_gen_mm_walk *walk = args->private; |
| 3778 | |
| 3779 | if (!vma_is_accessible(vma)) |
| 3780 | return true; |
| 3781 | |
| 3782 | if (is_vm_hugetlb_page(vma)) |
| 3783 | return true; |
| 3784 | |
| 3785 | if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL | VM_SEQ_READ | VM_RAND_READ)) |
| 3786 | return true; |
| 3787 | |
| 3788 | if (vma == get_gate_vma(vma->vm_mm)) |
| 3789 | return true; |
| 3790 | |
| 3791 | if (vma_is_anonymous(vma)) |
| 3792 | return !walk->can_swap; |
| 3793 | |
| 3794 | if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping)) |
| 3795 | return true; |
| 3796 | |
| 3797 | mapping = vma->vm_file->f_mapping; |
| 3798 | if (mapping_unevictable(mapping)) |
| 3799 | return true; |
| 3800 | |
| 3801 | if (shmem_mapping(mapping)) |
| 3802 | return !walk->can_swap; |
| 3803 | |
| 3804 | /* to exclude special mappings like dax, etc. */ |
| 3805 | return !mapping->a_ops->read_folio; |
| 3806 | } |
| 3807 | |
| 3808 | /* |
| 3809 | * Some userspace memory allocators map many single-page VMAs. Instead of |
| 3810 | * returning back to the PGD table for each of such VMAs, finish an entire PMD |
| 3811 | * table to reduce zigzags and improve cache performance. |
| 3812 | */ |
| 3813 | static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args, |
| 3814 | unsigned long *vm_start, unsigned long *vm_end) |
| 3815 | { |
| 3816 | unsigned long start = round_up(*vm_end, size); |
| 3817 | unsigned long end = (start | ~mask) + 1; |
| 3818 | VMA_ITERATOR(vmi, args->mm, start); |
| 3819 | |
| 3820 | VM_WARN_ON_ONCE(mask & size); |
| 3821 | VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask)); |
| 3822 | |
| 3823 | for_each_vma(vmi, args->vma) { |
| 3824 | if (end && end <= args->vma->vm_start) |
| 3825 | return false; |
| 3826 | |
| 3827 | if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) |
| 3828 | continue; |
| 3829 | |
| 3830 | *vm_start = max(start, args->vma->vm_start); |
| 3831 | *vm_end = min(end - 1, args->vma->vm_end - 1) + 1; |
| 3832 | |
| 3833 | return true; |
| 3834 | } |
| 3835 | |
| 3836 | return false; |
| 3837 | } |
| 3838 | |
| 3839 | static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr) |
| 3840 | { |
| 3841 | unsigned long pfn = pte_pfn(pte); |
| 3842 | |
| 3843 | VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); |
| 3844 | |
| 3845 | if (!pte_present(pte) || is_zero_pfn(pfn)) |
| 3846 | return -1; |
| 3847 | |
| 3848 | if (WARN_ON_ONCE(pte_devmap(pte) || pte_special(pte))) |
| 3849 | return -1; |
| 3850 | |
| 3851 | if (WARN_ON_ONCE(!pfn_valid(pfn))) |
| 3852 | return -1; |
| 3853 | |
| 3854 | return pfn; |
| 3855 | } |
| 3856 | |
| 3857 | #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) |
| 3858 | static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr) |
| 3859 | { |
| 3860 | unsigned long pfn = pmd_pfn(pmd); |
| 3861 | |
| 3862 | VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); |
| 3863 | |
| 3864 | if (!pmd_present(pmd) || is_huge_zero_pmd(pmd)) |
| 3865 | return -1; |
| 3866 | |
| 3867 | if (WARN_ON_ONCE(pmd_devmap(pmd))) |
| 3868 | return -1; |
| 3869 | |
| 3870 | if (WARN_ON_ONCE(!pfn_valid(pfn))) |
| 3871 | return -1; |
| 3872 | |
| 3873 | return pfn; |
| 3874 | } |
| 3875 | #endif |
| 3876 | |
| 3877 | static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg, |
| 3878 | struct pglist_data *pgdat, bool can_swap) |
| 3879 | { |
| 3880 | struct folio *folio; |
| 3881 | |
| 3882 | /* try to avoid unnecessary memory loads */ |
| 3883 | if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) |
| 3884 | return NULL; |
| 3885 | |
| 3886 | folio = pfn_folio(pfn); |
| 3887 | if (folio_nid(folio) != pgdat->node_id) |
| 3888 | return NULL; |
| 3889 | |
| 3890 | if (folio_memcg_rcu(folio) != memcg) |
| 3891 | return NULL; |
| 3892 | |
| 3893 | /* file VMAs can contain anon pages from COW */ |
| 3894 | if (!folio_is_file_lru(folio) && !can_swap) |
| 3895 | return NULL; |
| 3896 | |
| 3897 | return folio; |
| 3898 | } |
| 3899 | |
| 3900 | static bool suitable_to_scan(int total, int young) |
| 3901 | { |
| 3902 | int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8); |
| 3903 | |
| 3904 | /* suitable if the average number of young PTEs per cacheline is >=1 */ |
| 3905 | return young * n >= total; |
| 3906 | } |
| 3907 | |
| 3908 | static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, |
| 3909 | struct mm_walk *args) |
| 3910 | { |
| 3911 | int i; |
| 3912 | pte_t *pte; |
| 3913 | spinlock_t *ptl; |
| 3914 | unsigned long addr; |
| 3915 | int total = 0; |
| 3916 | int young = 0; |
| 3917 | struct lru_gen_mm_walk *walk = args->private; |
| 3918 | struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); |
| 3919 | struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); |
| 3920 | int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); |
| 3921 | |
| 3922 | VM_WARN_ON_ONCE(pmd_leaf(*pmd)); |
| 3923 | |
| 3924 | ptl = pte_lockptr(args->mm, pmd); |
| 3925 | if (!spin_trylock(ptl)) |
| 3926 | return false; |
| 3927 | |
| 3928 | arch_enter_lazy_mmu_mode(); |
| 3929 | |
| 3930 | pte = pte_offset_map(pmd, start & PMD_MASK); |
| 3931 | restart: |
| 3932 | for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) { |
| 3933 | unsigned long pfn; |
| 3934 | struct folio *folio; |
| 3935 | |
| 3936 | total++; |
| 3937 | walk->mm_stats[MM_LEAF_TOTAL]++; |
| 3938 | |
| 3939 | pfn = get_pte_pfn(pte[i], args->vma, addr); |
| 3940 | if (pfn == -1) |
| 3941 | continue; |
| 3942 | |
| 3943 | if (!pte_young(pte[i])) { |
| 3944 | walk->mm_stats[MM_LEAF_OLD]++; |
| 3945 | continue; |
| 3946 | } |
| 3947 | |
| 3948 | folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); |
| 3949 | if (!folio) |
| 3950 | continue; |
| 3951 | |
| 3952 | if (!ptep_test_and_clear_young(args->vma, addr, pte + i)) |
| 3953 | VM_WARN_ON_ONCE(true); |
| 3954 | |
| 3955 | young++; |
| 3956 | walk->mm_stats[MM_LEAF_YOUNG]++; |
| 3957 | |
| 3958 | if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && |
| 3959 | !(folio_test_anon(folio) && folio_test_swapbacked(folio) && |
| 3960 | !folio_test_swapcache(folio))) |
| 3961 | folio_mark_dirty(folio); |
| 3962 | |
| 3963 | old_gen = folio_update_gen(folio, new_gen); |
| 3964 | if (old_gen >= 0 && old_gen != new_gen) |
| 3965 | update_batch_size(walk, folio, old_gen, new_gen); |
| 3966 | } |
| 3967 | |
| 3968 | if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) |
| 3969 | goto restart; |
| 3970 | |
| 3971 | pte_unmap(pte); |
| 3972 | |
| 3973 | arch_leave_lazy_mmu_mode(); |
| 3974 | spin_unlock(ptl); |
| 3975 | |
| 3976 | return suitable_to_scan(total, young); |
| 3977 | } |
| 3978 | |
| 3979 | #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) |
| 3980 | static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, |
| 3981 | struct mm_walk *args, unsigned long *bitmap, unsigned long *start) |
| 3982 | { |
| 3983 | int i; |
| 3984 | pmd_t *pmd; |
| 3985 | spinlock_t *ptl; |
| 3986 | struct lru_gen_mm_walk *walk = args->private; |
| 3987 | struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); |
| 3988 | struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); |
| 3989 | int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); |
| 3990 | |
| 3991 | VM_WARN_ON_ONCE(pud_leaf(*pud)); |
| 3992 | |
| 3993 | /* try to batch at most 1+MIN_LRU_BATCH+1 entries */ |
| 3994 | if (*start == -1) { |
| 3995 | *start = next; |
| 3996 | return; |
| 3997 | } |
| 3998 | |
| 3999 | i = next == -1 ? 0 : pmd_index(next) - pmd_index(*start); |
| 4000 | if (i && i <= MIN_LRU_BATCH) { |
| 4001 | __set_bit(i - 1, bitmap); |
| 4002 | return; |
| 4003 | } |
| 4004 | |
| 4005 | pmd = pmd_offset(pud, *start); |
| 4006 | |
| 4007 | ptl = pmd_lockptr(args->mm, pmd); |
| 4008 | if (!spin_trylock(ptl)) |
| 4009 | goto done; |
| 4010 | |
| 4011 | arch_enter_lazy_mmu_mode(); |
| 4012 | |
| 4013 | do { |
| 4014 | unsigned long pfn; |
| 4015 | struct folio *folio; |
| 4016 | unsigned long addr = i ? (*start & PMD_MASK) + i * PMD_SIZE : *start; |
| 4017 | |
| 4018 | pfn = get_pmd_pfn(pmd[i], vma, addr); |
| 4019 | if (pfn == -1) |
| 4020 | goto next; |
| 4021 | |
| 4022 | if (!pmd_trans_huge(pmd[i])) { |
| 4023 | if (arch_has_hw_nonleaf_pmd_young() && |
| 4024 | get_cap(LRU_GEN_NONLEAF_YOUNG)) |
| 4025 | pmdp_test_and_clear_young(vma, addr, pmd + i); |
| 4026 | goto next; |
| 4027 | } |
| 4028 | |
| 4029 | folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); |
| 4030 | if (!folio) |
| 4031 | goto next; |
| 4032 | |
| 4033 | if (!pmdp_test_and_clear_young(vma, addr, pmd + i)) |
| 4034 | goto next; |
| 4035 | |
| 4036 | walk->mm_stats[MM_LEAF_YOUNG]++; |
| 4037 | |
| 4038 | if (pmd_dirty(pmd[i]) && !folio_test_dirty(folio) && |
| 4039 | !(folio_test_anon(folio) && folio_test_swapbacked(folio) && |
| 4040 | !folio_test_swapcache(folio))) |
| 4041 | folio_mark_dirty(folio); |
| 4042 | |
| 4043 | old_gen = folio_update_gen(folio, new_gen); |
| 4044 | if (old_gen >= 0 && old_gen != new_gen) |
| 4045 | update_batch_size(walk, folio, old_gen, new_gen); |
| 4046 | next: |
| 4047 | i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; |
| 4048 | } while (i <= MIN_LRU_BATCH); |
| 4049 | |
| 4050 | arch_leave_lazy_mmu_mode(); |
| 4051 | spin_unlock(ptl); |
| 4052 | done: |
| 4053 | *start = -1; |
| 4054 | bitmap_zero(bitmap, MIN_LRU_BATCH); |
| 4055 | } |
| 4056 | #else |
| 4057 | static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, |
| 4058 | struct mm_walk *args, unsigned long *bitmap, unsigned long *start) |
| 4059 | { |
| 4060 | } |
| 4061 | #endif |
| 4062 | |
| 4063 | static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, |
| 4064 | struct mm_walk *args) |
| 4065 | { |
| 4066 | int i; |
| 4067 | pmd_t *pmd; |
| 4068 | unsigned long next; |
| 4069 | unsigned long addr; |
| 4070 | struct vm_area_struct *vma; |
| 4071 | unsigned long pos = -1; |
| 4072 | struct lru_gen_mm_walk *walk = args->private; |
| 4073 | unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; |
| 4074 | |
| 4075 | VM_WARN_ON_ONCE(pud_leaf(*pud)); |
| 4076 | |
| 4077 | /* |
| 4078 | * Finish an entire PMD in two passes: the first only reaches to PTE |
| 4079 | * tables to avoid taking the PMD lock; the second, if necessary, takes |
| 4080 | * the PMD lock to clear the accessed bit in PMD entries. |
| 4081 | */ |
| 4082 | pmd = pmd_offset(pud, start & PUD_MASK); |
| 4083 | restart: |
| 4084 | /* walk_pte_range() may call get_next_vma() */ |
| 4085 | vma = args->vma; |
| 4086 | for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { |
| 4087 | pmd_t val = pmdp_get_lockless(pmd + i); |
| 4088 | |
| 4089 | next = pmd_addr_end(addr, end); |
| 4090 | |
| 4091 | if (!pmd_present(val) || is_huge_zero_pmd(val)) { |
| 4092 | walk->mm_stats[MM_LEAF_TOTAL]++; |
| 4093 | continue; |
| 4094 | } |
| 4095 | |
| 4096 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| 4097 | if (pmd_trans_huge(val)) { |
| 4098 | unsigned long pfn = pmd_pfn(val); |
| 4099 | struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); |
| 4100 | |
| 4101 | walk->mm_stats[MM_LEAF_TOTAL]++; |
| 4102 | |
| 4103 | if (!pmd_young(val)) { |
| 4104 | walk->mm_stats[MM_LEAF_OLD]++; |
| 4105 | continue; |
| 4106 | } |
| 4107 | |
| 4108 | /* try to avoid unnecessary memory loads */ |
| 4109 | if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) |
| 4110 | continue; |
| 4111 | |
| 4112 | walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); |
| 4113 | continue; |
| 4114 | } |
| 4115 | #endif |
| 4116 | walk->mm_stats[MM_NONLEAF_TOTAL]++; |
| 4117 | |
| 4118 | if (arch_has_hw_nonleaf_pmd_young() && |
| 4119 | get_cap(LRU_GEN_NONLEAF_YOUNG)) { |
| 4120 | if (!pmd_young(val)) |
| 4121 | continue; |
| 4122 | |
| 4123 | walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); |
| 4124 | } |
| 4125 | |
| 4126 | if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i)) |
| 4127 | continue; |
| 4128 | |
| 4129 | walk->mm_stats[MM_NONLEAF_FOUND]++; |
| 4130 | |
| 4131 | if (!walk_pte_range(&val, addr, next, args)) |
| 4132 | continue; |
| 4133 | |
| 4134 | walk->mm_stats[MM_NONLEAF_ADDED]++; |
| 4135 | |
| 4136 | /* carry over to the next generation */ |
| 4137 | update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i); |
| 4138 | } |
| 4139 | |
| 4140 | walk_pmd_range_locked(pud, -1, vma, args, bitmap, &pos); |
| 4141 | |
| 4142 | if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) |
| 4143 | goto restart; |
| 4144 | } |
| 4145 | |
| 4146 | static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, |
| 4147 | struct mm_walk *args) |
| 4148 | { |
| 4149 | int i; |
| 4150 | pud_t *pud; |
| 4151 | unsigned long addr; |
| 4152 | unsigned long next; |
| 4153 | struct lru_gen_mm_walk *walk = args->private; |
| 4154 | |
| 4155 | VM_WARN_ON_ONCE(p4d_leaf(*p4d)); |
| 4156 | |
| 4157 | pud = pud_offset(p4d, start & P4D_MASK); |
| 4158 | restart: |
| 4159 | for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { |
| 4160 | pud_t val = READ_ONCE(pud[i]); |
| 4161 | |
| 4162 | next = pud_addr_end(addr, end); |
| 4163 | |
| 4164 | if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) |
| 4165 | continue; |
| 4166 | |
| 4167 | walk_pmd_range(&val, addr, next, args); |
| 4168 | |
| 4169 | /* a racy check to curtail the waiting time */ |
| 4170 | if (wq_has_sleeper(&walk->lruvec->mm_state.wait)) |
| 4171 | return 1; |
| 4172 | |
| 4173 | if (need_resched() || walk->batched >= MAX_LRU_BATCH) { |
| 4174 | end = (addr | ~PUD_MASK) + 1; |
| 4175 | goto done; |
| 4176 | } |
| 4177 | } |
| 4178 | |
| 4179 | if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) |
| 4180 | goto restart; |
| 4181 | |
| 4182 | end = round_up(end, P4D_SIZE); |
| 4183 | done: |
| 4184 | if (!end || !args->vma) |
| 4185 | return 1; |
| 4186 | |
| 4187 | walk->next_addr = max(end, args->vma->vm_start); |
| 4188 | |
| 4189 | return -EAGAIN; |
| 4190 | } |
| 4191 | |
| 4192 | static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_mm_walk *walk) |
| 4193 | { |
| 4194 | static const struct mm_walk_ops mm_walk_ops = { |
| 4195 | .test_walk = should_skip_vma, |
| 4196 | .p4d_entry = walk_pud_range, |
| 4197 | }; |
| 4198 | |
| 4199 | int err; |
| 4200 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 4201 | |
| 4202 | walk->next_addr = FIRST_USER_ADDRESS; |
| 4203 | |
| 4204 | do { |
| 4205 | err = -EBUSY; |
| 4206 | |
| 4207 | /* folio_update_gen() requires stable folio_memcg() */ |
| 4208 | if (!mem_cgroup_trylock_pages(memcg)) |
| 4209 | break; |
| 4210 | |
| 4211 | /* the caller might be holding the lock for write */ |
| 4212 | if (mmap_read_trylock(mm)) { |
| 4213 | err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk); |
| 4214 | |
| 4215 | mmap_read_unlock(mm); |
| 4216 | } |
| 4217 | |
| 4218 | mem_cgroup_unlock_pages(); |
| 4219 | |
| 4220 | if (walk->batched) { |
| 4221 | spin_lock_irq(&lruvec->lru_lock); |
| 4222 | reset_batch_size(lruvec, walk); |
| 4223 | spin_unlock_irq(&lruvec->lru_lock); |
| 4224 | } |
| 4225 | |
| 4226 | cond_resched(); |
| 4227 | } while (err == -EAGAIN); |
| 4228 | } |
| 4229 | |
| 4230 | static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat) |
| 4231 | { |
| 4232 | struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; |
| 4233 | |
| 4234 | if (pgdat && current_is_kswapd()) { |
| 4235 | VM_WARN_ON_ONCE(walk); |
| 4236 | |
| 4237 | walk = &pgdat->mm_walk; |
| 4238 | } else if (!pgdat && !walk) { |
| 4239 | VM_WARN_ON_ONCE(current_is_kswapd()); |
| 4240 | |
| 4241 | walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); |
| 4242 | } |
| 4243 | |
| 4244 | current->reclaim_state->mm_walk = walk; |
| 4245 | |
| 4246 | return walk; |
| 4247 | } |
| 4248 | |
| 4249 | static void clear_mm_walk(void) |
| 4250 | { |
| 4251 | struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; |
| 4252 | |
| 4253 | VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages))); |
| 4254 | VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats))); |
| 4255 | |
| 4256 | current->reclaim_state->mm_walk = NULL; |
| 4257 | |
| 4258 | if (!current_is_kswapd()) |
| 4259 | kfree(walk); |
| 4260 | } |
| 4261 | |
| 4262 | static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap) |
| 4263 | { |
| 4264 | int zone; |
| 4265 | int remaining = MAX_LRU_BATCH; |
| 4266 | struct lru_gen_struct *lrugen = &lruvec->lrugen; |
| 4267 | int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); |
| 4268 | |
| 4269 | if (type == LRU_GEN_ANON && !can_swap) |
| 4270 | goto done; |
| 4271 | |
| 4272 | /* prevent cold/hot inversion if force_scan is true */ |
| 4273 | for (zone = 0; zone < MAX_NR_ZONES; zone++) { |
| 4274 | struct list_head *head = &lrugen->lists[old_gen][type][zone]; |
| 4275 | |
| 4276 | while (!list_empty(head)) { |
| 4277 | struct folio *folio = lru_to_folio(head); |
| 4278 | |
| 4279 | VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); |
| 4280 | VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); |
| 4281 | VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); |
| 4282 | VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); |
| 4283 | |
| 4284 | new_gen = folio_inc_gen(lruvec, folio, false); |
| 4285 | list_move_tail(&folio->lru, &lrugen->lists[new_gen][type][zone]); |
| 4286 | |
| 4287 | if (!--remaining) |
| 4288 | return false; |
| 4289 | } |
| 4290 | } |
| 4291 | done: |
| 4292 | reset_ctrl_pos(lruvec, type, true); |
| 4293 | WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); |
| 4294 | |
| 4295 | return true; |
| 4296 | } |
| 4297 | |
| 4298 | static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap) |
| 4299 | { |
| 4300 | int gen, type, zone; |
| 4301 | bool success = false; |
| 4302 | struct lru_gen_struct *lrugen = &lruvec->lrugen; |
| 4303 | DEFINE_MIN_SEQ(lruvec); |
| 4304 | |
| 4305 | VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); |
| 4306 | |
| 4307 | /* find the oldest populated generation */ |
| 4308 | for (type = !can_swap; type < ANON_AND_FILE; type++) { |
| 4309 | while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) { |
| 4310 | gen = lru_gen_from_seq(min_seq[type]); |
| 4311 | |
| 4312 | for (zone = 0; zone < MAX_NR_ZONES; zone++) { |
| 4313 | if (!list_empty(&lrugen->lists[gen][type][zone])) |
| 4314 | goto next; |
| 4315 | } |
| 4316 | |
| 4317 | min_seq[type]++; |
| 4318 | } |
| 4319 | next: |
| 4320 | ; |
| 4321 | } |
| 4322 | |
| 4323 | /* see the comment on lru_gen_struct */ |
| 4324 | if (can_swap) { |
| 4325 | min_seq[LRU_GEN_ANON] = min(min_seq[LRU_GEN_ANON], min_seq[LRU_GEN_FILE]); |
| 4326 | min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]); |
| 4327 | } |
| 4328 | |
| 4329 | for (type = !can_swap; type < ANON_AND_FILE; type++) { |
| 4330 | if (min_seq[type] == lrugen->min_seq[type]) |
| 4331 | continue; |
| 4332 | |
| 4333 | reset_ctrl_pos(lruvec, type, true); |
| 4334 | WRITE_ONCE(lrugen->min_seq[type], min_seq[type]); |
| 4335 | success = true; |
| 4336 | } |
| 4337 | |
| 4338 | return success; |
| 4339 | } |
| 4340 | |
| 4341 | static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan) |
| 4342 | { |
| 4343 | int prev, next; |
| 4344 | int type, zone; |
| 4345 | struct lru_gen_struct *lrugen = &lruvec->lrugen; |
| 4346 | |
| 4347 | spin_lock_irq(&lruvec->lru_lock); |
| 4348 | |
| 4349 | VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); |
| 4350 | |
| 4351 | for (type = ANON_AND_FILE - 1; type >= 0; type--) { |
| 4352 | if (get_nr_gens(lruvec, type) != MAX_NR_GENS) |
| 4353 | continue; |
| 4354 | |
| 4355 | VM_WARN_ON_ONCE(!force_scan && (type == LRU_GEN_FILE || can_swap)); |
| 4356 | |
| 4357 | while (!inc_min_seq(lruvec, type, can_swap)) { |
| 4358 | spin_unlock_irq(&lruvec->lru_lock); |
| 4359 | cond_resched(); |
| 4360 | spin_lock_irq(&lruvec->lru_lock); |
| 4361 | } |
| 4362 | } |
| 4363 | |
| 4364 | /* |
| 4365 | * Update the active/inactive LRU sizes for compatibility. Both sides of |
| 4366 | * the current max_seq need to be covered, since max_seq+1 can overlap |
| 4367 | * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do |
| 4368 | * overlap, cold/hot inversion happens. |
| 4369 | */ |
| 4370 | prev = lru_gen_from_seq(lrugen->max_seq - 1); |
| 4371 | next = lru_gen_from_seq(lrugen->max_seq + 1); |
| 4372 | |
| 4373 | for (type = 0; type < ANON_AND_FILE; type++) { |
| 4374 | for (zone = 0; zone < MAX_NR_ZONES; zone++) { |
| 4375 | enum lru_list lru = type * LRU_INACTIVE_FILE; |
| 4376 | long delta = lrugen->nr_pages[prev][type][zone] - |
| 4377 | lrugen->nr_pages[next][type][zone]; |
| 4378 | |
| 4379 | if (!delta) |
| 4380 | continue; |
| 4381 | |
| 4382 | __update_lru_size(lruvec, lru, zone, delta); |
| 4383 | __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta); |
| 4384 | } |
| 4385 | } |
| 4386 | |
| 4387 | for (type = 0; type < ANON_AND_FILE; type++) |
| 4388 | reset_ctrl_pos(lruvec, type, false); |
| 4389 | |
| 4390 | WRITE_ONCE(lrugen->timestamps[next], jiffies); |
| 4391 | /* make sure preceding modifications appear */ |
| 4392 | smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); |
| 4393 | |
| 4394 | spin_unlock_irq(&lruvec->lru_lock); |
| 4395 | } |
| 4396 | |
| 4397 | static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, |
| 4398 | struct scan_control *sc, bool can_swap, bool force_scan) |
| 4399 | { |
| 4400 | bool success; |
| 4401 | struct lru_gen_mm_walk *walk; |
| 4402 | struct mm_struct *mm = NULL; |
| 4403 | struct lru_gen_struct *lrugen = &lruvec->lrugen; |
| 4404 | |
| 4405 | VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq)); |
| 4406 | |
| 4407 | /* see the comment in iterate_mm_list() */ |
| 4408 | if (max_seq <= READ_ONCE(lruvec->mm_state.seq)) { |
| 4409 | success = false; |
| 4410 | goto done; |
| 4411 | } |
| 4412 | |
| 4413 | /* |
| 4414 | * If the hardware doesn't automatically set the accessed bit, fallback |
| 4415 | * to lru_gen_look_around(), which only clears the accessed bit in a |
| 4416 | * handful of PTEs. Spreading the work out over a period of time usually |
| 4417 | * is less efficient, but it avoids bursty page faults. |
| 4418 | */ |
| 4419 | if (!force_scan && !(arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK))) { |
| 4420 | success = iterate_mm_list_nowalk(lruvec, max_seq); |
| 4421 | goto done; |
| 4422 | } |
| 4423 | |
| 4424 | walk = set_mm_walk(NULL); |
| 4425 | if (!walk) { |
| 4426 | success = iterate_mm_list_nowalk(lruvec, max_seq); |
| 4427 | goto done; |
| 4428 | } |
| 4429 | |
| 4430 | walk->lruvec = lruvec; |
| 4431 | walk->max_seq = max_seq; |
| 4432 | walk->can_swap = can_swap; |
| 4433 | walk->force_scan = force_scan; |
| 4434 | |
| 4435 | do { |
| 4436 | success = iterate_mm_list(lruvec, walk, &mm); |
| 4437 | if (mm) |
| 4438 | walk_mm(lruvec, mm, walk); |
| 4439 | |
| 4440 | cond_resched(); |
| 4441 | } while (mm); |
| 4442 | done: |
| 4443 | if (!success) { |
| 4444 | if (sc->priority <= DEF_PRIORITY - 2) |
| 4445 | wait_event_killable(lruvec->mm_state.wait, |
| 4446 | max_seq < READ_ONCE(lrugen->max_seq)); |
| 4447 | |
| 4448 | return max_seq < READ_ONCE(lrugen->max_seq); |
| 4449 | } |
| 4450 | |
| 4451 | VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq)); |
| 4452 | |
| 4453 | inc_max_seq(lruvec, can_swap, force_scan); |
| 4454 | /* either this sees any waiters or they will see updated max_seq */ |
| 4455 | if (wq_has_sleeper(&lruvec->mm_state.wait)) |
| 4456 | wake_up_all(&lruvec->mm_state.wait); |
| 4457 | |
| 4458 | return true; |
| 4459 | } |
| 4460 | |
| 4461 | static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsigned long *min_seq, |
| 4462 | struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan) |
| 4463 | { |
| 4464 | int gen, type, zone; |
| 4465 | unsigned long old = 0; |
| 4466 | unsigned long young = 0; |
| 4467 | unsigned long total = 0; |
| 4468 | struct lru_gen_struct *lrugen = &lruvec->lrugen; |
| 4469 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 4470 | |
| 4471 | for (type = !can_swap; type < ANON_AND_FILE; type++) { |
| 4472 | unsigned long seq; |
| 4473 | |
| 4474 | for (seq = min_seq[type]; seq <= max_seq; seq++) { |
| 4475 | unsigned long size = 0; |
| 4476 | |
| 4477 | gen = lru_gen_from_seq(seq); |
| 4478 | |
| 4479 | for (zone = 0; zone < MAX_NR_ZONES; zone++) |
| 4480 | size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); |
| 4481 | |
| 4482 | total += size; |
| 4483 | if (seq == max_seq) |
| 4484 | young += size; |
| 4485 | else if (seq + MIN_NR_GENS == max_seq) |
| 4486 | old += size; |
| 4487 | } |
| 4488 | } |
| 4489 | |
| 4490 | /* try to scrape all its memory if this memcg was deleted */ |
| 4491 | *nr_to_scan = mem_cgroup_online(memcg) ? (total >> sc->priority) : total; |
| 4492 | |
| 4493 | /* |
| 4494 | * The aging tries to be lazy to reduce the overhead, while the eviction |
| 4495 | * stalls when the number of generations reaches MIN_NR_GENS. Hence, the |
| 4496 | * ideal number of generations is MIN_NR_GENS+1. |
| 4497 | */ |
| 4498 | if (min_seq[!can_swap] + MIN_NR_GENS > max_seq) |
| 4499 | return true; |
| 4500 | if (min_seq[!can_swap] + MIN_NR_GENS < max_seq) |
| 4501 | return false; |
| 4502 | |
| 4503 | /* |
| 4504 | * It's also ideal to spread pages out evenly, i.e., 1/(MIN_NR_GENS+1) |
| 4505 | * of the total number of pages for each generation. A reasonable range |
| 4506 | * for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The |
| 4507 | * aging cares about the upper bound of hot pages, while the eviction |
| 4508 | * cares about the lower bound of cold pages. |
| 4509 | */ |
| 4510 | if (young * MIN_NR_GENS > total) |
| 4511 | return true; |
| 4512 | if (old * (MIN_NR_GENS + 2) < total) |
| 4513 | return true; |
| 4514 | |
| 4515 | return false; |
| 4516 | } |
| 4517 | |
| 4518 | static bool age_lruvec(struct lruvec *lruvec, struct scan_control *sc, unsigned long min_ttl) |
| 4519 | { |
| 4520 | bool need_aging; |
| 4521 | unsigned long nr_to_scan; |
| 4522 | int swappiness = get_swappiness(lruvec, sc); |
| 4523 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 4524 | DEFINE_MAX_SEQ(lruvec); |
| 4525 | DEFINE_MIN_SEQ(lruvec); |
| 4526 | |
| 4527 | VM_WARN_ON_ONCE(sc->memcg_low_reclaim); |
| 4528 | |
| 4529 | mem_cgroup_calculate_protection(NULL, memcg); |
| 4530 | |
| 4531 | if (mem_cgroup_below_min(NULL, memcg)) |
| 4532 | return false; |
| 4533 | |
| 4534 | need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, swappiness, &nr_to_scan); |
| 4535 | |
| 4536 | if (min_ttl) { |
| 4537 | int gen = lru_gen_from_seq(min_seq[LRU_GEN_FILE]); |
| 4538 | unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); |
| 4539 | |
| 4540 | if (time_is_after_jiffies(birth + min_ttl)) |
| 4541 | return false; |
| 4542 | |
| 4543 | /* the size is likely too small to be helpful */ |
| 4544 | if (!nr_to_scan && sc->priority != DEF_PRIORITY) |
| 4545 | return false; |
| 4546 | } |
| 4547 | |
| 4548 | if (need_aging) |
| 4549 | try_to_inc_max_seq(lruvec, max_seq, sc, swappiness, false); |
| 4550 | |
| 4551 | return true; |
| 4552 | } |
| 4553 | |
| 4554 | /* to protect the working set of the last N jiffies */ |
| 4555 | static unsigned long lru_gen_min_ttl __read_mostly; |
| 4556 | |
| 4557 | static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) |
| 4558 | { |
| 4559 | struct mem_cgroup *memcg; |
| 4560 | bool success = false; |
| 4561 | unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl); |
| 4562 | |
| 4563 | VM_WARN_ON_ONCE(!current_is_kswapd()); |
| 4564 | |
| 4565 | sc->last_reclaimed = sc->nr_reclaimed; |
| 4566 | |
| 4567 | /* |
| 4568 | * To reduce the chance of going into the aging path, which can be |
| 4569 | * costly, optimistically skip it if the flag below was cleared in the |
| 4570 | * eviction path. This improves the overall performance when multiple |
| 4571 | * memcgs are available. |
| 4572 | */ |
| 4573 | if (!sc->memcgs_need_aging) { |
| 4574 | sc->memcgs_need_aging = true; |
| 4575 | return; |
| 4576 | } |
| 4577 | |
| 4578 | set_mm_walk(pgdat); |
| 4579 | |
| 4580 | memcg = mem_cgroup_iter(NULL, NULL, NULL); |
| 4581 | do { |
| 4582 | struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); |
| 4583 | |
| 4584 | if (age_lruvec(lruvec, sc, min_ttl)) |
| 4585 | success = true; |
| 4586 | |
| 4587 | cond_resched(); |
| 4588 | } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); |
| 4589 | |
| 4590 | clear_mm_walk(); |
| 4591 | |
| 4592 | /* check the order to exclude compaction-induced reclaim */ |
| 4593 | if (success || !min_ttl || sc->order) |
| 4594 | return; |
| 4595 | |
| 4596 | /* |
| 4597 | * The main goal is to OOM kill if every generation from all memcgs is |
| 4598 | * younger than min_ttl. However, another possibility is all memcgs are |
| 4599 | * either below min or empty. |
| 4600 | */ |
| 4601 | if (mutex_trylock(&oom_lock)) { |
| 4602 | struct oom_control oc = { |
| 4603 | .gfp_mask = sc->gfp_mask, |
| 4604 | }; |
| 4605 | |
| 4606 | out_of_memory(&oc); |
| 4607 | |
| 4608 | mutex_unlock(&oom_lock); |
| 4609 | } |
| 4610 | } |
| 4611 | |
| 4612 | /* |
| 4613 | * This function exploits spatial locality when shrink_folio_list() walks the |
| 4614 | * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If |
| 4615 | * the scan was done cacheline efficiently, it adds the PMD entry pointing to |
| 4616 | * the PTE table to the Bloom filter. This forms a feedback loop between the |
| 4617 | * eviction and the aging. |
| 4618 | */ |
| 4619 | void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) |
| 4620 | { |
| 4621 | int i; |
| 4622 | pte_t *pte; |
| 4623 | unsigned long start; |
| 4624 | unsigned long end; |
| 4625 | unsigned long addr; |
| 4626 | struct lru_gen_mm_walk *walk; |
| 4627 | int young = 0; |
| 4628 | unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; |
| 4629 | struct folio *folio = pfn_folio(pvmw->pfn); |
| 4630 | struct mem_cgroup *memcg = folio_memcg(folio); |
| 4631 | struct pglist_data *pgdat = folio_pgdat(folio); |
| 4632 | struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); |
| 4633 | DEFINE_MAX_SEQ(lruvec); |
| 4634 | int old_gen, new_gen = lru_gen_from_seq(max_seq); |
| 4635 | |
| 4636 | lockdep_assert_held(pvmw->ptl); |
| 4637 | VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); |
| 4638 | |
| 4639 | if (spin_is_contended(pvmw->ptl)) |
| 4640 | return; |
| 4641 | |
| 4642 | /* avoid taking the LRU lock under the PTL when possible */ |
| 4643 | walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; |
| 4644 | |
| 4645 | start = max(pvmw->address & PMD_MASK, pvmw->vma->vm_start); |
| 4646 | end = min(pvmw->address | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1; |
| 4647 | |
| 4648 | if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { |
| 4649 | if (pvmw->address - start < MIN_LRU_BATCH * PAGE_SIZE / 2) |
| 4650 | end = start + MIN_LRU_BATCH * PAGE_SIZE; |
| 4651 | else if (end - pvmw->address < MIN_LRU_BATCH * PAGE_SIZE / 2) |
| 4652 | start = end - MIN_LRU_BATCH * PAGE_SIZE; |
| 4653 | else { |
| 4654 | start = pvmw->address - MIN_LRU_BATCH * PAGE_SIZE / 2; |
| 4655 | end = pvmw->address + MIN_LRU_BATCH * PAGE_SIZE / 2; |
| 4656 | } |
| 4657 | } |
| 4658 | |
| 4659 | pte = pvmw->pte - (pvmw->address - start) / PAGE_SIZE; |
| 4660 | |
| 4661 | rcu_read_lock(); |
| 4662 | arch_enter_lazy_mmu_mode(); |
| 4663 | |
| 4664 | for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) { |
| 4665 | unsigned long pfn; |
| 4666 | |
| 4667 | pfn = get_pte_pfn(pte[i], pvmw->vma, addr); |
| 4668 | if (pfn == -1) |
| 4669 | continue; |
| 4670 | |
| 4671 | if (!pte_young(pte[i])) |
| 4672 | continue; |
| 4673 | |
| 4674 | folio = get_pfn_folio(pfn, memcg, pgdat, !walk || walk->can_swap); |
| 4675 | if (!folio) |
| 4676 | continue; |
| 4677 | |
| 4678 | if (!ptep_test_and_clear_young(pvmw->vma, addr, pte + i)) |
| 4679 | VM_WARN_ON_ONCE(true); |
| 4680 | |
| 4681 | young++; |
| 4682 | |
| 4683 | if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && |
| 4684 | !(folio_test_anon(folio) && folio_test_swapbacked(folio) && |
| 4685 | !folio_test_swapcache(folio))) |
| 4686 | folio_mark_dirty(folio); |
| 4687 | |
| 4688 | old_gen = folio_lru_gen(folio); |
| 4689 | if (old_gen < 0) |
| 4690 | folio_set_referenced(folio); |
| 4691 | else if (old_gen != new_gen) |
| 4692 | __set_bit(i, bitmap); |
| 4693 | } |
| 4694 | |
| 4695 | arch_leave_lazy_mmu_mode(); |
| 4696 | rcu_read_unlock(); |
| 4697 | |
| 4698 | /* feedback from rmap walkers to page table walkers */ |
| 4699 | if (suitable_to_scan(i, young)) |
| 4700 | update_bloom_filter(lruvec, max_seq, pvmw->pmd); |
| 4701 | |
| 4702 | if (!walk && bitmap_weight(bitmap, MIN_LRU_BATCH) < PAGEVEC_SIZE) { |
| 4703 | for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { |
| 4704 | folio = pfn_folio(pte_pfn(pte[i])); |
| 4705 | folio_activate(folio); |
| 4706 | } |
| 4707 | return; |
| 4708 | } |
| 4709 | |
| 4710 | /* folio_update_gen() requires stable folio_memcg() */ |
| 4711 | if (!mem_cgroup_trylock_pages(memcg)) |
| 4712 | return; |
| 4713 | |
| 4714 | if (!walk) { |
| 4715 | spin_lock_irq(&lruvec->lru_lock); |
| 4716 | new_gen = lru_gen_from_seq(lruvec->lrugen.max_seq); |
| 4717 | } |
| 4718 | |
| 4719 | for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { |
| 4720 | folio = pfn_folio(pte_pfn(pte[i])); |
| 4721 | if (folio_memcg_rcu(folio) != memcg) |
| 4722 | continue; |
| 4723 | |
| 4724 | old_gen = folio_update_gen(folio, new_gen); |
| 4725 | if (old_gen < 0 || old_gen == new_gen) |
| 4726 | continue; |
| 4727 | |
| 4728 | if (walk) |
| 4729 | update_batch_size(walk, folio, old_gen, new_gen); |
| 4730 | else |
| 4731 | lru_gen_update_size(lruvec, folio, old_gen, new_gen); |
| 4732 | } |
| 4733 | |
| 4734 | if (!walk) |
| 4735 | spin_unlock_irq(&lruvec->lru_lock); |
| 4736 | |
| 4737 | mem_cgroup_unlock_pages(); |
| 4738 | } |
| 4739 | |
| 4740 | /****************************************************************************** |
| 4741 | * the eviction |
| 4742 | ******************************************************************************/ |
| 4743 | |
| 4744 | static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx) |
| 4745 | { |
| 4746 | bool success; |
| 4747 | int gen = folio_lru_gen(folio); |
| 4748 | int type = folio_is_file_lru(folio); |
| 4749 | int zone = folio_zonenum(folio); |
| 4750 | int delta = folio_nr_pages(folio); |
| 4751 | int refs = folio_lru_refs(folio); |
| 4752 | int tier = lru_tier_from_refs(refs); |
| 4753 | struct lru_gen_struct *lrugen = &lruvec->lrugen; |
| 4754 | |
| 4755 | VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio); |
| 4756 | |
| 4757 | /* unevictable */ |
| 4758 | if (!folio_evictable(folio)) { |
| 4759 | success = lru_gen_del_folio(lruvec, folio, true); |
| 4760 | VM_WARN_ON_ONCE_FOLIO(!success, folio); |
| 4761 | folio_set_unevictable(folio); |
| 4762 | lruvec_add_folio(lruvec, folio); |
| 4763 | __count_vm_events(UNEVICTABLE_PGCULLED, delta); |
| 4764 | return true; |
| 4765 | } |
| 4766 | |
| 4767 | /* dirty lazyfree */ |
| 4768 | if (type == LRU_GEN_FILE && folio_test_anon(folio) && folio_test_dirty(folio)) { |
| 4769 | success = lru_gen_del_folio(lruvec, folio, true); |
| 4770 | VM_WARN_ON_ONCE_FOLIO(!success, folio); |
| 4771 | folio_set_swapbacked(folio); |
| 4772 | lruvec_add_folio_tail(lruvec, folio); |
| 4773 | return true; |
| 4774 | } |
| 4775 | |
| 4776 | /* promoted */ |
| 4777 | if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { |
| 4778 | list_move(&folio->lru, &lrugen->lists[gen][type][zone]); |
| 4779 | return true; |
| 4780 | } |
| 4781 | |
| 4782 | /* protected */ |
| 4783 | if (tier > tier_idx) { |
| 4784 | int hist = lru_hist_from_seq(lrugen->min_seq[type]); |
| 4785 | |
| 4786 | gen = folio_inc_gen(lruvec, folio, false); |
| 4787 | list_move_tail(&folio->lru, &lrugen->lists[gen][type][zone]); |
| 4788 | |
| 4789 | WRITE_ONCE(lrugen->protected[hist][type][tier - 1], |
| 4790 | lrugen->protected[hist][type][tier - 1] + delta); |
| 4791 | __mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta); |
| 4792 | return true; |
| 4793 | } |
| 4794 | |
| 4795 | /* waiting for writeback */ |
| 4796 | if (folio_test_locked(folio) || folio_test_writeback(folio) || |
| 4797 | (type == LRU_GEN_FILE && folio_test_dirty(folio))) { |
| 4798 | gen = folio_inc_gen(lruvec, folio, true); |
| 4799 | list_move(&folio->lru, &lrugen->lists[gen][type][zone]); |
| 4800 | return true; |
| 4801 | } |
| 4802 | |
| 4803 | return false; |
| 4804 | } |
| 4805 | |
| 4806 | static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc) |
| 4807 | { |
| 4808 | bool success; |
| 4809 | |
| 4810 | /* unmapping inhibited */ |
| 4811 | if (!sc->may_unmap && folio_mapped(folio)) |
| 4812 | return false; |
| 4813 | |
| 4814 | /* swapping inhibited */ |
| 4815 | if (!(sc->may_writepage && (sc->gfp_mask & __GFP_IO)) && |
| 4816 | (folio_test_dirty(folio) || |
| 4817 | (folio_test_anon(folio) && !folio_test_swapcache(folio)))) |
| 4818 | return false; |
| 4819 | |
| 4820 | /* raced with release_pages() */ |
| 4821 | if (!folio_try_get(folio)) |
| 4822 | return false; |
| 4823 | |
| 4824 | /* raced with another isolation */ |
| 4825 | if (!folio_test_clear_lru(folio)) { |
| 4826 | folio_put(folio); |
| 4827 | return false; |
| 4828 | } |
| 4829 | |
| 4830 | /* see the comment on MAX_NR_TIERS */ |
| 4831 | if (!folio_test_referenced(folio)) |
| 4832 | set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0); |
| 4833 | |
| 4834 | /* for shrink_folio_list() */ |
| 4835 | folio_clear_reclaim(folio); |
| 4836 | folio_clear_referenced(folio); |
| 4837 | |
| 4838 | success = lru_gen_del_folio(lruvec, folio, true); |
| 4839 | VM_WARN_ON_ONCE_FOLIO(!success, folio); |
| 4840 | |
| 4841 | return true; |
| 4842 | } |
| 4843 | |
| 4844 | static int scan_folios(struct lruvec *lruvec, struct scan_control *sc, |
| 4845 | int type, int tier, struct list_head *list) |
| 4846 | { |
| 4847 | int gen, zone; |
| 4848 | enum vm_event_item item; |
| 4849 | int sorted = 0; |
| 4850 | int scanned = 0; |
| 4851 | int isolated = 0; |
| 4852 | int remaining = MAX_LRU_BATCH; |
| 4853 | struct lru_gen_struct *lrugen = &lruvec->lrugen; |
| 4854 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 4855 | |
| 4856 | VM_WARN_ON_ONCE(!list_empty(list)); |
| 4857 | |
| 4858 | if (get_nr_gens(lruvec, type) == MIN_NR_GENS) |
| 4859 | return 0; |
| 4860 | |
| 4861 | gen = lru_gen_from_seq(lrugen->min_seq[type]); |
| 4862 | |
| 4863 | for (zone = sc->reclaim_idx; zone >= 0; zone--) { |
| 4864 | LIST_HEAD(moved); |
| 4865 | int skipped = 0; |
| 4866 | struct list_head *head = &lrugen->lists[gen][type][zone]; |
| 4867 | |
| 4868 | while (!list_empty(head)) { |
| 4869 | struct folio *folio = lru_to_folio(head); |
| 4870 | int delta = folio_nr_pages(folio); |
| 4871 | |
| 4872 | VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); |
| 4873 | VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); |
| 4874 | VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); |
| 4875 | VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); |
| 4876 | |
| 4877 | scanned += delta; |
| 4878 | |
| 4879 | if (sort_folio(lruvec, folio, tier)) |
| 4880 | sorted += delta; |
| 4881 | else if (isolate_folio(lruvec, folio, sc)) { |
| 4882 | list_add(&folio->lru, list); |
| 4883 | isolated += delta; |
| 4884 | } else { |
| 4885 | list_move(&folio->lru, &moved); |
| 4886 | skipped += delta; |
| 4887 | } |
| 4888 | |
| 4889 | if (!--remaining || max(isolated, skipped) >= MIN_LRU_BATCH) |
| 4890 | break; |
| 4891 | } |
| 4892 | |
| 4893 | if (skipped) { |
| 4894 | list_splice(&moved, head); |
| 4895 | __count_zid_vm_events(PGSCAN_SKIP, zone, skipped); |
| 4896 | } |
| 4897 | |
| 4898 | if (!remaining || isolated >= MIN_LRU_BATCH) |
| 4899 | break; |
| 4900 | } |
| 4901 | |
| 4902 | item = PGSCAN_KSWAPD + reclaimer_offset(); |
| 4903 | if (!cgroup_reclaim(sc)) { |
| 4904 | __count_vm_events(item, isolated); |
| 4905 | __count_vm_events(PGREFILL, sorted); |
| 4906 | } |
| 4907 | __count_memcg_events(memcg, item, isolated); |
| 4908 | __count_memcg_events(memcg, PGREFILL, sorted); |
| 4909 | __count_vm_events(PGSCAN_ANON + type, isolated); |
| 4910 | |
| 4911 | /* |
| 4912 | * There might not be eligible pages due to reclaim_idx, may_unmap and |
| 4913 | * may_writepage. Check the remaining to prevent livelock if it's not |
| 4914 | * making progress. |
| 4915 | */ |
| 4916 | return isolated || !remaining ? scanned : 0; |
| 4917 | } |
| 4918 | |
| 4919 | static int get_tier_idx(struct lruvec *lruvec, int type) |
| 4920 | { |
| 4921 | int tier; |
| 4922 | struct ctrl_pos sp, pv; |
| 4923 | |
| 4924 | /* |
| 4925 | * To leave a margin for fluctuations, use a larger gain factor (1:2). |
| 4926 | * This value is chosen because any other tier would have at least twice |
| 4927 | * as many refaults as the first tier. |
| 4928 | */ |
| 4929 | read_ctrl_pos(lruvec, type, 0, 1, &sp); |
| 4930 | for (tier = 1; tier < MAX_NR_TIERS; tier++) { |
| 4931 | read_ctrl_pos(lruvec, type, tier, 2, &pv); |
| 4932 | if (!positive_ctrl_err(&sp, &pv)) |
| 4933 | break; |
| 4934 | } |
| 4935 | |
| 4936 | return tier - 1; |
| 4937 | } |
| 4938 | |
| 4939 | static int get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_idx) |
| 4940 | { |
| 4941 | int type, tier; |
| 4942 | struct ctrl_pos sp, pv; |
| 4943 | int gain[ANON_AND_FILE] = { swappiness, 200 - swappiness }; |
| 4944 | |
| 4945 | /* |
| 4946 | * Compare the first tier of anon with that of file to determine which |
| 4947 | * type to scan. Also need to compare other tiers of the selected type |
| 4948 | * with the first tier of the other type to determine the last tier (of |
| 4949 | * the selected type) to evict. |
| 4950 | */ |
| 4951 | read_ctrl_pos(lruvec, LRU_GEN_ANON, 0, gain[LRU_GEN_ANON], &sp); |
| 4952 | read_ctrl_pos(lruvec, LRU_GEN_FILE, 0, gain[LRU_GEN_FILE], &pv); |
| 4953 | type = positive_ctrl_err(&sp, &pv); |
| 4954 | |
| 4955 | read_ctrl_pos(lruvec, !type, 0, gain[!type], &sp); |
| 4956 | for (tier = 1; tier < MAX_NR_TIERS; tier++) { |
| 4957 | read_ctrl_pos(lruvec, type, tier, gain[type], &pv); |
| 4958 | if (!positive_ctrl_err(&sp, &pv)) |
| 4959 | break; |
| 4960 | } |
| 4961 | |
| 4962 | *tier_idx = tier - 1; |
| 4963 | |
| 4964 | return type; |
| 4965 | } |
| 4966 | |
| 4967 | static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, |
| 4968 | int *type_scanned, struct list_head *list) |
| 4969 | { |
| 4970 | int i; |
| 4971 | int type; |
| 4972 | int scanned; |
| 4973 | int tier = -1; |
| 4974 | DEFINE_MIN_SEQ(lruvec); |
| 4975 | |
| 4976 | /* |
| 4977 | * Try to make the obvious choice first. When anon and file are both |
| 4978 | * available from the same generation, interpret swappiness 1 as file |
| 4979 | * first and 200 as anon first. |
| 4980 | */ |
| 4981 | if (!swappiness) |
| 4982 | type = LRU_GEN_FILE; |
| 4983 | else if (min_seq[LRU_GEN_ANON] < min_seq[LRU_GEN_FILE]) |
| 4984 | type = LRU_GEN_ANON; |
| 4985 | else if (swappiness == 1) |
| 4986 | type = LRU_GEN_FILE; |
| 4987 | else if (swappiness == 200) |
| 4988 | type = LRU_GEN_ANON; |
| 4989 | else |
| 4990 | type = get_type_to_scan(lruvec, swappiness, &tier); |
| 4991 | |
| 4992 | for (i = !swappiness; i < ANON_AND_FILE; i++) { |
| 4993 | if (tier < 0) |
| 4994 | tier = get_tier_idx(lruvec, type); |
| 4995 | |
| 4996 | scanned = scan_folios(lruvec, sc, type, tier, list); |
| 4997 | if (scanned) |
| 4998 | break; |
| 4999 | |
| 5000 | type = !type; |
| 5001 | tier = -1; |
| 5002 | } |
| 5003 | |
| 5004 | *type_scanned = type; |
| 5005 | |
| 5006 | return scanned; |
| 5007 | } |
| 5008 | |
| 5009 | static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, |
| 5010 | bool *need_swapping) |
| 5011 | { |
| 5012 | int type; |
| 5013 | int scanned; |
| 5014 | int reclaimed; |
| 5015 | LIST_HEAD(list); |
| 5016 | LIST_HEAD(clean); |
| 5017 | struct folio *folio; |
| 5018 | struct folio *next; |
| 5019 | enum vm_event_item item; |
| 5020 | struct reclaim_stat stat; |
| 5021 | struct lru_gen_mm_walk *walk; |
| 5022 | bool skip_retry = false; |
| 5023 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 5024 | struct pglist_data *pgdat = lruvec_pgdat(lruvec); |
| 5025 | |
| 5026 | spin_lock_irq(&lruvec->lru_lock); |
| 5027 | |
| 5028 | scanned = isolate_folios(lruvec, sc, swappiness, &type, &list); |
| 5029 | |
| 5030 | scanned += try_to_inc_min_seq(lruvec, swappiness); |
| 5031 | |
| 5032 | if (get_nr_gens(lruvec, !swappiness) == MIN_NR_GENS) |
| 5033 | scanned = 0; |
| 5034 | |
| 5035 | spin_unlock_irq(&lruvec->lru_lock); |
| 5036 | |
| 5037 | if (list_empty(&list)) |
| 5038 | return scanned; |
| 5039 | retry: |
| 5040 | reclaimed = shrink_folio_list(&list, pgdat, sc, &stat, false); |
| 5041 | sc->nr_reclaimed += reclaimed; |
| 5042 | |
| 5043 | list_for_each_entry_safe_reverse(folio, next, &list, lru) { |
| 5044 | if (!folio_evictable(folio)) { |
| 5045 | list_del(&folio->lru); |
| 5046 | folio_putback_lru(folio); |
| 5047 | continue; |
| 5048 | } |
| 5049 | |
| 5050 | if (folio_test_reclaim(folio) && |
| 5051 | (folio_test_dirty(folio) || folio_test_writeback(folio))) { |
| 5052 | /* restore LRU_REFS_FLAGS cleared by isolate_folio() */ |
| 5053 | if (folio_test_workingset(folio)) |
| 5054 | folio_set_referenced(folio); |
| 5055 | continue; |
| 5056 | } |
| 5057 | |
| 5058 | if (skip_retry || folio_test_active(folio) || folio_test_referenced(folio) || |
| 5059 | folio_mapped(folio) || folio_test_locked(folio) || |
| 5060 | folio_test_dirty(folio) || folio_test_writeback(folio)) { |
| 5061 | /* don't add rejected folios to the oldest generation */ |
| 5062 | set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, |
| 5063 | BIT(PG_active)); |
| 5064 | continue; |
| 5065 | } |
| 5066 | |
| 5067 | /* retry folios that may have missed folio_rotate_reclaimable() */ |
| 5068 | list_move(&folio->lru, &clean); |
| 5069 | sc->nr_scanned -= folio_nr_pages(folio); |
| 5070 | } |
| 5071 | |
| 5072 | spin_lock_irq(&lruvec->lru_lock); |
| 5073 | |
| 5074 | move_folios_to_lru(lruvec, &list); |
| 5075 | |
| 5076 | walk = current->reclaim_state->mm_walk; |
| 5077 | if (walk && walk->batched) |
| 5078 | reset_batch_size(lruvec, walk); |
| 5079 | |
| 5080 | item = PGSTEAL_KSWAPD + reclaimer_offset(); |
| 5081 | if (!cgroup_reclaim(sc)) |
| 5082 | __count_vm_events(item, reclaimed); |
| 5083 | __count_memcg_events(memcg, item, reclaimed); |
| 5084 | __count_vm_events(PGSTEAL_ANON + type, reclaimed); |
| 5085 | |
| 5086 | spin_unlock_irq(&lruvec->lru_lock); |
| 5087 | |
| 5088 | mem_cgroup_uncharge_list(&list); |
| 5089 | free_unref_page_list(&list); |
| 5090 | |
| 5091 | INIT_LIST_HEAD(&list); |
| 5092 | list_splice_init(&clean, &list); |
| 5093 | |
| 5094 | if (!list_empty(&list)) { |
| 5095 | skip_retry = true; |
| 5096 | goto retry; |
| 5097 | } |
| 5098 | |
| 5099 | if (need_swapping && type == LRU_GEN_ANON) |
| 5100 | *need_swapping = true; |
| 5101 | |
| 5102 | return scanned; |
| 5103 | } |
| 5104 | |
| 5105 | /* |
| 5106 | * For future optimizations: |
| 5107 | * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg |
| 5108 | * reclaim. |
| 5109 | */ |
| 5110 | static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, |
| 5111 | bool can_swap, bool *need_aging) |
| 5112 | { |
| 5113 | unsigned long nr_to_scan; |
| 5114 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 5115 | DEFINE_MAX_SEQ(lruvec); |
| 5116 | DEFINE_MIN_SEQ(lruvec); |
| 5117 | |
| 5118 | if (mem_cgroup_below_min(sc->target_mem_cgroup, memcg) || |
| 5119 | (mem_cgroup_below_low(sc->target_mem_cgroup, memcg) && |
| 5120 | !sc->memcg_low_reclaim)) |
| 5121 | return 0; |
| 5122 | |
| 5123 | *need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, can_swap, &nr_to_scan); |
| 5124 | if (!*need_aging) |
| 5125 | return nr_to_scan; |
| 5126 | |
| 5127 | /* skip the aging path at the default priority */ |
| 5128 | if (sc->priority == DEF_PRIORITY) |
| 5129 | goto done; |
| 5130 | |
| 5131 | /* leave the work to lru_gen_age_node() */ |
| 5132 | if (current_is_kswapd()) |
| 5133 | return 0; |
| 5134 | |
| 5135 | if (try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false)) |
| 5136 | return nr_to_scan; |
| 5137 | done: |
| 5138 | return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0; |
| 5139 | } |
| 5140 | |
| 5141 | static bool should_abort_scan(struct lruvec *lruvec, unsigned long seq, |
| 5142 | struct scan_control *sc, bool need_swapping) |
| 5143 | { |
| 5144 | int i; |
| 5145 | DEFINE_MAX_SEQ(lruvec); |
| 5146 | |
| 5147 | if (!current_is_kswapd()) { |
| 5148 | /* age each memcg at most once to ensure fairness */ |
| 5149 | if (max_seq - seq > 1) |
| 5150 | return true; |
| 5151 | |
| 5152 | /* over-swapping can increase allocation latency */ |
| 5153 | if (sc->nr_reclaimed >= sc->nr_to_reclaim && need_swapping) |
| 5154 | return true; |
| 5155 | |
| 5156 | /* give this thread a chance to exit and free its memory */ |
| 5157 | if (fatal_signal_pending(current)) { |
| 5158 | sc->nr_reclaimed += MIN_LRU_BATCH; |
| 5159 | return true; |
| 5160 | } |
| 5161 | |
| 5162 | if (cgroup_reclaim(sc)) |
| 5163 | return false; |
| 5164 | } else if (sc->nr_reclaimed - sc->last_reclaimed < sc->nr_to_reclaim) |
| 5165 | return false; |
| 5166 | |
| 5167 | /* keep scanning at low priorities to ensure fairness */ |
| 5168 | if (sc->priority > DEF_PRIORITY - 2) |
| 5169 | return false; |
| 5170 | |
| 5171 | /* |
| 5172 | * A minimum amount of work was done under global memory pressure. For |
| 5173 | * kswapd, it may be overshooting. For direct reclaim, the allocation |
| 5174 | * may succeed if all suitable zones are somewhat safe. In either case, |
| 5175 | * it's better to stop now, and restart later if necessary. |
| 5176 | */ |
| 5177 | for (i = 0; i <= sc->reclaim_idx; i++) { |
| 5178 | unsigned long wmark; |
| 5179 | struct zone *zone = lruvec_pgdat(lruvec)->node_zones + i; |
| 5180 | |
| 5181 | if (!managed_zone(zone)) |
| 5182 | continue; |
| 5183 | |
| 5184 | wmark = current_is_kswapd() ? high_wmark_pages(zone) : low_wmark_pages(zone); |
| 5185 | if (wmark > zone_page_state(zone, NR_FREE_PAGES)) |
| 5186 | return false; |
| 5187 | } |
| 5188 | |
| 5189 | sc->nr_reclaimed += MIN_LRU_BATCH; |
| 5190 | |
| 5191 | return true; |
| 5192 | } |
| 5193 | |
| 5194 | static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) |
| 5195 | { |
| 5196 | struct blk_plug plug; |
| 5197 | bool need_aging = false; |
| 5198 | bool need_swapping = false; |
| 5199 | unsigned long scanned = 0; |
| 5200 | unsigned long reclaimed = sc->nr_reclaimed; |
| 5201 | DEFINE_MAX_SEQ(lruvec); |
| 5202 | |
| 5203 | lru_add_drain(); |
| 5204 | |
| 5205 | blk_start_plug(&plug); |
| 5206 | |
| 5207 | set_mm_walk(lruvec_pgdat(lruvec)); |
| 5208 | |
| 5209 | while (true) { |
| 5210 | int delta; |
| 5211 | int swappiness; |
| 5212 | unsigned long nr_to_scan; |
| 5213 | |
| 5214 | if (sc->may_swap) |
| 5215 | swappiness = get_swappiness(lruvec, sc); |
| 5216 | else if (!cgroup_reclaim(sc) && get_swappiness(lruvec, sc)) |
| 5217 | swappiness = 1; |
| 5218 | else |
| 5219 | swappiness = 0; |
| 5220 | |
| 5221 | nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness, &need_aging); |
| 5222 | if (!nr_to_scan) |
| 5223 | goto done; |
| 5224 | |
| 5225 | delta = evict_folios(lruvec, sc, swappiness, &need_swapping); |
| 5226 | if (!delta) |
| 5227 | goto done; |
| 5228 | |
| 5229 | scanned += delta; |
| 5230 | if (scanned >= nr_to_scan) |
| 5231 | break; |
| 5232 | |
| 5233 | if (should_abort_scan(lruvec, max_seq, sc, need_swapping)) |
| 5234 | break; |
| 5235 | |
| 5236 | cond_resched(); |
| 5237 | } |
| 5238 | |
| 5239 | /* see the comment in lru_gen_age_node() */ |
| 5240 | if (sc->nr_reclaimed - reclaimed >= MIN_LRU_BATCH && !need_aging) |
| 5241 | sc->memcgs_need_aging = false; |
| 5242 | done: |
| 5243 | clear_mm_walk(); |
| 5244 | |
| 5245 | blk_finish_plug(&plug); |
| 5246 | } |
| 5247 | |
| 5248 | /****************************************************************************** |
| 5249 | * state change |
| 5250 | ******************************************************************************/ |
| 5251 | |
| 5252 | static bool __maybe_unused state_is_valid(struct lruvec *lruvec) |
| 5253 | { |
| 5254 | struct lru_gen_struct *lrugen = &lruvec->lrugen; |
| 5255 | |
| 5256 | if (lrugen->enabled) { |
| 5257 | enum lru_list lru; |
| 5258 | |
| 5259 | for_each_evictable_lru(lru) { |
| 5260 | if (!list_empty(&lruvec->lists[lru])) |
| 5261 | return false; |
| 5262 | } |
| 5263 | } else { |
| 5264 | int gen, type, zone; |
| 5265 | |
| 5266 | for_each_gen_type_zone(gen, type, zone) { |
| 5267 | if (!list_empty(&lrugen->lists[gen][type][zone])) |
| 5268 | return false; |
| 5269 | } |
| 5270 | } |
| 5271 | |
| 5272 | return true; |
| 5273 | } |
| 5274 | |
| 5275 | static bool fill_evictable(struct lruvec *lruvec) |
| 5276 | { |
| 5277 | enum lru_list lru; |
| 5278 | int remaining = MAX_LRU_BATCH; |
| 5279 | |
| 5280 | for_each_evictable_lru(lru) { |
| 5281 | int type = is_file_lru(lru); |
| 5282 | bool active = is_active_lru(lru); |
| 5283 | struct list_head *head = &lruvec->lists[lru]; |
| 5284 | |
| 5285 | while (!list_empty(head)) { |
| 5286 | bool success; |
| 5287 | struct folio *folio = lru_to_folio(head); |
| 5288 | |
| 5289 | VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); |
| 5290 | VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio); |
| 5291 | VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); |
| 5292 | VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); |
| 5293 | |
| 5294 | lruvec_del_folio(lruvec, folio); |
| 5295 | success = lru_gen_add_folio(lruvec, folio, false); |
| 5296 | VM_WARN_ON_ONCE(!success); |
| 5297 | |
| 5298 | if (!--remaining) |
| 5299 | return false; |
| 5300 | } |
| 5301 | } |
| 5302 | |
| 5303 | return true; |
| 5304 | } |
| 5305 | |
| 5306 | static bool drain_evictable(struct lruvec *lruvec) |
| 5307 | { |
| 5308 | int gen, type, zone; |
| 5309 | int remaining = MAX_LRU_BATCH; |
| 5310 | |
| 5311 | for_each_gen_type_zone(gen, type, zone) { |
| 5312 | struct list_head *head = &lruvec->lrugen.lists[gen][type][zone]; |
| 5313 | |
| 5314 | while (!list_empty(head)) { |
| 5315 | bool success; |
| 5316 | struct folio *folio = lru_to_folio(head); |
| 5317 | |
| 5318 | VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); |
| 5319 | VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); |
| 5320 | VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); |
| 5321 | VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); |
| 5322 | |
| 5323 | success = lru_gen_del_folio(lruvec, folio, false); |
| 5324 | VM_WARN_ON_ONCE(!success); |
| 5325 | lruvec_add_folio(lruvec, folio); |
| 5326 | |
| 5327 | if (!--remaining) |
| 5328 | return false; |
| 5329 | } |
| 5330 | } |
| 5331 | |
| 5332 | return true; |
| 5333 | } |
| 5334 | |
| 5335 | static void lru_gen_change_state(bool enabled) |
| 5336 | { |
| 5337 | static DEFINE_MUTEX(state_mutex); |
| 5338 | |
| 5339 | struct mem_cgroup *memcg; |
| 5340 | |
| 5341 | cgroup_lock(); |
| 5342 | cpus_read_lock(); |
| 5343 | get_online_mems(); |
| 5344 | mutex_lock(&state_mutex); |
| 5345 | |
| 5346 | if (enabled == lru_gen_enabled()) |
| 5347 | goto unlock; |
| 5348 | |
| 5349 | if (enabled) |
| 5350 | static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); |
| 5351 | else |
| 5352 | static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); |
| 5353 | |
| 5354 | memcg = mem_cgroup_iter(NULL, NULL, NULL); |
| 5355 | do { |
| 5356 | int nid; |
| 5357 | |
| 5358 | for_each_node(nid) { |
| 5359 | struct lruvec *lruvec = get_lruvec(memcg, nid); |
| 5360 | |
| 5361 | spin_lock_irq(&lruvec->lru_lock); |
| 5362 | |
| 5363 | VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); |
| 5364 | VM_WARN_ON_ONCE(!state_is_valid(lruvec)); |
| 5365 | |
| 5366 | lruvec->lrugen.enabled = enabled; |
| 5367 | |
| 5368 | while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) { |
| 5369 | spin_unlock_irq(&lruvec->lru_lock); |
| 5370 | cond_resched(); |
| 5371 | spin_lock_irq(&lruvec->lru_lock); |
| 5372 | } |
| 5373 | |
| 5374 | spin_unlock_irq(&lruvec->lru_lock); |
| 5375 | } |
| 5376 | |
| 5377 | cond_resched(); |
| 5378 | } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); |
| 5379 | unlock: |
| 5380 | mutex_unlock(&state_mutex); |
| 5381 | put_online_mems(); |
| 5382 | cpus_read_unlock(); |
| 5383 | cgroup_unlock(); |
| 5384 | } |
| 5385 | |
| 5386 | /****************************************************************************** |
| 5387 | * sysfs interface |
| 5388 | ******************************************************************************/ |
| 5389 | |
| 5390 | static ssize_t show_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, char *buf) |
| 5391 | { |
| 5392 | return sprintf(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl))); |
| 5393 | } |
| 5394 | |
| 5395 | /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ |
| 5396 | static ssize_t store_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, |
| 5397 | const char *buf, size_t len) |
| 5398 | { |
| 5399 | unsigned int msecs; |
| 5400 | |
| 5401 | if (kstrtouint(buf, 0, &msecs)) |
| 5402 | return -EINVAL; |
| 5403 | |
| 5404 | WRITE_ONCE(lru_gen_min_ttl, msecs_to_jiffies(msecs)); |
| 5405 | |
| 5406 | return len; |
| 5407 | } |
| 5408 | |
| 5409 | static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR( |
| 5410 | min_ttl_ms, 0644, show_min_ttl, store_min_ttl |
| 5411 | ); |
| 5412 | |
| 5413 | static ssize_t show_enabled(struct kobject *kobj, struct kobj_attribute *attr, char *buf) |
| 5414 | { |
| 5415 | unsigned int caps = 0; |
| 5416 | |
| 5417 | if (get_cap(LRU_GEN_CORE)) |
| 5418 | caps |= BIT(LRU_GEN_CORE); |
| 5419 | |
| 5420 | if (arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK)) |
| 5421 | caps |= BIT(LRU_GEN_MM_WALK); |
| 5422 | |
| 5423 | if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG)) |
| 5424 | caps |= BIT(LRU_GEN_NONLEAF_YOUNG); |
| 5425 | |
| 5426 | return sysfs_emit(buf, "0x%04x\n", caps); |
| 5427 | } |
| 5428 | |
| 5429 | /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ |
| 5430 | static ssize_t store_enabled(struct kobject *kobj, struct kobj_attribute *attr, |
| 5431 | const char *buf, size_t len) |
| 5432 | { |
| 5433 | int i; |
| 5434 | unsigned int caps; |
| 5435 | |
| 5436 | if (tolower(*buf) == 'n') |
| 5437 | caps = 0; |
| 5438 | else if (tolower(*buf) == 'y') |
| 5439 | caps = -1; |
| 5440 | else if (kstrtouint(buf, 0, &caps)) |
| 5441 | return -EINVAL; |
| 5442 | |
| 5443 | for (i = 0; i < NR_LRU_GEN_CAPS; i++) { |
| 5444 | bool enabled = caps & BIT(i); |
| 5445 | |
| 5446 | if (i == LRU_GEN_CORE) |
| 5447 | lru_gen_change_state(enabled); |
| 5448 | else if (enabled) |
| 5449 | static_branch_enable(&lru_gen_caps[i]); |
| 5450 | else |
| 5451 | static_branch_disable(&lru_gen_caps[i]); |
| 5452 | } |
| 5453 | |
| 5454 | return len; |
| 5455 | } |
| 5456 | |
| 5457 | static struct kobj_attribute lru_gen_enabled_attr = __ATTR( |
| 5458 | enabled, 0644, show_enabled, store_enabled |
| 5459 | ); |
| 5460 | |
| 5461 | static struct attribute *lru_gen_attrs[] = { |
| 5462 | &lru_gen_min_ttl_attr.attr, |
| 5463 | &lru_gen_enabled_attr.attr, |
| 5464 | NULL |
| 5465 | }; |
| 5466 | |
| 5467 | static struct attribute_group lru_gen_attr_group = { |
| 5468 | .name = "lru_gen", |
| 5469 | .attrs = lru_gen_attrs, |
| 5470 | }; |
| 5471 | |
| 5472 | /****************************************************************************** |
| 5473 | * debugfs interface |
| 5474 | ******************************************************************************/ |
| 5475 | |
| 5476 | static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos) |
| 5477 | { |
| 5478 | struct mem_cgroup *memcg; |
| 5479 | loff_t nr_to_skip = *pos; |
| 5480 | |
| 5481 | m->private = kvmalloc(PATH_MAX, GFP_KERNEL); |
| 5482 | if (!m->private) |
| 5483 | return ERR_PTR(-ENOMEM); |
| 5484 | |
| 5485 | memcg = mem_cgroup_iter(NULL, NULL, NULL); |
| 5486 | do { |
| 5487 | int nid; |
| 5488 | |
| 5489 | for_each_node_state(nid, N_MEMORY) { |
| 5490 | if (!nr_to_skip--) |
| 5491 | return get_lruvec(memcg, nid); |
| 5492 | } |
| 5493 | } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); |
| 5494 | |
| 5495 | return NULL; |
| 5496 | } |
| 5497 | |
| 5498 | static void lru_gen_seq_stop(struct seq_file *m, void *v) |
| 5499 | { |
| 5500 | if (!IS_ERR_OR_NULL(v)) |
| 5501 | mem_cgroup_iter_break(NULL, lruvec_memcg(v)); |
| 5502 | |
| 5503 | kvfree(m->private); |
| 5504 | m->private = NULL; |
| 5505 | } |
| 5506 | |
| 5507 | static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos) |
| 5508 | { |
| 5509 | int nid = lruvec_pgdat(v)->node_id; |
| 5510 | struct mem_cgroup *memcg = lruvec_memcg(v); |
| 5511 | |
| 5512 | ++*pos; |
| 5513 | |
| 5514 | nid = next_memory_node(nid); |
| 5515 | if (nid == MAX_NUMNODES) { |
| 5516 | memcg = mem_cgroup_iter(NULL, memcg, NULL); |
| 5517 | if (!memcg) |
| 5518 | return NULL; |
| 5519 | |
| 5520 | nid = first_memory_node; |
| 5521 | } |
| 5522 | |
| 5523 | return get_lruvec(memcg, nid); |
| 5524 | } |
| 5525 | |
| 5526 | static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, |
| 5527 | unsigned long max_seq, unsigned long *min_seq, |
| 5528 | unsigned long seq) |
| 5529 | { |
| 5530 | int i; |
| 5531 | int type, tier; |
| 5532 | int hist = lru_hist_from_seq(seq); |
| 5533 | struct lru_gen_struct *lrugen = &lruvec->lrugen; |
| 5534 | |
| 5535 | for (tier = 0; tier < MAX_NR_TIERS; tier++) { |
| 5536 | seq_printf(m, " %10d", tier); |
| 5537 | for (type = 0; type < ANON_AND_FILE; type++) { |
| 5538 | const char *s = " "; |
| 5539 | unsigned long n[3] = {}; |
| 5540 | |
| 5541 | if (seq == max_seq) { |
| 5542 | s = "RT "; |
| 5543 | n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]); |
| 5544 | n[1] = READ_ONCE(lrugen->avg_total[type][tier]); |
| 5545 | } else if (seq == min_seq[type] || NR_HIST_GENS > 1) { |
| 5546 | s = "rep"; |
| 5547 | n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]); |
| 5548 | n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]); |
| 5549 | if (tier) |
| 5550 | n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]); |
| 5551 | } |
| 5552 | |
| 5553 | for (i = 0; i < 3; i++) |
| 5554 | seq_printf(m, " %10lu%c", n[i], s[i]); |
| 5555 | } |
| 5556 | seq_putc(m, '\n'); |
| 5557 | } |
| 5558 | |
| 5559 | seq_puts(m, " "); |
| 5560 | for (i = 0; i < NR_MM_STATS; i++) { |
| 5561 | const char *s = " "; |
| 5562 | unsigned long n = 0; |
| 5563 | |
| 5564 | if (seq == max_seq && NR_HIST_GENS == 1) { |
| 5565 | s = "LOYNFA"; |
| 5566 | n = READ_ONCE(lruvec->mm_state.stats[hist][i]); |
| 5567 | } else if (seq != max_seq && NR_HIST_GENS > 1) { |
| 5568 | s = "loynfa"; |
| 5569 | n = READ_ONCE(lruvec->mm_state.stats[hist][i]); |
| 5570 | } |
| 5571 | |
| 5572 | seq_printf(m, " %10lu%c", n, s[i]); |
| 5573 | } |
| 5574 | seq_putc(m, '\n'); |
| 5575 | } |
| 5576 | |
| 5577 | /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ |
| 5578 | static int lru_gen_seq_show(struct seq_file *m, void *v) |
| 5579 | { |
| 5580 | unsigned long seq; |
| 5581 | bool full = !debugfs_real_fops(m->file)->write; |
| 5582 | struct lruvec *lruvec = v; |
| 5583 | struct lru_gen_struct *lrugen = &lruvec->lrugen; |
| 5584 | int nid = lruvec_pgdat(lruvec)->node_id; |
| 5585 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 5586 | DEFINE_MAX_SEQ(lruvec); |
| 5587 | DEFINE_MIN_SEQ(lruvec); |
| 5588 | |
| 5589 | if (nid == first_memory_node) { |
| 5590 | const char *path = memcg ? m->private : ""; |
| 5591 | |
| 5592 | #ifdef CONFIG_MEMCG |
| 5593 | if (memcg) |
| 5594 | cgroup_path(memcg->css.cgroup, m->private, PATH_MAX); |
| 5595 | #endif |
| 5596 | seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), path); |
| 5597 | } |
| 5598 | |
| 5599 | seq_printf(m, " node %5d\n", nid); |
| 5600 | |
| 5601 | if (!full) |
| 5602 | seq = min_seq[LRU_GEN_ANON]; |
| 5603 | else if (max_seq >= MAX_NR_GENS) |
| 5604 | seq = max_seq - MAX_NR_GENS + 1; |
| 5605 | else |
| 5606 | seq = 0; |
| 5607 | |
| 5608 | for (; seq <= max_seq; seq++) { |
| 5609 | int type, zone; |
| 5610 | int gen = lru_gen_from_seq(seq); |
| 5611 | unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); |
| 5612 | |
| 5613 | seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth)); |
| 5614 | |
| 5615 | for (type = 0; type < ANON_AND_FILE; type++) { |
| 5616 | unsigned long size = 0; |
| 5617 | char mark = full && seq < min_seq[type] ? 'x' : ' '; |
| 5618 | |
| 5619 | for (zone = 0; zone < MAX_NR_ZONES; zone++) |
| 5620 | size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); |
| 5621 | |
| 5622 | seq_printf(m, " %10lu%c", size, mark); |
| 5623 | } |
| 5624 | |
| 5625 | seq_putc(m, '\n'); |
| 5626 | |
| 5627 | if (full) |
| 5628 | lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq); |
| 5629 | } |
| 5630 | |
| 5631 | return 0; |
| 5632 | } |
| 5633 | |
| 5634 | static const struct seq_operations lru_gen_seq_ops = { |
| 5635 | .start = lru_gen_seq_start, |
| 5636 | .stop = lru_gen_seq_stop, |
| 5637 | .next = lru_gen_seq_next, |
| 5638 | .show = lru_gen_seq_show, |
| 5639 | }; |
| 5640 | |
| 5641 | static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, |
| 5642 | bool can_swap, bool force_scan) |
| 5643 | { |
| 5644 | DEFINE_MAX_SEQ(lruvec); |
| 5645 | DEFINE_MIN_SEQ(lruvec); |
| 5646 | |
| 5647 | if (seq < max_seq) |
| 5648 | return 0; |
| 5649 | |
| 5650 | if (seq > max_seq) |
| 5651 | return -EINVAL; |
| 5652 | |
| 5653 | if (!force_scan && min_seq[!can_swap] + MAX_NR_GENS - 1 <= max_seq) |
| 5654 | return -ERANGE; |
| 5655 | |
| 5656 | try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, force_scan); |
| 5657 | |
| 5658 | return 0; |
| 5659 | } |
| 5660 | |
| 5661 | static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, |
| 5662 | int swappiness, unsigned long nr_to_reclaim) |
| 5663 | { |
| 5664 | DEFINE_MAX_SEQ(lruvec); |
| 5665 | |
| 5666 | if (seq + MIN_NR_GENS > max_seq) |
| 5667 | return -EINVAL; |
| 5668 | |
| 5669 | sc->nr_reclaimed = 0; |
| 5670 | |
| 5671 | while (!signal_pending(current)) { |
| 5672 | DEFINE_MIN_SEQ(lruvec); |
| 5673 | |
| 5674 | if (seq < min_seq[!swappiness]) |
| 5675 | return 0; |
| 5676 | |
| 5677 | if (sc->nr_reclaimed >= nr_to_reclaim) |
| 5678 | return 0; |
| 5679 | |
| 5680 | if (!evict_folios(lruvec, sc, swappiness, NULL)) |
| 5681 | return 0; |
| 5682 | |
| 5683 | cond_resched(); |
| 5684 | } |
| 5685 | |
| 5686 | return -EINTR; |
| 5687 | } |
| 5688 | |
| 5689 | static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq, |
| 5690 | struct scan_control *sc, int swappiness, unsigned long opt) |
| 5691 | { |
| 5692 | struct lruvec *lruvec; |
| 5693 | int err = -EINVAL; |
| 5694 | struct mem_cgroup *memcg = NULL; |
| 5695 | |
| 5696 | if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY)) |
| 5697 | return -EINVAL; |
| 5698 | |
| 5699 | if (!mem_cgroup_disabled()) { |
| 5700 | rcu_read_lock(); |
| 5701 | memcg = mem_cgroup_from_id(memcg_id); |
| 5702 | #ifdef CONFIG_MEMCG |
| 5703 | if (memcg && !css_tryget(&memcg->css)) |
| 5704 | memcg = NULL; |
| 5705 | #endif |
| 5706 | rcu_read_unlock(); |
| 5707 | |
| 5708 | if (!memcg) |
| 5709 | return -EINVAL; |
| 5710 | } |
| 5711 | |
| 5712 | if (memcg_id != mem_cgroup_id(memcg)) |
| 5713 | goto done; |
| 5714 | |
| 5715 | lruvec = get_lruvec(memcg, nid); |
| 5716 | |
| 5717 | if (swappiness < 0) |
| 5718 | swappiness = get_swappiness(lruvec, sc); |
| 5719 | else if (swappiness > 200) |
| 5720 | goto done; |
| 5721 | |
| 5722 | switch (cmd) { |
| 5723 | case '+': |
| 5724 | err = run_aging(lruvec, seq, sc, swappiness, opt); |
| 5725 | break; |
| 5726 | case '-': |
| 5727 | err = run_eviction(lruvec, seq, sc, swappiness, opt); |
| 5728 | break; |
| 5729 | } |
| 5730 | done: |
| 5731 | mem_cgroup_put(memcg); |
| 5732 | |
| 5733 | return err; |
| 5734 | } |
| 5735 | |
| 5736 | /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ |
| 5737 | static ssize_t lru_gen_seq_write(struct file *file, const char __user *src, |
| 5738 | size_t len, loff_t *pos) |
| 5739 | { |
| 5740 | void *buf; |
| 5741 | char *cur, *next; |
| 5742 | unsigned int flags; |
| 5743 | struct blk_plug plug; |
| 5744 | int err = -EINVAL; |
| 5745 | struct scan_control sc = { |
| 5746 | .may_writepage = true, |
| 5747 | .may_unmap = true, |
| 5748 | .may_swap = true, |
| 5749 | .reclaim_idx = MAX_NR_ZONES - 1, |
| 5750 | .gfp_mask = GFP_KERNEL, |
| 5751 | }; |
| 5752 | |
| 5753 | buf = kvmalloc(len + 1, GFP_KERNEL); |
| 5754 | if (!buf) |
| 5755 | return -ENOMEM; |
| 5756 | |
| 5757 | if (copy_from_user(buf, src, len)) { |
| 5758 | kvfree(buf); |
| 5759 | return -EFAULT; |
| 5760 | } |
| 5761 | |
| 5762 | set_task_reclaim_state(current, &sc.reclaim_state); |
| 5763 | flags = memalloc_noreclaim_save(); |
| 5764 | blk_start_plug(&plug); |
| 5765 | if (!set_mm_walk(NULL)) { |
| 5766 | err = -ENOMEM; |
| 5767 | goto done; |
| 5768 | } |
| 5769 | |
| 5770 | next = buf; |
| 5771 | next[len] = '\0'; |
| 5772 | |
| 5773 | while ((cur = strsep(&next, ",;\n"))) { |
| 5774 | int n; |
| 5775 | int end; |
| 5776 | char cmd; |
| 5777 | unsigned int memcg_id; |
| 5778 | unsigned int nid; |
| 5779 | unsigned long seq; |
| 5780 | unsigned int swappiness = -1; |
| 5781 | unsigned long opt = -1; |
| 5782 | |
| 5783 | cur = skip_spaces(cur); |
| 5784 | if (!*cur) |
| 5785 | continue; |
| 5786 | |
| 5787 | n = sscanf(cur, "%c %u %u %lu %n %u %n %lu %n", &cmd, &memcg_id, &nid, |
| 5788 | &seq, &end, &swappiness, &end, &opt, &end); |
| 5789 | if (n < 4 || cur[end]) { |
| 5790 | err = -EINVAL; |
| 5791 | break; |
| 5792 | } |
| 5793 | |
| 5794 | err = run_cmd(cmd, memcg_id, nid, seq, &sc, swappiness, opt); |
| 5795 | if (err) |
| 5796 | break; |
| 5797 | } |
| 5798 | done: |
| 5799 | clear_mm_walk(); |
| 5800 | blk_finish_plug(&plug); |
| 5801 | memalloc_noreclaim_restore(flags); |
| 5802 | set_task_reclaim_state(current, NULL); |
| 5803 | |
| 5804 | kvfree(buf); |
| 5805 | |
| 5806 | return err ? : len; |
| 5807 | } |
| 5808 | |
| 5809 | static int lru_gen_seq_open(struct inode *inode, struct file *file) |
| 5810 | { |
| 5811 | return seq_open(file, &lru_gen_seq_ops); |
| 5812 | } |
| 5813 | |
| 5814 | static const struct file_operations lru_gen_rw_fops = { |
| 5815 | .open = lru_gen_seq_open, |
| 5816 | .read = seq_read, |
| 5817 | .write = lru_gen_seq_write, |
| 5818 | .llseek = seq_lseek, |
| 5819 | .release = seq_release, |
| 5820 | }; |
| 5821 | |
| 5822 | static const struct file_operations lru_gen_ro_fops = { |
| 5823 | .open = lru_gen_seq_open, |
| 5824 | .read = seq_read, |
| 5825 | .llseek = seq_lseek, |
| 5826 | .release = seq_release, |
| 5827 | }; |
| 5828 | |
| 5829 | /****************************************************************************** |
| 5830 | * initialization |
| 5831 | ******************************************************************************/ |
| 5832 | |
| 5833 | void lru_gen_init_lruvec(struct lruvec *lruvec) |
| 5834 | { |
| 5835 | int i; |
| 5836 | int gen, type, zone; |
| 5837 | struct lru_gen_struct *lrugen = &lruvec->lrugen; |
| 5838 | |
| 5839 | lrugen->max_seq = MIN_NR_GENS + 1; |
| 5840 | lrugen->enabled = lru_gen_enabled(); |
| 5841 | |
| 5842 | for (i = 0; i <= MIN_NR_GENS + 1; i++) |
| 5843 | lrugen->timestamps[i] = jiffies; |
| 5844 | |
| 5845 | for_each_gen_type_zone(gen, type, zone) |
| 5846 | INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]); |
| 5847 | |
| 5848 | lruvec->mm_state.seq = MIN_NR_GENS; |
| 5849 | init_waitqueue_head(&lruvec->mm_state.wait); |
| 5850 | } |
| 5851 | |
| 5852 | #ifdef CONFIG_MEMCG |
| 5853 | void lru_gen_init_memcg(struct mem_cgroup *memcg) |
| 5854 | { |
| 5855 | INIT_LIST_HEAD(&memcg->mm_list.fifo); |
| 5856 | spin_lock_init(&memcg->mm_list.lock); |
| 5857 | } |
| 5858 | |
| 5859 | void lru_gen_exit_memcg(struct mem_cgroup *memcg) |
| 5860 | { |
| 5861 | int i; |
| 5862 | int nid; |
| 5863 | |
| 5864 | for_each_node(nid) { |
| 5865 | struct lruvec *lruvec = get_lruvec(memcg, nid); |
| 5866 | |
| 5867 | VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, |
| 5868 | sizeof(lruvec->lrugen.nr_pages))); |
| 5869 | |
| 5870 | for (i = 0; i < NR_BLOOM_FILTERS; i++) { |
| 5871 | bitmap_free(lruvec->mm_state.filters[i]); |
| 5872 | lruvec->mm_state.filters[i] = NULL; |
| 5873 | } |
| 5874 | } |
| 5875 | } |
| 5876 | #endif |
| 5877 | |
| 5878 | static int __init init_lru_gen(void) |
| 5879 | { |
| 5880 | BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS); |
| 5881 | BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); |
| 5882 | |
| 5883 | if (sysfs_create_group(mm_kobj, &lru_gen_attr_group)) |
| 5884 | pr_err("lru_gen: failed to create sysfs group\n"); |
| 5885 | |
| 5886 | debugfs_create_file("lru_gen", 0644, NULL, NULL, &lru_gen_rw_fops); |
| 5887 | debugfs_create_file("lru_gen_full", 0444, NULL, NULL, &lru_gen_ro_fops); |
| 5888 | |
| 5889 | return 0; |
| 5890 | }; |
| 5891 | late_initcall(init_lru_gen); |
| 5892 | |
| 5893 | #else /* !CONFIG_LRU_GEN */ |
| 5894 | |
| 5895 | static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) |
| 5896 | { |
| 5897 | } |
| 5898 | |
| 5899 | static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) |
| 5900 | { |
| 5901 | } |
| 5902 | |
| 5903 | #endif /* CONFIG_LRU_GEN */ |
| 5904 | |
| 5905 | static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) |
| 5906 | { |
| 5907 | unsigned long nr[NR_LRU_LISTS]; |
| 5908 | unsigned long targets[NR_LRU_LISTS]; |
| 5909 | unsigned long nr_to_scan; |
| 5910 | enum lru_list lru; |
| 5911 | unsigned long nr_reclaimed = 0; |
| 5912 | unsigned long nr_to_reclaim = sc->nr_to_reclaim; |
| 5913 | bool proportional_reclaim; |
| 5914 | struct blk_plug plug; |
| 5915 | |
| 5916 | if (lru_gen_enabled()) { |
| 5917 | lru_gen_shrink_lruvec(lruvec, sc); |
| 5918 | return; |
| 5919 | } |
| 5920 | |
| 5921 | get_scan_count(lruvec, sc, nr); |
| 5922 | |
| 5923 | /* Record the original scan target for proportional adjustments later */ |
| 5924 | memcpy(targets, nr, sizeof(nr)); |
| 5925 | |
| 5926 | /* |
| 5927 | * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal |
| 5928 | * event that can occur when there is little memory pressure e.g. |
| 5929 | * multiple streaming readers/writers. Hence, we do not abort scanning |
| 5930 | * when the requested number of pages are reclaimed when scanning at |
| 5931 | * DEF_PRIORITY on the assumption that the fact we are direct |
| 5932 | * reclaiming implies that kswapd is not keeping up and it is best to |
| 5933 | * do a batch of work at once. For memcg reclaim one check is made to |
| 5934 | * abort proportional reclaim if either the file or anon lru has already |
| 5935 | * dropped to zero at the first pass. |
| 5936 | */ |
| 5937 | proportional_reclaim = (!cgroup_reclaim(sc) && !current_is_kswapd() && |
| 5938 | sc->priority == DEF_PRIORITY); |
| 5939 | |
| 5940 | blk_start_plug(&plug); |
| 5941 | while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || |
| 5942 | nr[LRU_INACTIVE_FILE]) { |
| 5943 | unsigned long nr_anon, nr_file, percentage; |
| 5944 | unsigned long nr_scanned; |
| 5945 | |
| 5946 | for_each_evictable_lru(lru) { |
| 5947 | if (nr[lru]) { |
| 5948 | nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); |
| 5949 | nr[lru] -= nr_to_scan; |
| 5950 | |
| 5951 | nr_reclaimed += shrink_list(lru, nr_to_scan, |
| 5952 | lruvec, sc); |
| 5953 | } |
| 5954 | } |
| 5955 | |
| 5956 | cond_resched(); |
| 5957 | |
| 5958 | if (nr_reclaimed < nr_to_reclaim || proportional_reclaim) |
| 5959 | continue; |
| 5960 | |
| 5961 | /* |
| 5962 | * For kswapd and memcg, reclaim at least the number of pages |
| 5963 | * requested. Ensure that the anon and file LRUs are scanned |
| 5964 | * proportionally what was requested by get_scan_count(). We |
| 5965 | * stop reclaiming one LRU and reduce the amount scanning |
| 5966 | * proportional to the original scan target. |
| 5967 | */ |
| 5968 | nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; |
| 5969 | nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; |
| 5970 | |
| 5971 | /* |
| 5972 | * It's just vindictive to attack the larger once the smaller |
| 5973 | * has gone to zero. And given the way we stop scanning the |
| 5974 | * smaller below, this makes sure that we only make one nudge |
| 5975 | * towards proportionality once we've got nr_to_reclaim. |
| 5976 | */ |
| 5977 | if (!nr_file || !nr_anon) |
| 5978 | break; |
| 5979 | |
| 5980 | if (nr_file > nr_anon) { |
| 5981 | unsigned long scan_target = targets[LRU_INACTIVE_ANON] + |
| 5982 | targets[LRU_ACTIVE_ANON] + 1; |
| 5983 | lru = LRU_BASE; |
| 5984 | percentage = nr_anon * 100 / scan_target; |
| 5985 | } else { |
| 5986 | unsigned long scan_target = targets[LRU_INACTIVE_FILE] + |
| 5987 | targets[LRU_ACTIVE_FILE] + 1; |
| 5988 | lru = LRU_FILE; |
| 5989 | percentage = nr_file * 100 / scan_target; |
| 5990 | } |
| 5991 | |
| 5992 | /* Stop scanning the smaller of the LRU */ |
| 5993 | nr[lru] = 0; |
| 5994 | nr[lru + LRU_ACTIVE] = 0; |
| 5995 | |
| 5996 | /* |
| 5997 | * Recalculate the other LRU scan count based on its original |
| 5998 | * scan target and the percentage scanning already complete |
| 5999 | */ |
| 6000 | lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; |
| 6001 | nr_scanned = targets[lru] - nr[lru]; |
| 6002 | nr[lru] = targets[lru] * (100 - percentage) / 100; |
| 6003 | nr[lru] -= min(nr[lru], nr_scanned); |
| 6004 | |
| 6005 | lru += LRU_ACTIVE; |
| 6006 | nr_scanned = targets[lru] - nr[lru]; |
| 6007 | nr[lru] = targets[lru] * (100 - percentage) / 100; |
| 6008 | nr[lru] -= min(nr[lru], nr_scanned); |
| 6009 | } |
| 6010 | blk_finish_plug(&plug); |
| 6011 | sc->nr_reclaimed += nr_reclaimed; |
| 6012 | |
| 6013 | /* |
| 6014 | * Even if we did not try to evict anon pages at all, we want to |
| 6015 | * rebalance the anon lru active/inactive ratio. |
| 6016 | */ |
| 6017 | if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) && |
| 6018 | inactive_is_low(lruvec, LRU_INACTIVE_ANON)) |
| 6019 | shrink_active_list(SWAP_CLUSTER_MAX, lruvec, |
| 6020 | sc, LRU_ACTIVE_ANON); |
| 6021 | } |
| 6022 | |
| 6023 | /* Use reclaim/compaction for costly allocs or under memory pressure */ |
| 6024 | static bool in_reclaim_compaction(struct scan_control *sc) |
| 6025 | { |
| 6026 | if (IS_ENABLED(CONFIG_COMPACTION) && sc->order && |
| 6027 | (sc->order > PAGE_ALLOC_COSTLY_ORDER || |
| 6028 | sc->priority < DEF_PRIORITY - 2)) |
| 6029 | return true; |
| 6030 | |
| 6031 | return false; |
| 6032 | } |
| 6033 | |
| 6034 | /* |
| 6035 | * Reclaim/compaction is used for high-order allocation requests. It reclaims |
| 6036 | * order-0 pages before compacting the zone. should_continue_reclaim() returns |
| 6037 | * true if more pages should be reclaimed such that when the page allocator |
| 6038 | * calls try_to_compact_pages() that it will have enough free pages to succeed. |
| 6039 | * It will give up earlier than that if there is difficulty reclaiming pages. |
| 6040 | */ |
| 6041 | static inline bool should_continue_reclaim(struct pglist_data *pgdat, |
| 6042 | unsigned long nr_reclaimed, |
| 6043 | struct scan_control *sc) |
| 6044 | { |
| 6045 | unsigned long pages_for_compaction; |
| 6046 | unsigned long inactive_lru_pages; |
| 6047 | int z; |
| 6048 | |
| 6049 | /* If not in reclaim/compaction mode, stop */ |
| 6050 | if (!in_reclaim_compaction(sc)) |
| 6051 | return false; |
| 6052 | |
| 6053 | /* |
| 6054 | * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX |
| 6055 | * number of pages that were scanned. This will return to the caller |
| 6056 | * with the risk reclaim/compaction and the resulting allocation attempt |
| 6057 | * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL |
| 6058 | * allocations through requiring that the full LRU list has been scanned |
| 6059 | * first, by assuming that zero delta of sc->nr_scanned means full LRU |
| 6060 | * scan, but that approximation was wrong, and there were corner cases |
| 6061 | * where always a non-zero amount of pages were scanned. |
| 6062 | */ |
| 6063 | if (!nr_reclaimed) |
| 6064 | return false; |
| 6065 | |
| 6066 | /* If compaction would go ahead or the allocation would succeed, stop */ |
| 6067 | for (z = 0; z <= sc->reclaim_idx; z++) { |
| 6068 | struct zone *zone = &pgdat->node_zones[z]; |
| 6069 | if (!managed_zone(zone)) |
| 6070 | continue; |
| 6071 | |
| 6072 | switch (compaction_suitable(zone, sc->order, 0, sc->reclaim_idx)) { |
| 6073 | case COMPACT_SUCCESS: |
| 6074 | case COMPACT_CONTINUE: |
| 6075 | return false; |
| 6076 | default: |
| 6077 | /* check next zone */ |
| 6078 | ; |
| 6079 | } |
| 6080 | } |
| 6081 | |
| 6082 | /* |
| 6083 | * If we have not reclaimed enough pages for compaction and the |
| 6084 | * inactive lists are large enough, continue reclaiming |
| 6085 | */ |
| 6086 | pages_for_compaction = compact_gap(sc->order); |
| 6087 | inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE); |
| 6088 | if (can_reclaim_anon_pages(NULL, pgdat->node_id, sc)) |
| 6089 | inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON); |
| 6090 | |
| 6091 | return inactive_lru_pages > pages_for_compaction; |
| 6092 | } |
| 6093 | |
| 6094 | static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc) |
| 6095 | { |
| 6096 | struct mem_cgroup *target_memcg = sc->target_mem_cgroup; |
| 6097 | struct mem_cgroup *memcg; |
| 6098 | |
| 6099 | memcg = mem_cgroup_iter(target_memcg, NULL, NULL); |
| 6100 | do { |
| 6101 | struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); |
| 6102 | unsigned long reclaimed; |
| 6103 | unsigned long scanned; |
| 6104 | |
| 6105 | /* |
| 6106 | * This loop can become CPU-bound when target memcgs |
| 6107 | * aren't eligible for reclaim - either because they |
| 6108 | * don't have any reclaimable pages, or because their |
| 6109 | * memory is explicitly protected. Avoid soft lockups. |
| 6110 | */ |
| 6111 | cond_resched(); |
| 6112 | |
| 6113 | mem_cgroup_calculate_protection(target_memcg, memcg); |
| 6114 | |
| 6115 | if (mem_cgroup_below_min(target_memcg, memcg)) { |
| 6116 | /* |
| 6117 | * Hard protection. |
| 6118 | * If there is no reclaimable memory, OOM. |
| 6119 | */ |
| 6120 | continue; |
| 6121 | } else if (mem_cgroup_below_low(target_memcg, memcg)) { |
| 6122 | /* |
| 6123 | * Soft protection. |
| 6124 | * Respect the protection only as long as |
| 6125 | * there is an unprotected supply |
| 6126 | * of reclaimable memory from other cgroups. |
| 6127 | */ |
| 6128 | if (!sc->memcg_low_reclaim) { |
| 6129 | sc->memcg_low_skipped = 1; |
| 6130 | continue; |
| 6131 | } |
| 6132 | memcg_memory_event(memcg, MEMCG_LOW); |
| 6133 | } |
| 6134 | |
| 6135 | reclaimed = sc->nr_reclaimed; |
| 6136 | scanned = sc->nr_scanned; |
| 6137 | |
| 6138 | shrink_lruvec(lruvec, sc); |
| 6139 | |
| 6140 | shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, |
| 6141 | sc->priority); |
| 6142 | |
| 6143 | /* Record the group's reclaim efficiency */ |
| 6144 | if (!sc->proactive) |
| 6145 | vmpressure(sc->gfp_mask, memcg, false, |
| 6146 | sc->nr_scanned - scanned, |
| 6147 | sc->nr_reclaimed - reclaimed); |
| 6148 | |
| 6149 | } while ((memcg = mem_cgroup_iter(target_memcg, memcg, NULL))); |
| 6150 | } |
| 6151 | |
| 6152 | static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) |
| 6153 | { |
| 6154 | struct reclaim_state *reclaim_state = current->reclaim_state; |
| 6155 | unsigned long nr_reclaimed, nr_scanned; |
| 6156 | struct lruvec *target_lruvec; |
| 6157 | bool reclaimable = false; |
| 6158 | |
| 6159 | target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); |
| 6160 | |
| 6161 | again: |
| 6162 | memset(&sc->nr, 0, sizeof(sc->nr)); |
| 6163 | |
| 6164 | nr_reclaimed = sc->nr_reclaimed; |
| 6165 | nr_scanned = sc->nr_scanned; |
| 6166 | |
| 6167 | prepare_scan_count(pgdat, sc); |
| 6168 | |
| 6169 | shrink_node_memcgs(pgdat, sc); |
| 6170 | |
| 6171 | if (reclaim_state) { |
| 6172 | sc->nr_reclaimed += reclaim_state->reclaimed_slab; |
| 6173 | reclaim_state->reclaimed_slab = 0; |
| 6174 | } |
| 6175 | |
| 6176 | /* Record the subtree's reclaim efficiency */ |
| 6177 | if (!sc->proactive) |
| 6178 | vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true, |
| 6179 | sc->nr_scanned - nr_scanned, |
| 6180 | sc->nr_reclaimed - nr_reclaimed); |
| 6181 | |
| 6182 | if (sc->nr_reclaimed - nr_reclaimed) |
| 6183 | reclaimable = true; |
| 6184 | |
| 6185 | if (current_is_kswapd()) { |
| 6186 | /* |
| 6187 | * If reclaim is isolating dirty pages under writeback, |
| 6188 | * it implies that the long-lived page allocation rate |
| 6189 | * is exceeding the page laundering rate. Either the |
| 6190 | * global limits are not being effective at throttling |
| 6191 | * processes due to the page distribution throughout |
| 6192 | * zones or there is heavy usage of a slow backing |
| 6193 | * device. The only option is to throttle from reclaim |
| 6194 | * context which is not ideal as there is no guarantee |
| 6195 | * the dirtying process is throttled in the same way |
| 6196 | * balance_dirty_pages() manages. |
| 6197 | * |
| 6198 | * Once a node is flagged PGDAT_WRITEBACK, kswapd will |
| 6199 | * count the number of pages under pages flagged for |
| 6200 | * immediate reclaim and stall if any are encountered |
| 6201 | * in the nr_immediate check below. |
| 6202 | */ |
| 6203 | if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken) |
| 6204 | set_bit(PGDAT_WRITEBACK, &pgdat->flags); |
| 6205 | |
| 6206 | /* Allow kswapd to start writing pages during reclaim.*/ |
| 6207 | if (sc->nr.unqueued_dirty == sc->nr.file_taken) |
| 6208 | set_bit(PGDAT_DIRTY, &pgdat->flags); |
| 6209 | |
| 6210 | /* |
| 6211 | * If kswapd scans pages marked for immediate |
| 6212 | * reclaim and under writeback (nr_immediate), it |
| 6213 | * implies that pages are cycling through the LRU |
| 6214 | * faster than they are written so forcibly stall |
| 6215 | * until some pages complete writeback. |
| 6216 | */ |
| 6217 | if (sc->nr.immediate) |
| 6218 | reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK); |
| 6219 | } |
| 6220 | |
| 6221 | /* |
| 6222 | * Tag a node/memcg as congested if all the dirty pages were marked |
| 6223 | * for writeback and immediate reclaim (counted in nr.congested). |
| 6224 | * |
| 6225 | * Legacy memcg will stall in page writeback so avoid forcibly |
| 6226 | * stalling in reclaim_throttle(). |
| 6227 | */ |
| 6228 | if ((current_is_kswapd() || |
| 6229 | (cgroup_reclaim(sc) && writeback_throttling_sane(sc))) && |
| 6230 | sc->nr.dirty && sc->nr.dirty == sc->nr.congested) |
| 6231 | set_bit(LRUVEC_CONGESTED, &target_lruvec->flags); |
| 6232 | |
| 6233 | /* |
| 6234 | * Stall direct reclaim for IO completions if the lruvec is |
| 6235 | * node is congested. Allow kswapd to continue until it |
| 6236 | * starts encountering unqueued dirty pages or cycling through |
| 6237 | * the LRU too quickly. |
| 6238 | */ |
| 6239 | if (!current_is_kswapd() && current_may_throttle() && |
| 6240 | !sc->hibernation_mode && |
| 6241 | test_bit(LRUVEC_CONGESTED, &target_lruvec->flags)) |
| 6242 | reclaim_throttle(pgdat, VMSCAN_THROTTLE_CONGESTED); |
| 6243 | |
| 6244 | if (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed, |
| 6245 | sc)) |
| 6246 | goto again; |
| 6247 | |
| 6248 | /* |
| 6249 | * Kswapd gives up on balancing particular nodes after too |
| 6250 | * many failures to reclaim anything from them and goes to |
| 6251 | * sleep. On reclaim progress, reset the failure counter. A |
| 6252 | * successful direct reclaim run will revive a dormant kswapd. |
| 6253 | */ |
| 6254 | if (reclaimable) |
| 6255 | pgdat->kswapd_failures = 0; |
| 6256 | } |
| 6257 | |
| 6258 | /* |
| 6259 | * Returns true if compaction should go ahead for a costly-order request, or |
| 6260 | * the allocation would already succeed without compaction. Return false if we |
| 6261 | * should reclaim first. |
| 6262 | */ |
| 6263 | static inline bool compaction_ready(struct zone *zone, struct scan_control *sc) |
| 6264 | { |
| 6265 | unsigned long watermark; |
| 6266 | enum compact_result suitable; |
| 6267 | |
| 6268 | suitable = compaction_suitable(zone, sc->order, 0, sc->reclaim_idx); |
| 6269 | if (suitable == COMPACT_SUCCESS) |
| 6270 | /* Allocation should succeed already. Don't reclaim. */ |
| 6271 | return true; |
| 6272 | if (suitable == COMPACT_SKIPPED) |
| 6273 | /* Compaction cannot yet proceed. Do reclaim. */ |
| 6274 | return false; |
| 6275 | |
| 6276 | /* |
| 6277 | * Compaction is already possible, but it takes time to run and there |
| 6278 | * are potentially other callers using the pages just freed. So proceed |
| 6279 | * with reclaim to make a buffer of free pages available to give |
| 6280 | * compaction a reasonable chance of completing and allocating the page. |
| 6281 | * Note that we won't actually reclaim the whole buffer in one attempt |
| 6282 | * as the target watermark in should_continue_reclaim() is lower. But if |
| 6283 | * we are already above the high+gap watermark, don't reclaim at all. |
| 6284 | */ |
| 6285 | watermark = high_wmark_pages(zone) + compact_gap(sc->order); |
| 6286 | |
| 6287 | return zone_watermark_ok_safe(zone, 0, watermark, sc->reclaim_idx); |
| 6288 | } |
| 6289 | |
| 6290 | static void consider_reclaim_throttle(pg_data_t *pgdat, struct scan_control *sc) |
| 6291 | { |
| 6292 | /* |
| 6293 | * If reclaim is making progress greater than 12% efficiency then |
| 6294 | * wake all the NOPROGRESS throttled tasks. |
| 6295 | */ |
| 6296 | if (sc->nr_reclaimed > (sc->nr_scanned >> 3)) { |
| 6297 | wait_queue_head_t *wqh; |
| 6298 | |
| 6299 | wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_NOPROGRESS]; |
| 6300 | if (waitqueue_active(wqh)) |
| 6301 | wake_up(wqh); |
| 6302 | |
| 6303 | return; |
| 6304 | } |
| 6305 | |
| 6306 | /* |
| 6307 | * Do not throttle kswapd or cgroup reclaim on NOPROGRESS as it will |
| 6308 | * throttle on VMSCAN_THROTTLE_WRITEBACK if there are too many pages |
| 6309 | * under writeback and marked for immediate reclaim at the tail of the |
| 6310 | * LRU. |
| 6311 | */ |
| 6312 | if (current_is_kswapd() || cgroup_reclaim(sc)) |
| 6313 | return; |
| 6314 | |
| 6315 | /* Throttle if making no progress at high prioities. */ |
| 6316 | if (sc->priority == 1 && !sc->nr_reclaimed) |
| 6317 | reclaim_throttle(pgdat, VMSCAN_THROTTLE_NOPROGRESS); |
| 6318 | } |
| 6319 | |
| 6320 | /* |
| 6321 | * This is the direct reclaim path, for page-allocating processes. We only |
| 6322 | * try to reclaim pages from zones which will satisfy the caller's allocation |
| 6323 | * request. |
| 6324 | * |
| 6325 | * If a zone is deemed to be full of pinned pages then just give it a light |
| 6326 | * scan then give up on it. |
| 6327 | */ |
| 6328 | static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc) |
| 6329 | { |
| 6330 | struct zoneref *z; |
| 6331 | struct zone *zone; |
| 6332 | unsigned long nr_soft_reclaimed; |
| 6333 | unsigned long nr_soft_scanned; |
| 6334 | gfp_t orig_mask; |
| 6335 | pg_data_t *last_pgdat = NULL; |
| 6336 | pg_data_t *first_pgdat = NULL; |
| 6337 | |
| 6338 | /* |
| 6339 | * If the number of buffer_heads in the machine exceeds the maximum |
| 6340 | * allowed level, force direct reclaim to scan the highmem zone as |
| 6341 | * highmem pages could be pinning lowmem pages storing buffer_heads |
| 6342 | */ |
| 6343 | orig_mask = sc->gfp_mask; |
| 6344 | if (buffer_heads_over_limit) { |
| 6345 | sc->gfp_mask |= __GFP_HIGHMEM; |
| 6346 | sc->reclaim_idx = gfp_zone(sc->gfp_mask); |
| 6347 | } |
| 6348 | |
| 6349 | for_each_zone_zonelist_nodemask(zone, z, zonelist, |
| 6350 | sc->reclaim_idx, sc->nodemask) { |
| 6351 | /* |
| 6352 | * Take care memory controller reclaiming has small influence |
| 6353 | * to global LRU. |
| 6354 | */ |
| 6355 | if (!cgroup_reclaim(sc)) { |
| 6356 | if (!cpuset_zone_allowed(zone, |
| 6357 | GFP_KERNEL | __GFP_HARDWALL)) |
| 6358 | continue; |
| 6359 | |
| 6360 | /* |
| 6361 | * If we already have plenty of memory free for |
| 6362 | * compaction in this zone, don't free any more. |
| 6363 | * Even though compaction is invoked for any |
| 6364 | * non-zero order, only frequent costly order |
| 6365 | * reclamation is disruptive enough to become a |
| 6366 | * noticeable problem, like transparent huge |
| 6367 | * page allocations. |
| 6368 | */ |
| 6369 | if (IS_ENABLED(CONFIG_COMPACTION) && |
| 6370 | sc->order > PAGE_ALLOC_COSTLY_ORDER && |
| 6371 | compaction_ready(zone, sc)) { |
| 6372 | sc->compaction_ready = true; |
| 6373 | continue; |
| 6374 | } |
| 6375 | |
| 6376 | /* |
| 6377 | * Shrink each node in the zonelist once. If the |
| 6378 | * zonelist is ordered by zone (not the default) then a |
| 6379 | * node may be shrunk multiple times but in that case |
| 6380 | * the user prefers lower zones being preserved. |
| 6381 | */ |
| 6382 | if (zone->zone_pgdat == last_pgdat) |
| 6383 | continue; |
| 6384 | |
| 6385 | /* |
| 6386 | * This steals pages from memory cgroups over softlimit |
| 6387 | * and returns the number of reclaimed pages and |
| 6388 | * scanned pages. This works for global memory pressure |
| 6389 | * and balancing, not for a memcg's limit. |
| 6390 | */ |
| 6391 | nr_soft_scanned = 0; |
| 6392 | nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone->zone_pgdat, |
| 6393 | sc->order, sc->gfp_mask, |
| 6394 | &nr_soft_scanned); |
| 6395 | sc->nr_reclaimed += nr_soft_reclaimed; |
| 6396 | sc->nr_scanned += nr_soft_scanned; |
| 6397 | /* need some check for avoid more shrink_zone() */ |
| 6398 | } |
| 6399 | |
| 6400 | if (!first_pgdat) |
| 6401 | first_pgdat = zone->zone_pgdat; |
| 6402 | |
| 6403 | /* See comment about same check for global reclaim above */ |
| 6404 | if (zone->zone_pgdat == last_pgdat) |
| 6405 | continue; |
| 6406 | last_pgdat = zone->zone_pgdat; |
| 6407 | shrink_node(zone->zone_pgdat, sc); |
| 6408 | } |
| 6409 | |
| 6410 | if (first_pgdat) |
| 6411 | consider_reclaim_throttle(first_pgdat, sc); |
| 6412 | |
| 6413 | /* |
| 6414 | * Restore to original mask to avoid the impact on the caller if we |
| 6415 | * promoted it to __GFP_HIGHMEM. |
| 6416 | */ |
| 6417 | sc->gfp_mask = orig_mask; |
| 6418 | } |
| 6419 | |
| 6420 | static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat) |
| 6421 | { |
| 6422 | struct lruvec *target_lruvec; |
| 6423 | unsigned long refaults; |
| 6424 | |
| 6425 | if (lru_gen_enabled()) |
| 6426 | return; |
| 6427 | |
| 6428 | target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat); |
| 6429 | refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON); |
| 6430 | target_lruvec->refaults[WORKINGSET_ANON] = refaults; |
| 6431 | refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE); |
| 6432 | target_lruvec->refaults[WORKINGSET_FILE] = refaults; |
| 6433 | } |
| 6434 | |
| 6435 | /* |
| 6436 | * This is the main entry point to direct page reclaim. |
| 6437 | * |
| 6438 | * If a full scan of the inactive list fails to free enough memory then we |
| 6439 | * are "out of memory" and something needs to be killed. |
| 6440 | * |
| 6441 | * If the caller is !__GFP_FS then the probability of a failure is reasonably |
| 6442 | * high - the zone may be full of dirty or under-writeback pages, which this |
| 6443 | * caller can't do much about. We kick the writeback threads and take explicit |
| 6444 | * naps in the hope that some of these pages can be written. But if the |
| 6445 | * allocating task holds filesystem locks which prevent writeout this might not |
| 6446 | * work, and the allocation attempt will fail. |
| 6447 | * |
| 6448 | * returns: 0, if no pages reclaimed |
| 6449 | * else, the number of pages reclaimed |
| 6450 | */ |
| 6451 | static unsigned long do_try_to_free_pages(struct zonelist *zonelist, |
| 6452 | struct scan_control *sc) |
| 6453 | { |
| 6454 | int initial_priority = sc->priority; |
| 6455 | pg_data_t *last_pgdat; |
| 6456 | struct zoneref *z; |
| 6457 | struct zone *zone; |
| 6458 | retry: |
| 6459 | delayacct_freepages_start(); |
| 6460 | |
| 6461 | if (!cgroup_reclaim(sc)) |
| 6462 | __count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1); |
| 6463 | |
| 6464 | do { |
| 6465 | if (!sc->proactive) |
| 6466 | vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup, |
| 6467 | sc->priority); |
| 6468 | sc->nr_scanned = 0; |
| 6469 | shrink_zones(zonelist, sc); |
| 6470 | |
| 6471 | if (sc->nr_reclaimed >= sc->nr_to_reclaim) |
| 6472 | break; |
| 6473 | |
| 6474 | if (sc->compaction_ready) |
| 6475 | break; |
| 6476 | |
| 6477 | /* |
| 6478 | * If we're getting trouble reclaiming, start doing |
| 6479 | * writepage even in laptop mode. |
| 6480 | */ |
| 6481 | if (sc->priority < DEF_PRIORITY - 2) |
| 6482 | sc->may_writepage = 1; |
| 6483 | } while (--sc->priority >= 0); |
| 6484 | |
| 6485 | last_pgdat = NULL; |
| 6486 | for_each_zone_zonelist_nodemask(zone, z, zonelist, sc->reclaim_idx, |
| 6487 | sc->nodemask) { |
| 6488 | if (zone->zone_pgdat == last_pgdat) |
| 6489 | continue; |
| 6490 | last_pgdat = zone->zone_pgdat; |
| 6491 | |
| 6492 | snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat); |
| 6493 | |
| 6494 | if (cgroup_reclaim(sc)) { |
| 6495 | struct lruvec *lruvec; |
| 6496 | |
| 6497 | lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, |
| 6498 | zone->zone_pgdat); |
| 6499 | clear_bit(LRUVEC_CONGESTED, &lruvec->flags); |
| 6500 | } |
| 6501 | } |
| 6502 | |
| 6503 | delayacct_freepages_end(); |
| 6504 | |
| 6505 | if (sc->nr_reclaimed) |
| 6506 | return sc->nr_reclaimed; |
| 6507 | |
| 6508 | /* Aborted reclaim to try compaction? don't OOM, then */ |
| 6509 | if (sc->compaction_ready) |
| 6510 | return 1; |
| 6511 | |
| 6512 | /* |
| 6513 | * We make inactive:active ratio decisions based on the node's |
| 6514 | * composition of memory, but a restrictive reclaim_idx or a |
| 6515 | * memory.low cgroup setting can exempt large amounts of |
| 6516 | * memory from reclaim. Neither of which are very common, so |
| 6517 | * instead of doing costly eligibility calculations of the |
| 6518 | * entire cgroup subtree up front, we assume the estimates are |
| 6519 | * good, and retry with forcible deactivation if that fails. |
| 6520 | */ |
| 6521 | if (sc->skipped_deactivate) { |
| 6522 | sc->priority = initial_priority; |
| 6523 | sc->force_deactivate = 1; |
| 6524 | sc->skipped_deactivate = 0; |
| 6525 | goto retry; |
| 6526 | } |
| 6527 | |
| 6528 | /* Untapped cgroup reserves? Don't OOM, retry. */ |
| 6529 | if (sc->memcg_low_skipped) { |
| 6530 | sc->priority = initial_priority; |
| 6531 | sc->force_deactivate = 0; |
| 6532 | sc->memcg_low_reclaim = 1; |
| 6533 | sc->memcg_low_skipped = 0; |
| 6534 | goto retry; |
| 6535 | } |
| 6536 | |
| 6537 | return 0; |
| 6538 | } |
| 6539 | |
| 6540 | static bool allow_direct_reclaim(pg_data_t *pgdat) |
| 6541 | { |
| 6542 | struct zone *zone; |
| 6543 | unsigned long pfmemalloc_reserve = 0; |
| 6544 | unsigned long free_pages = 0; |
| 6545 | int i; |
| 6546 | bool wmark_ok; |
| 6547 | |
| 6548 | if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES) |
| 6549 | return true; |
| 6550 | |
| 6551 | for (i = 0; i <= ZONE_NORMAL; i++) { |
| 6552 | zone = &pgdat->node_zones[i]; |
| 6553 | if (!managed_zone(zone)) |
| 6554 | continue; |
| 6555 | |
| 6556 | if (!zone_reclaimable_pages(zone)) |
| 6557 | continue; |
| 6558 | |
| 6559 | pfmemalloc_reserve += min_wmark_pages(zone); |
| 6560 | free_pages += zone_page_state(zone, NR_FREE_PAGES); |
| 6561 | } |
| 6562 | |
| 6563 | /* If there are no reserves (unexpected config) then do not throttle */ |
| 6564 | if (!pfmemalloc_reserve) |
| 6565 | return true; |
| 6566 | |
| 6567 | wmark_ok = free_pages > pfmemalloc_reserve / 2; |
| 6568 | |
| 6569 | /* kswapd must be awake if processes are being throttled */ |
| 6570 | if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) { |
| 6571 | if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL) |
| 6572 | WRITE_ONCE(pgdat->kswapd_highest_zoneidx, ZONE_NORMAL); |
| 6573 | |
| 6574 | wake_up_interruptible(&pgdat->kswapd_wait); |
| 6575 | } |
| 6576 | |
| 6577 | return wmark_ok; |
| 6578 | } |
| 6579 | |
| 6580 | /* |
| 6581 | * Throttle direct reclaimers if backing storage is backed by the network |
| 6582 | * and the PFMEMALLOC reserve for the preferred node is getting dangerously |
| 6583 | * depleted. kswapd will continue to make progress and wake the processes |
| 6584 | * when the low watermark is reached. |
| 6585 | * |
| 6586 | * Returns true if a fatal signal was delivered during throttling. If this |
| 6587 | * happens, the page allocator should not consider triggering the OOM killer. |
| 6588 | */ |
| 6589 | static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist, |
| 6590 | nodemask_t *nodemask) |
| 6591 | { |
| 6592 | struct zoneref *z; |
| 6593 | struct zone *zone; |
| 6594 | pg_data_t *pgdat = NULL; |
| 6595 | |
| 6596 | /* |
| 6597 | * Kernel threads should not be throttled as they may be indirectly |
| 6598 | * responsible for cleaning pages necessary for reclaim to make forward |
| 6599 | * progress. kjournald for example may enter direct reclaim while |
| 6600 | * committing a transaction where throttling it could forcing other |
| 6601 | * processes to block on log_wait_commit(). |
| 6602 | */ |
| 6603 | if (current->flags & PF_KTHREAD) |
| 6604 | goto out; |
| 6605 | |
| 6606 | /* |
| 6607 | * If a fatal signal is pending, this process should not throttle. |
| 6608 | * It should return quickly so it can exit and free its memory |
| 6609 | */ |
| 6610 | if (fatal_signal_pending(current)) |
| 6611 | goto out; |
| 6612 | |
| 6613 | /* |
| 6614 | * Check if the pfmemalloc reserves are ok by finding the first node |
| 6615 | * with a usable ZONE_NORMAL or lower zone. The expectation is that |
| 6616 | * GFP_KERNEL will be required for allocating network buffers when |
| 6617 | * swapping over the network so ZONE_HIGHMEM is unusable. |
| 6618 | * |
| 6619 | * Throttling is based on the first usable node and throttled processes |
| 6620 | * wait on a queue until kswapd makes progress and wakes them. There |
| 6621 | * is an affinity then between processes waking up and where reclaim |
| 6622 | * progress has been made assuming the process wakes on the same node. |
| 6623 | * More importantly, processes running on remote nodes will not compete |
| 6624 | * for remote pfmemalloc reserves and processes on different nodes |
| 6625 | * should make reasonable progress. |
| 6626 | */ |
| 6627 | for_each_zone_zonelist_nodemask(zone, z, zonelist, |
| 6628 | gfp_zone(gfp_mask), nodemask) { |
| 6629 | if (zone_idx(zone) > ZONE_NORMAL) |
| 6630 | continue; |
| 6631 | |
| 6632 | /* Throttle based on the first usable node */ |
| 6633 | pgdat = zone->zone_pgdat; |
| 6634 | if (allow_direct_reclaim(pgdat)) |
| 6635 | goto out; |
| 6636 | break; |
| 6637 | } |
| 6638 | |
| 6639 | /* If no zone was usable by the allocation flags then do not throttle */ |
| 6640 | if (!pgdat) |
| 6641 | goto out; |
| 6642 | |
| 6643 | /* Account for the throttling */ |
| 6644 | count_vm_event(PGSCAN_DIRECT_THROTTLE); |
| 6645 | |
| 6646 | /* |
| 6647 | * If the caller cannot enter the filesystem, it's possible that it |
| 6648 | * is due to the caller holding an FS lock or performing a journal |
| 6649 | * transaction in the case of a filesystem like ext[3|4]. In this case, |
| 6650 | * it is not safe to block on pfmemalloc_wait as kswapd could be |
| 6651 | * blocked waiting on the same lock. Instead, throttle for up to a |
| 6652 | * second before continuing. |
| 6653 | */ |
| 6654 | if (!(gfp_mask & __GFP_FS)) |
| 6655 | wait_event_interruptible_timeout(pgdat->pfmemalloc_wait, |
| 6656 | allow_direct_reclaim(pgdat), HZ); |
| 6657 | else |
| 6658 | /* Throttle until kswapd wakes the process */ |
| 6659 | wait_event_killable(zone->zone_pgdat->pfmemalloc_wait, |
| 6660 | allow_direct_reclaim(pgdat)); |
| 6661 | |
| 6662 | if (fatal_signal_pending(current)) |
| 6663 | return true; |
| 6664 | |
| 6665 | out: |
| 6666 | return false; |
| 6667 | } |
| 6668 | |
| 6669 | unsigned long try_to_free_pages(struct zonelist *zonelist, int order, |
| 6670 | gfp_t gfp_mask, nodemask_t *nodemask) |
| 6671 | { |
| 6672 | unsigned long nr_reclaimed; |
| 6673 | struct scan_control sc = { |
| 6674 | .nr_to_reclaim = SWAP_CLUSTER_MAX, |
| 6675 | .gfp_mask = current_gfp_context(gfp_mask), |
| 6676 | .reclaim_idx = gfp_zone(gfp_mask), |
| 6677 | .order = order, |
| 6678 | .nodemask = nodemask, |
| 6679 | .priority = DEF_PRIORITY, |
| 6680 | .may_writepage = !laptop_mode, |
| 6681 | .may_unmap = 1, |
| 6682 | .may_swap = 1, |
| 6683 | }; |
| 6684 | |
| 6685 | /* |
| 6686 | * scan_control uses s8 fields for order, priority, and reclaim_idx. |
| 6687 | * Confirm they are large enough for max values. |
| 6688 | */ |
| 6689 | BUILD_BUG_ON(MAX_ORDER > S8_MAX); |
| 6690 | BUILD_BUG_ON(DEF_PRIORITY > S8_MAX); |
| 6691 | BUILD_BUG_ON(MAX_NR_ZONES > S8_MAX); |
| 6692 | |
| 6693 | /* |
| 6694 | * Do not enter reclaim if fatal signal was delivered while throttled. |
| 6695 | * 1 is returned so that the page allocator does not OOM kill at this |
| 6696 | * point. |
| 6697 | */ |
| 6698 | if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask)) |
| 6699 | return 1; |
| 6700 | |
| 6701 | set_task_reclaim_state(current, &sc.reclaim_state); |
| 6702 | trace_mm_vmscan_direct_reclaim_begin(order, sc.gfp_mask); |
| 6703 | |
| 6704 | nr_reclaimed = do_try_to_free_pages(zonelist, &sc); |
| 6705 | |
| 6706 | trace_mm_vmscan_direct_reclaim_end(nr_reclaimed); |
| 6707 | set_task_reclaim_state(current, NULL); |
| 6708 | |
| 6709 | return nr_reclaimed; |
| 6710 | } |
| 6711 | |
| 6712 | #ifdef CONFIG_MEMCG |
| 6713 | |
| 6714 | /* Only used by soft limit reclaim. Do not reuse for anything else. */ |
| 6715 | unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, |
| 6716 | gfp_t gfp_mask, bool noswap, |
| 6717 | pg_data_t *pgdat, |
| 6718 | unsigned long *nr_scanned) |
| 6719 | { |
| 6720 | struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); |
| 6721 | struct scan_control sc = { |
| 6722 | .nr_to_reclaim = SWAP_CLUSTER_MAX, |
| 6723 | .target_mem_cgroup = memcg, |
| 6724 | .may_writepage = !laptop_mode, |
| 6725 | .may_unmap = 1, |
| 6726 | .reclaim_idx = MAX_NR_ZONES - 1, |
| 6727 | .may_swap = !noswap, |
| 6728 | }; |
| 6729 | |
| 6730 | WARN_ON_ONCE(!current->reclaim_state); |
| 6731 | |
| 6732 | sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) | |
| 6733 | (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK); |
| 6734 | |
| 6735 | trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order, |
| 6736 | sc.gfp_mask); |
| 6737 | |
| 6738 | /* |
| 6739 | * NOTE: Although we can get the priority field, using it |
| 6740 | * here is not a good idea, since it limits the pages we can scan. |
| 6741 | * if we don't reclaim here, the shrink_node from balance_pgdat |
| 6742 | * will pick up pages from other mem cgroup's as well. We hack |
| 6743 | * the priority and make it zero. |
| 6744 | */ |
| 6745 | shrink_lruvec(lruvec, &sc); |
| 6746 | |
| 6747 | trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed); |
| 6748 | |
| 6749 | *nr_scanned = sc.nr_scanned; |
| 6750 | |
| 6751 | return sc.nr_reclaimed; |
| 6752 | } |
| 6753 | |
| 6754 | unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, |
| 6755 | unsigned long nr_pages, |
| 6756 | gfp_t gfp_mask, |
| 6757 | unsigned int reclaim_options, |
| 6758 | nodemask_t *nodemask) |
| 6759 | { |
| 6760 | unsigned long nr_reclaimed; |
| 6761 | unsigned int noreclaim_flag; |
| 6762 | struct scan_control sc = { |
| 6763 | .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), |
| 6764 | .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) | |
| 6765 | (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK), |
| 6766 | .reclaim_idx = MAX_NR_ZONES - 1, |
| 6767 | .target_mem_cgroup = memcg, |
| 6768 | .priority = DEF_PRIORITY, |
| 6769 | .may_writepage = !laptop_mode, |
| 6770 | .may_unmap = 1, |
| 6771 | .may_swap = !!(reclaim_options & MEMCG_RECLAIM_MAY_SWAP), |
| 6772 | .proactive = !!(reclaim_options & MEMCG_RECLAIM_PROACTIVE), |
| 6773 | .nodemask = nodemask, |
| 6774 | }; |
| 6775 | /* |
| 6776 | * Traverse the ZONELIST_FALLBACK zonelist of the current node to put |
| 6777 | * equal pressure on all the nodes. This is based on the assumption that |
| 6778 | * the reclaim does not bail out early. |
| 6779 | */ |
| 6780 | struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); |
| 6781 | |
| 6782 | set_task_reclaim_state(current, &sc.reclaim_state); |
| 6783 | trace_mm_vmscan_memcg_reclaim_begin(0, sc.gfp_mask); |
| 6784 | noreclaim_flag = memalloc_noreclaim_save(); |
| 6785 | |
| 6786 | nr_reclaimed = do_try_to_free_pages(zonelist, &sc); |
| 6787 | |
| 6788 | memalloc_noreclaim_restore(noreclaim_flag); |
| 6789 | trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed); |
| 6790 | set_task_reclaim_state(current, NULL); |
| 6791 | |
| 6792 | return nr_reclaimed; |
| 6793 | } |
| 6794 | #endif |
| 6795 | |
| 6796 | static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc) |
| 6797 | { |
| 6798 | struct mem_cgroup *memcg; |
| 6799 | struct lruvec *lruvec; |
| 6800 | |
| 6801 | if (lru_gen_enabled()) { |
| 6802 | lru_gen_age_node(pgdat, sc); |
| 6803 | return; |
| 6804 | } |
| 6805 | |
| 6806 | if (!can_age_anon_pages(pgdat, sc)) |
| 6807 | return; |
| 6808 | |
| 6809 | lruvec = mem_cgroup_lruvec(NULL, pgdat); |
| 6810 | if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON)) |
| 6811 | return; |
| 6812 | |
| 6813 | memcg = mem_cgroup_iter(NULL, NULL, NULL); |
| 6814 | do { |
| 6815 | lruvec = mem_cgroup_lruvec(memcg, pgdat); |
| 6816 | shrink_active_list(SWAP_CLUSTER_MAX, lruvec, |
| 6817 | sc, LRU_ACTIVE_ANON); |
| 6818 | memcg = mem_cgroup_iter(NULL, memcg, NULL); |
| 6819 | } while (memcg); |
| 6820 | } |
| 6821 | |
| 6822 | static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx) |
| 6823 | { |
| 6824 | int i; |
| 6825 | struct zone *zone; |
| 6826 | |
| 6827 | /* |
| 6828 | * Check for watermark boosts top-down as the higher zones |
| 6829 | * are more likely to be boosted. Both watermarks and boosts |
| 6830 | * should not be checked at the same time as reclaim would |
| 6831 | * start prematurely when there is no boosting and a lower |
| 6832 | * zone is balanced. |
| 6833 | */ |
| 6834 | for (i = highest_zoneidx; i >= 0; i--) { |
| 6835 | zone = pgdat->node_zones + i; |
| 6836 | if (!managed_zone(zone)) |
| 6837 | continue; |
| 6838 | |
| 6839 | if (zone->watermark_boost) |
| 6840 | return true; |
| 6841 | } |
| 6842 | |
| 6843 | return false; |
| 6844 | } |
| 6845 | |
| 6846 | /* |
| 6847 | * Returns true if there is an eligible zone balanced for the request order |
| 6848 | * and highest_zoneidx |
| 6849 | */ |
| 6850 | static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx) |
| 6851 | { |
| 6852 | int i; |
| 6853 | unsigned long mark = -1; |
| 6854 | struct zone *zone; |
| 6855 | |
| 6856 | /* |
| 6857 | * Check watermarks bottom-up as lower zones are more likely to |
| 6858 | * meet watermarks. |
| 6859 | */ |
| 6860 | for (i = 0; i <= highest_zoneidx; i++) { |
| 6861 | zone = pgdat->node_zones + i; |
| 6862 | |
| 6863 | if (!managed_zone(zone)) |
| 6864 | continue; |
| 6865 | |
| 6866 | if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) |
| 6867 | mark = wmark_pages(zone, WMARK_PROMO); |
| 6868 | else |
| 6869 | mark = high_wmark_pages(zone); |
| 6870 | if (zone_watermark_ok_safe(zone, order, mark, highest_zoneidx)) |
| 6871 | return true; |
| 6872 | } |
| 6873 | |
| 6874 | /* |
| 6875 | * If a node has no managed zone within highest_zoneidx, it does not |
| 6876 | * need balancing by definition. This can happen if a zone-restricted |
| 6877 | * allocation tries to wake a remote kswapd. |
| 6878 | */ |
| 6879 | if (mark == -1) |
| 6880 | return true; |
| 6881 | |
| 6882 | return false; |
| 6883 | } |
| 6884 | |
| 6885 | /* Clear pgdat state for congested, dirty or under writeback. */ |
| 6886 | static void clear_pgdat_congested(pg_data_t *pgdat) |
| 6887 | { |
| 6888 | struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat); |
| 6889 | |
| 6890 | clear_bit(LRUVEC_CONGESTED, &lruvec->flags); |
| 6891 | clear_bit(PGDAT_DIRTY, &pgdat->flags); |
| 6892 | clear_bit(PGDAT_WRITEBACK, &pgdat->flags); |
| 6893 | } |
| 6894 | |
| 6895 | /* |
| 6896 | * Prepare kswapd for sleeping. This verifies that there are no processes |
| 6897 | * waiting in throttle_direct_reclaim() and that watermarks have been met. |
| 6898 | * |
| 6899 | * Returns true if kswapd is ready to sleep |
| 6900 | */ |
| 6901 | static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, |
| 6902 | int highest_zoneidx) |
| 6903 | { |
| 6904 | /* |
| 6905 | * The throttled processes are normally woken up in balance_pgdat() as |
| 6906 | * soon as allow_direct_reclaim() is true. But there is a potential |
| 6907 | * race between when kswapd checks the watermarks and a process gets |
| 6908 | * throttled. There is also a potential race if processes get |
| 6909 | * throttled, kswapd wakes, a large process exits thereby balancing the |
| 6910 | * zones, which causes kswapd to exit balance_pgdat() before reaching |
| 6911 | * the wake up checks. If kswapd is going to sleep, no process should |
| 6912 | * be sleeping on pfmemalloc_wait, so wake them now if necessary. If |
| 6913 | * the wake up is premature, processes will wake kswapd and get |
| 6914 | * throttled again. The difference from wake ups in balance_pgdat() is |
| 6915 | * that here we are under prepare_to_wait(). |
| 6916 | */ |
| 6917 | if (waitqueue_active(&pgdat->pfmemalloc_wait)) |
| 6918 | wake_up_all(&pgdat->pfmemalloc_wait); |
| 6919 | |
| 6920 | /* Hopeless node, leave it to direct reclaim */ |
| 6921 | if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES) |
| 6922 | return true; |
| 6923 | |
| 6924 | if (pgdat_balanced(pgdat, order, highest_zoneidx)) { |
| 6925 | clear_pgdat_congested(pgdat); |
| 6926 | return true; |
| 6927 | } |
| 6928 | |
| 6929 | return false; |
| 6930 | } |
| 6931 | |
| 6932 | /* |
| 6933 | * kswapd shrinks a node of pages that are at or below the highest usable |
| 6934 | * zone that is currently unbalanced. |
| 6935 | * |
| 6936 | * Returns true if kswapd scanned at least the requested number of pages to |
| 6937 | * reclaim or if the lack of progress was due to pages under writeback. |
| 6938 | * This is used to determine if the scanning priority needs to be raised. |
| 6939 | */ |
| 6940 | static bool kswapd_shrink_node(pg_data_t *pgdat, |
| 6941 | struct scan_control *sc) |
| 6942 | { |
| 6943 | struct zone *zone; |
| 6944 | int z; |
| 6945 | |
| 6946 | /* Reclaim a number of pages proportional to the number of zones */ |
| 6947 | sc->nr_to_reclaim = 0; |
| 6948 | for (z = 0; z <= sc->reclaim_idx; z++) { |
| 6949 | zone = pgdat->node_zones + z; |
| 6950 | if (!managed_zone(zone)) |
| 6951 | continue; |
| 6952 | |
| 6953 | sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX); |
| 6954 | } |
| 6955 | |
| 6956 | /* |
| 6957 | * Historically care was taken to put equal pressure on all zones but |
| 6958 | * now pressure is applied based on node LRU order. |
| 6959 | */ |
| 6960 | shrink_node(pgdat, sc); |
| 6961 | |
| 6962 | /* |
| 6963 | * Fragmentation may mean that the system cannot be rebalanced for |
| 6964 | * high-order allocations. If twice the allocation size has been |
| 6965 | * reclaimed then recheck watermarks only at order-0 to prevent |
| 6966 | * excessive reclaim. Assume that a process requested a high-order |
| 6967 | * can direct reclaim/compact. |
| 6968 | */ |
| 6969 | if (sc->order && sc->nr_reclaimed >= compact_gap(sc->order)) |
| 6970 | sc->order = 0; |
| 6971 | |
| 6972 | return sc->nr_scanned >= sc->nr_to_reclaim; |
| 6973 | } |
| 6974 | |
| 6975 | /* Page allocator PCP high watermark is lowered if reclaim is active. */ |
| 6976 | static inline void |
| 6977 | update_reclaim_active(pg_data_t *pgdat, int highest_zoneidx, bool active) |
| 6978 | { |
| 6979 | int i; |
| 6980 | struct zone *zone; |
| 6981 | |
| 6982 | for (i = 0; i <= highest_zoneidx; i++) { |
| 6983 | zone = pgdat->node_zones + i; |
| 6984 | |
| 6985 | if (!managed_zone(zone)) |
| 6986 | continue; |
| 6987 | |
| 6988 | if (active) |
| 6989 | set_bit(ZONE_RECLAIM_ACTIVE, &zone->flags); |
| 6990 | else |
| 6991 | clear_bit(ZONE_RECLAIM_ACTIVE, &zone->flags); |
| 6992 | } |
| 6993 | } |
| 6994 | |
| 6995 | static inline void |
| 6996 | set_reclaim_active(pg_data_t *pgdat, int highest_zoneidx) |
| 6997 | { |
| 6998 | update_reclaim_active(pgdat, highest_zoneidx, true); |
| 6999 | } |
| 7000 | |
| 7001 | static inline void |
| 7002 | clear_reclaim_active(pg_data_t *pgdat, int highest_zoneidx) |
| 7003 | { |
| 7004 | update_reclaim_active(pgdat, highest_zoneidx, false); |
| 7005 | } |
| 7006 | |
| 7007 | /* |
| 7008 | * For kswapd, balance_pgdat() will reclaim pages across a node from zones |
| 7009 | * that are eligible for use by the caller until at least one zone is |
| 7010 | * balanced. |
| 7011 | * |
| 7012 | * Returns the order kswapd finished reclaiming at. |
| 7013 | * |
| 7014 | * kswapd scans the zones in the highmem->normal->dma direction. It skips |
| 7015 | * zones which have free_pages > high_wmark_pages(zone), but once a zone is |
| 7016 | * found to have free_pages <= high_wmark_pages(zone), any page in that zone |
| 7017 | * or lower is eligible for reclaim until at least one usable zone is |
| 7018 | * balanced. |
| 7019 | */ |
| 7020 | static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx) |
| 7021 | { |
| 7022 | int i; |
| 7023 | unsigned long nr_soft_reclaimed; |
| 7024 | unsigned long nr_soft_scanned; |
| 7025 | unsigned long pflags; |
| 7026 | unsigned long nr_boost_reclaim; |
| 7027 | unsigned long zone_boosts[MAX_NR_ZONES] = { 0, }; |
| 7028 | bool boosted; |
| 7029 | struct zone *zone; |
| 7030 | struct scan_control sc = { |
| 7031 | .gfp_mask = GFP_KERNEL, |
| 7032 | .order = order, |
| 7033 | .may_unmap = 1, |
| 7034 | }; |
| 7035 | |
| 7036 | set_task_reclaim_state(current, &sc.reclaim_state); |
| 7037 | psi_memstall_enter(&pflags); |
| 7038 | __fs_reclaim_acquire(_THIS_IP_); |
| 7039 | |
| 7040 | count_vm_event(PAGEOUTRUN); |
| 7041 | |
| 7042 | /* |
| 7043 | * Account for the reclaim boost. Note that the zone boost is left in |
| 7044 | * place so that parallel allocations that are near the watermark will |
| 7045 | * stall or direct reclaim until kswapd is finished. |
| 7046 | */ |
| 7047 | nr_boost_reclaim = 0; |
| 7048 | for (i = 0; i <= highest_zoneidx; i++) { |
| 7049 | zone = pgdat->node_zones + i; |
| 7050 | if (!managed_zone(zone)) |
| 7051 | continue; |
| 7052 | |
| 7053 | nr_boost_reclaim += zone->watermark_boost; |
| 7054 | zone_boosts[i] = zone->watermark_boost; |
| 7055 | } |
| 7056 | boosted = nr_boost_reclaim; |
| 7057 | |
| 7058 | restart: |
| 7059 | set_reclaim_active(pgdat, highest_zoneidx); |
| 7060 | sc.priority = DEF_PRIORITY; |
| 7061 | do { |
| 7062 | unsigned long nr_reclaimed = sc.nr_reclaimed; |
| 7063 | bool raise_priority = true; |
| 7064 | bool balanced; |
| 7065 | bool ret; |
| 7066 | |
| 7067 | sc.reclaim_idx = highest_zoneidx; |
| 7068 | |
| 7069 | /* |
| 7070 | * If the number of buffer_heads exceeds the maximum allowed |
| 7071 | * then consider reclaiming from all zones. This has a dual |
| 7072 | * purpose -- on 64-bit systems it is expected that |
| 7073 | * buffer_heads are stripped during active rotation. On 32-bit |
| 7074 | * systems, highmem pages can pin lowmem memory and shrinking |
| 7075 | * buffers can relieve lowmem pressure. Reclaim may still not |
| 7076 | * go ahead if all eligible zones for the original allocation |
| 7077 | * request are balanced to avoid excessive reclaim from kswapd. |
| 7078 | */ |
| 7079 | if (buffer_heads_over_limit) { |
| 7080 | for (i = MAX_NR_ZONES - 1; i >= 0; i--) { |
| 7081 | zone = pgdat->node_zones + i; |
| 7082 | if (!managed_zone(zone)) |
| 7083 | continue; |
| 7084 | |
| 7085 | sc.reclaim_idx = i; |
| 7086 | break; |
| 7087 | } |
| 7088 | } |
| 7089 | |
| 7090 | /* |
| 7091 | * If the pgdat is imbalanced then ignore boosting and preserve |
| 7092 | * the watermarks for a later time and restart. Note that the |
| 7093 | * zone watermarks will be still reset at the end of balancing |
| 7094 | * on the grounds that the normal reclaim should be enough to |
| 7095 | * re-evaluate if boosting is required when kswapd next wakes. |
| 7096 | */ |
| 7097 | balanced = pgdat_balanced(pgdat, sc.order, highest_zoneidx); |
| 7098 | if (!balanced && nr_boost_reclaim) { |
| 7099 | nr_boost_reclaim = 0; |
| 7100 | goto restart; |
| 7101 | } |
| 7102 | |
| 7103 | /* |
| 7104 | * If boosting is not active then only reclaim if there are no |
| 7105 | * eligible zones. Note that sc.reclaim_idx is not used as |
| 7106 | * buffer_heads_over_limit may have adjusted it. |
| 7107 | */ |
| 7108 | if (!nr_boost_reclaim && balanced) |
| 7109 | goto out; |
| 7110 | |
| 7111 | /* Limit the priority of boosting to avoid reclaim writeback */ |
| 7112 | if (nr_boost_reclaim && sc.priority == DEF_PRIORITY - 2) |
| 7113 | raise_priority = false; |
| 7114 | |
| 7115 | /* |
| 7116 | * Do not writeback or swap pages for boosted reclaim. The |
| 7117 | * intent is to relieve pressure not issue sub-optimal IO |
| 7118 | * from reclaim context. If no pages are reclaimed, the |
| 7119 | * reclaim will be aborted. |
| 7120 | */ |
| 7121 | sc.may_writepage = !laptop_mode && !nr_boost_reclaim; |
| 7122 | sc.may_swap = !nr_boost_reclaim; |
| 7123 | |
| 7124 | /* |
| 7125 | * Do some background aging, to give pages a chance to be |
| 7126 | * referenced before reclaiming. All pages are rotated |
| 7127 | * regardless of classzone as this is about consistent aging. |
| 7128 | */ |
| 7129 | kswapd_age_node(pgdat, &sc); |
| 7130 | |
| 7131 | /* |
| 7132 | * If we're getting trouble reclaiming, start doing writepage |
| 7133 | * even in laptop mode. |
| 7134 | */ |
| 7135 | if (sc.priority < DEF_PRIORITY - 2) |
| 7136 | sc.may_writepage = 1; |
| 7137 | |
| 7138 | /* Call soft limit reclaim before calling shrink_node. */ |
| 7139 | sc.nr_scanned = 0; |
| 7140 | nr_soft_scanned = 0; |
| 7141 | nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(pgdat, sc.order, |
| 7142 | sc.gfp_mask, &nr_soft_scanned); |
| 7143 | sc.nr_reclaimed += nr_soft_reclaimed; |
| 7144 | |
| 7145 | /* |
| 7146 | * There should be no need to raise the scanning priority if |
| 7147 | * enough pages are already being scanned that that high |
| 7148 | * watermark would be met at 100% efficiency. |
| 7149 | */ |
| 7150 | if (kswapd_shrink_node(pgdat, &sc)) |
| 7151 | raise_priority = false; |
| 7152 | |
| 7153 | /* |
| 7154 | * If the low watermark is met there is no need for processes |
| 7155 | * to be throttled on pfmemalloc_wait as they should not be |
| 7156 | * able to safely make forward progress. Wake them |
| 7157 | */ |
| 7158 | if (waitqueue_active(&pgdat->pfmemalloc_wait) && |
| 7159 | allow_direct_reclaim(pgdat)) |
| 7160 | wake_up_all(&pgdat->pfmemalloc_wait); |
| 7161 | |
| 7162 | /* Check if kswapd should be suspending */ |
| 7163 | __fs_reclaim_release(_THIS_IP_); |
| 7164 | ret = try_to_freeze(); |
| 7165 | __fs_reclaim_acquire(_THIS_IP_); |
| 7166 | if (ret || kthread_should_stop()) |
| 7167 | break; |
| 7168 | |
| 7169 | /* |
| 7170 | * Raise priority if scanning rate is too low or there was no |
| 7171 | * progress in reclaiming pages |
| 7172 | */ |
| 7173 | nr_reclaimed = sc.nr_reclaimed - nr_reclaimed; |
| 7174 | nr_boost_reclaim -= min(nr_boost_reclaim, nr_reclaimed); |
| 7175 | |
| 7176 | /* |
| 7177 | * If reclaim made no progress for a boost, stop reclaim as |
| 7178 | * IO cannot be queued and it could be an infinite loop in |
| 7179 | * extreme circumstances. |
| 7180 | */ |
| 7181 | if (nr_boost_reclaim && !nr_reclaimed) |
| 7182 | break; |
| 7183 | |
| 7184 | if (raise_priority || !nr_reclaimed) |
| 7185 | sc.priority--; |
| 7186 | } while (sc.priority >= 1); |
| 7187 | |
| 7188 | if (!sc.nr_reclaimed) |
| 7189 | pgdat->kswapd_failures++; |
| 7190 | |
| 7191 | out: |
| 7192 | clear_reclaim_active(pgdat, highest_zoneidx); |
| 7193 | |
| 7194 | /* If reclaim was boosted, account for the reclaim done in this pass */ |
| 7195 | if (boosted) { |
| 7196 | unsigned long flags; |
| 7197 | |
| 7198 | for (i = 0; i <= highest_zoneidx; i++) { |
| 7199 | if (!zone_boosts[i]) |
| 7200 | continue; |
| 7201 | |
| 7202 | /* Increments are under the zone lock */ |
| 7203 | zone = pgdat->node_zones + i; |
| 7204 | spin_lock_irqsave(&zone->lock, flags); |
| 7205 | zone->watermark_boost -= min(zone->watermark_boost, zone_boosts[i]); |
| 7206 | spin_unlock_irqrestore(&zone->lock, flags); |
| 7207 | } |
| 7208 | |
| 7209 | /* |
| 7210 | * As there is now likely space, wakeup kcompact to defragment |
| 7211 | * pageblocks. |
| 7212 | */ |
| 7213 | wakeup_kcompactd(pgdat, pageblock_order, highest_zoneidx); |
| 7214 | } |
| 7215 | |
| 7216 | snapshot_refaults(NULL, pgdat); |
| 7217 | __fs_reclaim_release(_THIS_IP_); |
| 7218 | psi_memstall_leave(&pflags); |
| 7219 | set_task_reclaim_state(current, NULL); |
| 7220 | |
| 7221 | /* |
| 7222 | * Return the order kswapd stopped reclaiming at as |
| 7223 | * prepare_kswapd_sleep() takes it into account. If another caller |
| 7224 | * entered the allocator slow path while kswapd was awake, order will |
| 7225 | * remain at the higher level. |
| 7226 | */ |
| 7227 | return sc.order; |
| 7228 | } |
| 7229 | |
| 7230 | /* |
| 7231 | * The pgdat->kswapd_highest_zoneidx is used to pass the highest zone index to |
| 7232 | * be reclaimed by kswapd from the waker. If the value is MAX_NR_ZONES which is |
| 7233 | * not a valid index then either kswapd runs for first time or kswapd couldn't |
| 7234 | * sleep after previous reclaim attempt (node is still unbalanced). In that |
| 7235 | * case return the zone index of the previous kswapd reclaim cycle. |
| 7236 | */ |
| 7237 | static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat, |
| 7238 | enum zone_type prev_highest_zoneidx) |
| 7239 | { |
| 7240 | enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx); |
| 7241 | |
| 7242 | return curr_idx == MAX_NR_ZONES ? prev_highest_zoneidx : curr_idx; |
| 7243 | } |
| 7244 | |
| 7245 | static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order, |
| 7246 | unsigned int highest_zoneidx) |
| 7247 | { |
| 7248 | long remaining = 0; |
| 7249 | DEFINE_WAIT(wait); |
| 7250 | |
| 7251 | if (freezing(current) || kthread_should_stop()) |
| 7252 | return; |
| 7253 | |
| 7254 | prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); |
| 7255 | |
| 7256 | /* |
| 7257 | * Try to sleep for a short interval. Note that kcompactd will only be |
| 7258 | * woken if it is possible to sleep for a short interval. This is |
| 7259 | * deliberate on the assumption that if reclaim cannot keep an |
| 7260 | * eligible zone balanced that it's also unlikely that compaction will |
| 7261 | * succeed. |
| 7262 | */ |
| 7263 | if (prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) { |
| 7264 | /* |
| 7265 | * Compaction records what page blocks it recently failed to |
| 7266 | * isolate pages from and skips them in the future scanning. |
| 7267 | * When kswapd is going to sleep, it is reasonable to assume |
| 7268 | * that pages and compaction may succeed so reset the cache. |
| 7269 | */ |
| 7270 | reset_isolation_suitable(pgdat); |
| 7271 | |
| 7272 | /* |
| 7273 | * We have freed the memory, now we should compact it to make |
| 7274 | * allocation of the requested order possible. |
| 7275 | */ |
| 7276 | wakeup_kcompactd(pgdat, alloc_order, highest_zoneidx); |
| 7277 | |
| 7278 | remaining = schedule_timeout(HZ/10); |
| 7279 | |
| 7280 | /* |
| 7281 | * If woken prematurely then reset kswapd_highest_zoneidx and |
| 7282 | * order. The values will either be from a wakeup request or |
| 7283 | * the previous request that slept prematurely. |
| 7284 | */ |
| 7285 | if (remaining) { |
| 7286 | WRITE_ONCE(pgdat->kswapd_highest_zoneidx, |
| 7287 | kswapd_highest_zoneidx(pgdat, |
| 7288 | highest_zoneidx)); |
| 7289 | |
| 7290 | if (READ_ONCE(pgdat->kswapd_order) < reclaim_order) |
| 7291 | WRITE_ONCE(pgdat->kswapd_order, reclaim_order); |
| 7292 | } |
| 7293 | |
| 7294 | finish_wait(&pgdat->kswapd_wait, &wait); |
| 7295 | prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); |
| 7296 | } |
| 7297 | |
| 7298 | /* |
| 7299 | * After a short sleep, check if it was a premature sleep. If not, then |
| 7300 | * go fully to sleep until explicitly woken up. |
| 7301 | */ |
| 7302 | if (!remaining && |
| 7303 | prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) { |
| 7304 | trace_mm_vmscan_kswapd_sleep(pgdat->node_id); |
| 7305 | |
| 7306 | /* |
| 7307 | * vmstat counters are not perfectly accurate and the estimated |
| 7308 | * value for counters such as NR_FREE_PAGES can deviate from the |
| 7309 | * true value by nr_online_cpus * threshold. To avoid the zone |
| 7310 | * watermarks being breached while under pressure, we reduce the |
| 7311 | * per-cpu vmstat threshold while kswapd is awake and restore |
| 7312 | * them before going back to sleep. |
| 7313 | */ |
| 7314 | set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold); |
| 7315 | |
| 7316 | if (!kthread_should_stop()) |
| 7317 | schedule(); |
| 7318 | |
| 7319 | set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold); |
| 7320 | } else { |
| 7321 | if (remaining) |
| 7322 | count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY); |
| 7323 | else |
| 7324 | count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY); |
| 7325 | } |
| 7326 | finish_wait(&pgdat->kswapd_wait, &wait); |
| 7327 | } |
| 7328 | |
| 7329 | /* |
| 7330 | * The background pageout daemon, started as a kernel thread |
| 7331 | * from the init process. |
| 7332 | * |
| 7333 | * This basically trickles out pages so that we have _some_ |
| 7334 | * free memory available even if there is no other activity |
| 7335 | * that frees anything up. This is needed for things like routing |
| 7336 | * etc, where we otherwise might have all activity going on in |
| 7337 | * asynchronous contexts that cannot page things out. |
| 7338 | * |
| 7339 | * If there are applications that are active memory-allocators |
| 7340 | * (most normal use), this basically shouldn't matter. |
| 7341 | */ |
| 7342 | static int kswapd(void *p) |
| 7343 | { |
| 7344 | unsigned int alloc_order, reclaim_order; |
| 7345 | unsigned int highest_zoneidx = MAX_NR_ZONES - 1; |
| 7346 | pg_data_t *pgdat = (pg_data_t *)p; |
| 7347 | struct task_struct *tsk = current; |
| 7348 | const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); |
| 7349 | |
| 7350 | if (!cpumask_empty(cpumask)) |
| 7351 | set_cpus_allowed_ptr(tsk, cpumask); |
| 7352 | |
| 7353 | /* |
| 7354 | * Tell the memory management that we're a "memory allocator", |
| 7355 | * and that if we need more memory we should get access to it |
| 7356 | * regardless (see "__alloc_pages()"). "kswapd" should |
| 7357 | * never get caught in the normal page freeing logic. |
| 7358 | * |
| 7359 | * (Kswapd normally doesn't need memory anyway, but sometimes |
| 7360 | * you need a small amount of memory in order to be able to |
| 7361 | * page out something else, and this flag essentially protects |
| 7362 | * us from recursively trying to free more memory as we're |
| 7363 | * trying to free the first piece of memory in the first place). |
| 7364 | */ |
| 7365 | tsk->flags |= PF_MEMALLOC | PF_KSWAPD; |
| 7366 | set_freezable(); |
| 7367 | |
| 7368 | WRITE_ONCE(pgdat->kswapd_order, 0); |
| 7369 | WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES); |
| 7370 | atomic_set(&pgdat->nr_writeback_throttled, 0); |
| 7371 | for ( ; ; ) { |
| 7372 | bool ret; |
| 7373 | |
| 7374 | alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order); |
| 7375 | highest_zoneidx = kswapd_highest_zoneidx(pgdat, |
| 7376 | highest_zoneidx); |
| 7377 | |
| 7378 | kswapd_try_sleep: |
| 7379 | kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order, |
| 7380 | highest_zoneidx); |
| 7381 | |
| 7382 | /* Read the new order and highest_zoneidx */ |
| 7383 | alloc_order = READ_ONCE(pgdat->kswapd_order); |
| 7384 | highest_zoneidx = kswapd_highest_zoneidx(pgdat, |
| 7385 | highest_zoneidx); |
| 7386 | WRITE_ONCE(pgdat->kswapd_order, 0); |
| 7387 | WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES); |
| 7388 | |
| 7389 | ret = try_to_freeze(); |
| 7390 | if (kthread_should_stop()) |
| 7391 | break; |
| 7392 | |
| 7393 | /* |
| 7394 | * We can speed up thawing tasks if we don't call balance_pgdat |
| 7395 | * after returning from the refrigerator |
| 7396 | */ |
| 7397 | if (ret) |
| 7398 | continue; |
| 7399 | |
| 7400 | /* |
| 7401 | * Reclaim begins at the requested order but if a high-order |
| 7402 | * reclaim fails then kswapd falls back to reclaiming for |
| 7403 | * order-0. If that happens, kswapd will consider sleeping |
| 7404 | * for the order it finished reclaiming at (reclaim_order) |
| 7405 | * but kcompactd is woken to compact for the original |
| 7406 | * request (alloc_order). |
| 7407 | */ |
| 7408 | trace_mm_vmscan_kswapd_wake(pgdat->node_id, highest_zoneidx, |
| 7409 | alloc_order); |
| 7410 | reclaim_order = balance_pgdat(pgdat, alloc_order, |
| 7411 | highest_zoneidx); |
| 7412 | if (reclaim_order < alloc_order) |
| 7413 | goto kswapd_try_sleep; |
| 7414 | } |
| 7415 | |
| 7416 | tsk->flags &= ~(PF_MEMALLOC | PF_KSWAPD); |
| 7417 | |
| 7418 | return 0; |
| 7419 | } |
| 7420 | |
| 7421 | /* |
| 7422 | * A zone is low on free memory or too fragmented for high-order memory. If |
| 7423 | * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's |
| 7424 | * pgdat. It will wake up kcompactd after reclaiming memory. If kswapd reclaim |
| 7425 | * has failed or is not needed, still wake up kcompactd if only compaction is |
| 7426 | * needed. |
| 7427 | */ |
| 7428 | void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order, |
| 7429 | enum zone_type highest_zoneidx) |
| 7430 | { |
| 7431 | pg_data_t *pgdat; |
| 7432 | enum zone_type curr_idx; |
| 7433 | |
| 7434 | if (!managed_zone(zone)) |
| 7435 | return; |
| 7436 | |
| 7437 | if (!cpuset_zone_allowed(zone, gfp_flags)) |
| 7438 | return; |
| 7439 | |
| 7440 | pgdat = zone->zone_pgdat; |
| 7441 | curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx); |
| 7442 | |
| 7443 | if (curr_idx == MAX_NR_ZONES || curr_idx < highest_zoneidx) |
| 7444 | WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx); |
| 7445 | |
| 7446 | if (READ_ONCE(pgdat->kswapd_order) < order) |
| 7447 | WRITE_ONCE(pgdat->kswapd_order, order); |
| 7448 | |
| 7449 | if (!waitqueue_active(&pgdat->kswapd_wait)) |
| 7450 | return; |
| 7451 | |
| 7452 | /* Hopeless node, leave it to direct reclaim if possible */ |
| 7453 | if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES || |
| 7454 | (pgdat_balanced(pgdat, order, highest_zoneidx) && |
| 7455 | !pgdat_watermark_boosted(pgdat, highest_zoneidx))) { |
| 7456 | /* |
| 7457 | * There may be plenty of free memory available, but it's too |
| 7458 | * fragmented for high-order allocations. Wake up kcompactd |
| 7459 | * and rely on compaction_suitable() to determine if it's |
| 7460 | * needed. If it fails, it will defer subsequent attempts to |
| 7461 | * ratelimit its work. |
| 7462 | */ |
| 7463 | if (!(gfp_flags & __GFP_DIRECT_RECLAIM)) |
| 7464 | wakeup_kcompactd(pgdat, order, highest_zoneidx); |
| 7465 | return; |
| 7466 | } |
| 7467 | |
| 7468 | trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, highest_zoneidx, order, |
| 7469 | gfp_flags); |
| 7470 | wake_up_interruptible(&pgdat->kswapd_wait); |
| 7471 | } |
| 7472 | |
| 7473 | #ifdef CONFIG_HIBERNATION |
| 7474 | /* |
| 7475 | * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of |
| 7476 | * freed pages. |
| 7477 | * |
| 7478 | * Rather than trying to age LRUs the aim is to preserve the overall |
| 7479 | * LRU order by reclaiming preferentially |
| 7480 | * inactive > active > active referenced > active mapped |
| 7481 | */ |
| 7482 | unsigned long shrink_all_memory(unsigned long nr_to_reclaim) |
| 7483 | { |
| 7484 | struct scan_control sc = { |
| 7485 | .nr_to_reclaim = nr_to_reclaim, |
| 7486 | .gfp_mask = GFP_HIGHUSER_MOVABLE, |
| 7487 | .reclaim_idx = MAX_NR_ZONES - 1, |
| 7488 | .priority = DEF_PRIORITY, |
| 7489 | .may_writepage = 1, |
| 7490 | .may_unmap = 1, |
| 7491 | .may_swap = 1, |
| 7492 | .hibernation_mode = 1, |
| 7493 | }; |
| 7494 | struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); |
| 7495 | unsigned long nr_reclaimed; |
| 7496 | unsigned int noreclaim_flag; |
| 7497 | |
| 7498 | fs_reclaim_acquire(sc.gfp_mask); |
| 7499 | noreclaim_flag = memalloc_noreclaim_save(); |
| 7500 | set_task_reclaim_state(current, &sc.reclaim_state); |
| 7501 | |
| 7502 | nr_reclaimed = do_try_to_free_pages(zonelist, &sc); |
| 7503 | |
| 7504 | set_task_reclaim_state(current, NULL); |
| 7505 | memalloc_noreclaim_restore(noreclaim_flag); |
| 7506 | fs_reclaim_release(sc.gfp_mask); |
| 7507 | |
| 7508 | return nr_reclaimed; |
| 7509 | } |
| 7510 | #endif /* CONFIG_HIBERNATION */ |
| 7511 | |
| 7512 | /* |
| 7513 | * This kswapd start function will be called by init and node-hot-add. |
| 7514 | */ |
| 7515 | void kswapd_run(int nid) |
| 7516 | { |
| 7517 | pg_data_t *pgdat = NODE_DATA(nid); |
| 7518 | |
| 7519 | pgdat_kswapd_lock(pgdat); |
| 7520 | if (!pgdat->kswapd) { |
| 7521 | pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid); |
| 7522 | if (IS_ERR(pgdat->kswapd)) { |
| 7523 | /* failure at boot is fatal */ |
| 7524 | BUG_ON(system_state < SYSTEM_RUNNING); |
| 7525 | pr_err("Failed to start kswapd on node %d\n", nid); |
| 7526 | pgdat->kswapd = NULL; |
| 7527 | } |
| 7528 | } |
| 7529 | pgdat_kswapd_unlock(pgdat); |
| 7530 | } |
| 7531 | |
| 7532 | /* |
| 7533 | * Called by memory hotplug when all memory in a node is offlined. Caller must |
| 7534 | * be holding mem_hotplug_begin/done(). |
| 7535 | */ |
| 7536 | void kswapd_stop(int nid) |
| 7537 | { |
| 7538 | pg_data_t *pgdat = NODE_DATA(nid); |
| 7539 | struct task_struct *kswapd; |
| 7540 | |
| 7541 | pgdat_kswapd_lock(pgdat); |
| 7542 | kswapd = pgdat->kswapd; |
| 7543 | if (kswapd) { |
| 7544 | kthread_stop(kswapd); |
| 7545 | pgdat->kswapd = NULL; |
| 7546 | } |
| 7547 | pgdat_kswapd_unlock(pgdat); |
| 7548 | } |
| 7549 | |
| 7550 | static int __init kswapd_init(void) |
| 7551 | { |
| 7552 | int nid; |
| 7553 | |
| 7554 | swap_setup(); |
| 7555 | for_each_node_state(nid, N_MEMORY) |
| 7556 | kswapd_run(nid); |
| 7557 | return 0; |
| 7558 | } |
| 7559 | |
| 7560 | module_init(kswapd_init) |
| 7561 | |
| 7562 | #ifdef CONFIG_NUMA |
| 7563 | /* |
| 7564 | * Node reclaim mode |
| 7565 | * |
| 7566 | * If non-zero call node_reclaim when the number of free pages falls below |
| 7567 | * the watermarks. |
| 7568 | */ |
| 7569 | int node_reclaim_mode __read_mostly; |
| 7570 | |
| 7571 | /* |
| 7572 | * Priority for NODE_RECLAIM. This determines the fraction of pages |
| 7573 | * of a node considered for each zone_reclaim. 4 scans 1/16th of |
| 7574 | * a zone. |
| 7575 | */ |
| 7576 | #define NODE_RECLAIM_PRIORITY 4 |
| 7577 | |
| 7578 | /* |
| 7579 | * Percentage of pages in a zone that must be unmapped for node_reclaim to |
| 7580 | * occur. |
| 7581 | */ |
| 7582 | int sysctl_min_unmapped_ratio = 1; |
| 7583 | |
| 7584 | /* |
| 7585 | * If the number of slab pages in a zone grows beyond this percentage then |
| 7586 | * slab reclaim needs to occur. |
| 7587 | */ |
| 7588 | int sysctl_min_slab_ratio = 5; |
| 7589 | |
| 7590 | static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat) |
| 7591 | { |
| 7592 | unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED); |
| 7593 | unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) + |
| 7594 | node_page_state(pgdat, NR_ACTIVE_FILE); |
| 7595 | |
| 7596 | /* |
| 7597 | * It's possible for there to be more file mapped pages than |
| 7598 | * accounted for by the pages on the file LRU lists because |
| 7599 | * tmpfs pages accounted for as ANON can also be FILE_MAPPED |
| 7600 | */ |
| 7601 | return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0; |
| 7602 | } |
| 7603 | |
| 7604 | /* Work out how many page cache pages we can reclaim in this reclaim_mode */ |
| 7605 | static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat) |
| 7606 | { |
| 7607 | unsigned long nr_pagecache_reclaimable; |
| 7608 | unsigned long delta = 0; |
| 7609 | |
| 7610 | /* |
| 7611 | * If RECLAIM_UNMAP is set, then all file pages are considered |
| 7612 | * potentially reclaimable. Otherwise, we have to worry about |
| 7613 | * pages like swapcache and node_unmapped_file_pages() provides |
| 7614 | * a better estimate |
| 7615 | */ |
| 7616 | if (node_reclaim_mode & RECLAIM_UNMAP) |
| 7617 | nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES); |
| 7618 | else |
| 7619 | nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat); |
| 7620 | |
| 7621 | /* If we can't clean pages, remove dirty pages from consideration */ |
| 7622 | if (!(node_reclaim_mode & RECLAIM_WRITE)) |
| 7623 | delta += node_page_state(pgdat, NR_FILE_DIRTY); |
| 7624 | |
| 7625 | /* Watch for any possible underflows due to delta */ |
| 7626 | if (unlikely(delta > nr_pagecache_reclaimable)) |
| 7627 | delta = nr_pagecache_reclaimable; |
| 7628 | |
| 7629 | return nr_pagecache_reclaimable - delta; |
| 7630 | } |
| 7631 | |
| 7632 | /* |
| 7633 | * Try to free up some pages from this node through reclaim. |
| 7634 | */ |
| 7635 | static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) |
| 7636 | { |
| 7637 | /* Minimum pages needed in order to stay on node */ |
| 7638 | const unsigned long nr_pages = 1 << order; |
| 7639 | struct task_struct *p = current; |
| 7640 | unsigned int noreclaim_flag; |
| 7641 | struct scan_control sc = { |
| 7642 | .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), |
| 7643 | .gfp_mask = current_gfp_context(gfp_mask), |
| 7644 | .order = order, |
| 7645 | .priority = NODE_RECLAIM_PRIORITY, |
| 7646 | .may_writepage = !!(node_reclaim_mode & RECLAIM_WRITE), |
| 7647 | .may_unmap = !!(node_reclaim_mode & RECLAIM_UNMAP), |
| 7648 | .may_swap = 1, |
| 7649 | .reclaim_idx = gfp_zone(gfp_mask), |
| 7650 | }; |
| 7651 | unsigned long pflags; |
| 7652 | |
| 7653 | trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, order, |
| 7654 | sc.gfp_mask); |
| 7655 | |
| 7656 | cond_resched(); |
| 7657 | psi_memstall_enter(&pflags); |
| 7658 | fs_reclaim_acquire(sc.gfp_mask); |
| 7659 | /* |
| 7660 | * We need to be able to allocate from the reserves for RECLAIM_UNMAP |
| 7661 | */ |
| 7662 | noreclaim_flag = memalloc_noreclaim_save(); |
| 7663 | set_task_reclaim_state(p, &sc.reclaim_state); |
| 7664 | |
| 7665 | if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages || |
| 7666 | node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) > pgdat->min_slab_pages) { |
| 7667 | /* |
| 7668 | * Free memory by calling shrink node with increasing |
| 7669 | * priorities until we have enough memory freed. |
| 7670 | */ |
| 7671 | do { |
| 7672 | shrink_node(pgdat, &sc); |
| 7673 | } while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0); |
| 7674 | } |
| 7675 | |
| 7676 | set_task_reclaim_state(p, NULL); |
| 7677 | memalloc_noreclaim_restore(noreclaim_flag); |
| 7678 | fs_reclaim_release(sc.gfp_mask); |
| 7679 | psi_memstall_leave(&pflags); |
| 7680 | |
| 7681 | trace_mm_vmscan_node_reclaim_end(sc.nr_reclaimed); |
| 7682 | |
| 7683 | return sc.nr_reclaimed >= nr_pages; |
| 7684 | } |
| 7685 | |
| 7686 | int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) |
| 7687 | { |
| 7688 | int ret; |
| 7689 | |
| 7690 | /* |
| 7691 | * Node reclaim reclaims unmapped file backed pages and |
| 7692 | * slab pages if we are over the defined limits. |
| 7693 | * |
| 7694 | * A small portion of unmapped file backed pages is needed for |
| 7695 | * file I/O otherwise pages read by file I/O will be immediately |
| 7696 | * thrown out if the node is overallocated. So we do not reclaim |
| 7697 | * if less than a specified percentage of the node is used by |
| 7698 | * unmapped file backed pages. |
| 7699 | */ |
| 7700 | if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages && |
| 7701 | node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <= |
| 7702 | pgdat->min_slab_pages) |
| 7703 | return NODE_RECLAIM_FULL; |
| 7704 | |
| 7705 | /* |
| 7706 | * Do not scan if the allocation should not be delayed. |
| 7707 | */ |
| 7708 | if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC)) |
| 7709 | return NODE_RECLAIM_NOSCAN; |
| 7710 | |
| 7711 | /* |
| 7712 | * Only run node reclaim on the local node or on nodes that do not |
| 7713 | * have associated processors. This will favor the local processor |
| 7714 | * over remote processors and spread off node memory allocations |
| 7715 | * as wide as possible. |
| 7716 | */ |
| 7717 | if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id()) |
| 7718 | return NODE_RECLAIM_NOSCAN; |
| 7719 | |
| 7720 | if (test_and_set_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags)) |
| 7721 | return NODE_RECLAIM_NOSCAN; |
| 7722 | |
| 7723 | ret = __node_reclaim(pgdat, gfp_mask, order); |
| 7724 | clear_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags); |
| 7725 | |
| 7726 | if (!ret) |
| 7727 | count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED); |
| 7728 | |
| 7729 | return ret; |
| 7730 | } |
| 7731 | #endif |
| 7732 | |
| 7733 | void check_move_unevictable_pages(struct pagevec *pvec) |
| 7734 | { |
| 7735 | struct folio_batch fbatch; |
| 7736 | unsigned i; |
| 7737 | |
| 7738 | folio_batch_init(&fbatch); |
| 7739 | for (i = 0; i < pvec->nr; i++) { |
| 7740 | struct page *page = pvec->pages[i]; |
| 7741 | |
| 7742 | if (PageTransTail(page)) |
| 7743 | continue; |
| 7744 | folio_batch_add(&fbatch, page_folio(page)); |
| 7745 | } |
| 7746 | check_move_unevictable_folios(&fbatch); |
| 7747 | } |
| 7748 | EXPORT_SYMBOL_GPL(check_move_unevictable_pages); |
| 7749 | |
| 7750 | /** |
| 7751 | * check_move_unevictable_folios - Move evictable folios to appropriate zone |
| 7752 | * lru list |
| 7753 | * @fbatch: Batch of lru folios to check. |
| 7754 | * |
| 7755 | * Checks folios for evictability, if an evictable folio is in the unevictable |
| 7756 | * lru list, moves it to the appropriate evictable lru list. This function |
| 7757 | * should be only used for lru folios. |
| 7758 | */ |
| 7759 | void check_move_unevictable_folios(struct folio_batch *fbatch) |
| 7760 | { |
| 7761 | struct lruvec *lruvec = NULL; |
| 7762 | int pgscanned = 0; |
| 7763 | int pgrescued = 0; |
| 7764 | int i; |
| 7765 | |
| 7766 | for (i = 0; i < fbatch->nr; i++) { |
| 7767 | struct folio *folio = fbatch->folios[i]; |
| 7768 | int nr_pages = folio_nr_pages(folio); |
| 7769 | |
| 7770 | pgscanned += nr_pages; |
| 7771 | |
| 7772 | /* block memcg migration while the folio moves between lrus */ |
| 7773 | if (!folio_test_clear_lru(folio)) |
| 7774 | continue; |
| 7775 | |
| 7776 | lruvec = folio_lruvec_relock_irq(folio, lruvec); |
| 7777 | if (folio_evictable(folio) && folio_test_unevictable(folio)) { |
| 7778 | lruvec_del_folio(lruvec, folio); |
| 7779 | folio_clear_unevictable(folio); |
| 7780 | lruvec_add_folio(lruvec, folio); |
| 7781 | pgrescued += nr_pages; |
| 7782 | } |
| 7783 | folio_set_lru(folio); |
| 7784 | } |
| 7785 | |
| 7786 | if (lruvec) { |
| 7787 | __count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued); |
| 7788 | __count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned); |
| 7789 | unlock_page_lruvec_irq(lruvec); |
| 7790 | } else if (pgscanned) { |
| 7791 | count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned); |
| 7792 | } |
| 7793 | } |
| 7794 | EXPORT_SYMBOL_GPL(check_move_unevictable_folios); |