Merge tag 'loongarch-6.10' of git://git.kernel.org/pub/scm/linux/kernel/git/chenhuaca...

[linux-2.6-block.git] / mm / vmscan.c

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 3ef654addd44c26f999c84ba82a10f005804ece4..d55e8d07ffc4a75e68e587ec9e22dc802cf67d24 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -967,7 +967,8 @@ static unsigned int demote_folio_list(struct list_head *demote_folios,
                .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOWARN |
                        __GFP_NOMEMALLOC | GFP_NOWAIT,
                .nid = target_nid,
-               .nmask = &allowed_mask
+               .nmask = &allowed_mask,
+               .reason = MR_DEMOTION,
        };
 
        if (list_empty(demote_folios))
@@ -1205,25 +1206,28 @@ retry:
                                        if (!can_split_folio(folio, NULL))
                                                goto activate_locked;
                                        /*
-                                        * Split folios without a PMD map right
-                                        * away. Chances are some or all of the
-                                        * tail pages can be freed without IO.
+                                        * Split partially mapped folios right away.
+                                        * We can free the unmapped pages without IO.
                                         */
-                                       if (!folio_entire_mapcount(folio) &&
-                                           split_folio_to_list(folio,
-                                                               folio_list))
+                                       if (data_race(!list_empty(&folio->_deferred_list)) &&
+                                           split_folio_to_list(folio, folio_list))
                                                goto activate_locked;
                                }
                                if (!add_to_swap(folio)) {
+                                       int __maybe_unused order = folio_order(folio);
+
                                        if (!folio_test_large(folio))
                                                goto activate_locked_split;
                                        /* Fallback to swap normal pages */
-                                       if (split_folio_to_list(folio,
-                                                               folio_list))
+                                       if (split_folio_to_list(folio, folio_list))
                                                goto activate_locked;
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
-                                       count_memcg_folio_events(folio, THP_SWPOUT_FALLBACK, 1);
-                                       count_vm_event(THP_SWPOUT_FALLBACK);
+                                       if (nr_pages >= HPAGE_PMD_NR) {
+                                               count_memcg_folio_events(folio,
+                                                       THP_SWPOUT_FALLBACK, 1);
+                                               count_vm_event(THP_SWPOUT_FALLBACK);
+                                       }
+                                       count_mthp_stat(order, MTHP_STAT_ANON_SWPOUT_FALLBACK);
 #endif
                                        if (!add_to_swap(folio))
                                                goto activate_locked_split;
@@ -1256,6 +1260,20 @@ retry:
 
                        if (folio_test_pmd_mappable(folio))
                                flags |= TTU_SPLIT_HUGE_PMD;
+                       /*
+                        * Without TTU_SYNC, try_to_unmap will only begin to
+                        * hold PTL from the first present PTE within a large
+                        * folio. Some initial PTEs might be skipped due to
+                        * races with parallel PTE writes in which PTEs can be
+                        * cleared temporarily before being written new present
+                        * values. This will lead to a large folio is still
+                        * mapped while some subpages have been partially
+                        * unmapped after try_to_unmap; TTU_SYNC helps
+                        * try_to_unmap acquire PTL from the first PTE,
+                        * eliminating the influence of temporary PTE values.
+                        */
+                       if (folio_test_large(folio) && list_empty(&folio->_deferred_list))
+                               flags |= TTU_SYNC;
 
                        try_to_unmap(folio, flags);
                        if (folio_mapped(folio)) {
@@ -2091,8 +2109,7 @@ static void shrink_active_list(unsigned long nr_to_scan,
 }
 
 static unsigned int reclaim_folio_list(struct list_head *folio_list,
-                                     struct pglist_data *pgdat,
-                                     bool ignore_references)
+                                     struct pglist_data *pgdat)
 {
        struct reclaim_stat dummy_stat;
        unsigned int nr_reclaimed;
@@ -2105,7 +2122,7 @@ static unsigned int reclaim_folio_list(struct list_head *folio_list,
                .no_demotion = 1,
        };
 
-       nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &dummy_stat, ignore_references);
+       nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &dummy_stat, true);
        while (!list_empty(folio_list)) {
                folio = lru_to_folio(folio_list);
                list_del(&folio->lru);
@@ -2115,7 +2132,7 @@ static unsigned int reclaim_folio_list(struct list_head *folio_list,
        return nr_reclaimed;
 }
 
-unsigned long reclaim_pages(struct list_head *folio_list, bool ignore_references)
+unsigned long reclaim_pages(struct list_head *folio_list)
 {
        int nid;
        unsigned int nr_reclaimed = 0;
@@ -2137,12 +2154,11 @@ unsigned long reclaim_pages(struct list_head *folio_list, bool ignore_references
                        continue;
                }
 
-               nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid),
-                                                  ignore_references);
+               nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid));
                nid = folio_nid(lru_to_folio(folio_list));
        } while (!list_empty(folio_list));
 
-       nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid), ignore_references);
+       nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid));
 
        memalloc_noreclaim_restore(noreclaim_flag);