memory tiering: introduce folio_use_access_time() check

If memory tiering mode is on and a folio is not in the top tier memory,
folio's cpupid field is repurposed to store page access time.  Instead of
an open coded check, use a function to encapsulate the check.

Link: https://lkml.kernel.org/r/20240724130115.793641-3-ziy@nvidia.com
Signed-off-by: Zi Yan <ziy@nvidia.com>
Reviewed-by: "Huang, Ying" <ying.huang@intel.com>
Acked-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 6549d0979b28f93332db0457bfa70942ff2cf43a..6c2078ca3391452b3ab73eab69f2b49b873a1bc1 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1745,6 +1745,8 @@ static inline void vma_set_access_pid_bit(struct vm_area_struct *vma)
                __set_bit(pid_bit, &vma->numab_state->pids_active[1]);
        }
 }
+
+bool folio_use_access_time(struct folio *folio);
 #else /* !CONFIG_NUMA_BALANCING */
 static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid)
 {
@@ -1798,6 +1800,10 @@ static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
 static inline void vma_set_access_pid_bit(struct vm_area_struct *vma)
 {
 }
+static inline bool folio_use_access_time(struct folio *folio)
+{
+       return false;
+}
 #endif /* CONFIG_NUMA_BALANCING */
 
 #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 9057584ec06de948c0032f48d382354623cf1c6a..416e29b56cc4478b2d49cc4a93c5ffa2e3f8dd8e 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1840,8 +1840,7 @@ bool should_numa_migrate_memory(struct task_struct *p, struct folio *folio,
         * The pages in slow memory node should be migrated according
         * to hot/cold instead of private/shared.
         */
-       if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
-           !node_is_toptier(src_nid)) {
+       if (folio_use_access_time(folio)) {
                struct pglist_data *pgdat;
                unsigned long rate_limit;
                unsigned int latency, th, def_th;

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 632a92edec4ee4d9410ae916d9895eeda547ba86..29e76d285e4b5cbfa8dd31d1585050870a465139 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1707,8 +1707,7 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf)
         * For memory tiering mode, cpupid of slow memory page is used
         * to record page access time.  So use default value.
         */
-       if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) ||
-           node_is_toptier(nid))
+       if (!folio_use_access_time(folio))
                last_cpupid = folio_last_cpupid(folio);
        target_nid = numa_migrate_prep(folio, vmf, haddr, nid, &flags);
        if (target_nid == NUMA_NO_NODE)
@@ -2058,8 +2057,7 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
                    toptier)
                        goto unlock;
 
-               if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
-                   !toptier)
+               if (folio_use_access_time(folio))
                        folio_xchg_access_time(folio,
                                               jiffies_to_msecs(jiffies));
        }

diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c
index 4775b3a3dabe6bd583e016261d44025de2775888..2a642ea86cb20a5c09ca85b82f402da9bf7cad9e 100644 (file)
--- a/mm/memory-tiers.c
+++ b/mm/memory-tiers.c
@@ -6,6 +6,7 @@
 #include <linux/memory.h>
 #include <linux/memory-tiers.h>
 #include <linux/notifier.h>
+#include <linux/sched/sysctl.h>
 
 #include "internal.h"
 
@@ -50,6 +51,24 @@ static const struct bus_type memory_tier_subsys = {
        .dev_name = "memory_tier",
 };
 
+#ifdef CONFIG_NUMA_BALANCING
+/**
+ * folio_use_access_time - check if a folio reuses cpupid for page access time
+ * @folio: folio to check
+ *
+ * folio's _last_cpupid field is repurposed by memory tiering. In memory
+ * tiering mode, cpupid of slow memory folio (not toptier memory) is used to
+ * record page access time.
+ *
+ * Return: the folio _last_cpupid is used to record page access time
+ */
+bool folio_use_access_time(struct folio *folio)
+{
+       return (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
+              !node_is_toptier(folio_nid(folio));
+}
+#endif
+
 #ifdef CONFIG_MIGRATION
 static int top_tier_adistance;
 /*

diff --git a/mm/memory.c b/mm/memory.c
index 3c01d68065be21a45d8b9085d7f81785bc5cbfd8..a3d955098a192b97dd894903e71ac3aa0e278539 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -5337,8 +5337,7 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf)
         * For memory tiering mode, cpupid of slow memory page is used
         * to record page access time.  So use default value.
         */
-       if ((sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
-           !node_is_toptier(nid))
+       if (folio_use_access_time(folio))
                last_cpupid = (-1 & LAST_CPUPID_MASK);
        else
                last_cpupid = folio_last_cpupid(folio);

diff --git a/mm/mprotect.c b/mm/mprotect.c
index 222ab434da549dcd796d66144ed359efe0df98fa..37cf8d249405dc0ce4a7112faadf0ec658c52c08 100644 (file)
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -161,8 +161,7 @@ static long change_pte_range(struct mmu_gather *tlb,
                                if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) &&
                                    toptier)
                                        continue;
-                               if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
-                                   !toptier)
+                               if (folio_use_access_time(folio))
                                        folio_xchg_access_time(folio,
                                                jiffies_to_msecs(jiffies));
                        }