mm: add alloc_contig_migrate_range allocation statistics

alloc_contig_migrate_range has every information to be able to understand
big contiguous allocation latency.  For example, how many pages are
migrated, how many times they were needed to unmap from page tables.

This patch adds the trace event to collect the allocation statistics.  In
the field, it was quite useful to understand CMA allocation latency.

[akpm@linux-foundation.org: a/trace_mm_alloc_config_migrate_range_info_enabled/trace_mm_alloc_contig_migrate_range_info_enabled]
Link: https://lkml.kernel.org/r/20240228051127.2859472-1-richardycc@google.com
Signed-off-by: Richard Chang <richardycc@google.com>
Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org.
Cc: Martin Liu <liumartin@google.com>
Cc: "Masami Hiramatsu (Google)" <mhiramat@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/trace/events/kmem.h b/include/trace/events/kmem.h
index 58688768ef0f033b0202125b8eedb29afce0d2a4..6e62cc64cd92a647ad42aa109a5578c668d35617 100644 (file)
--- a/include/trace/events/kmem.h
+++ b/include/trace/events/kmem.h
@@ -304,6 +304,44 @@ TRACE_EVENT(mm_page_alloc_extfrag,
                __entry->change_ownership)
 );
 
+TRACE_EVENT(mm_alloc_contig_migrate_range_info,
+
+       TP_PROTO(unsigned long start,
+                unsigned long end,
+                unsigned long nr_migrated,
+                unsigned long nr_reclaimed,
+                unsigned long nr_mapped,
+                int migratetype),
+
+       TP_ARGS(start, end, nr_migrated, nr_reclaimed, nr_mapped, migratetype),
+
+       TP_STRUCT__entry(
+               __field(unsigned long, start)
+               __field(unsigned long, end)
+               __field(unsigned long, nr_migrated)
+               __field(unsigned long, nr_reclaimed)
+               __field(unsigned long, nr_mapped)
+               __field(int, migratetype)
+       ),
+
+       TP_fast_assign(
+               __entry->start = start;
+               __entry->end = end;
+               __entry->nr_migrated = nr_migrated;
+               __entry->nr_reclaimed = nr_reclaimed;
+               __entry->nr_mapped = nr_mapped;
+               __entry->migratetype = migratetype;
+       ),
+
+       TP_printk("start=0x%lx end=0x%lx migratetype=%d nr_migrated=%lu nr_reclaimed=%lu nr_mapped=%lu",
+                 __entry->start,
+                 __entry->end,
+                 __entry->migratetype,
+                 __entry->nr_migrated,
+                 __entry->nr_reclaimed,
+                 __entry->nr_mapped)
+);
+
 /*
  * Required for uniquely and securely identifying mm in rss_stat tracepoint.
  */

diff --git a/mm/internal.h b/mm/internal.h
index b1d806125e8d4651d405bee04781f207aff69ac6..5be91e2f99434afcd9a85d0ef87b13b62ab20bed 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -539,7 +539,8 @@ isolate_migratepages_range(struct compact_control *cc,
                           unsigned long low_pfn, unsigned long end_pfn);
 
 int __alloc_contig_migrate_range(struct compact_control *cc,
-                                       unsigned long start, unsigned long end);
+                                       unsigned long start, unsigned long end,
+                                       int migratetype);
 
 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */
 void init_cma_reserved_pageblock(struct page *page);

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 7873e937580245e05d0fb36e8d6e86d88b192862..29c982542dcc47f7f87c008bfd28249321f9410f 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -6222,9 +6222,14 @@ static void alloc_contig_dump_pages(struct list_head *page_list)
        }
 }
 
-/* [start, end) must belong to a single zone. */
+/*
+ * [start, end) must belong to a single zone.
+ * @migratetype: using migratetype to filter the type of migration in
+ *             trace_mm_alloc_contig_migrate_range_info.
+ */
 int __alloc_contig_migrate_range(struct compact_control *cc,
-                                       unsigned long start, unsigned long end)
+                                       unsigned long start, unsigned long end,
+                                       int migratetype)
 {
        /* This function is based on compact_zone() from compaction.c. */
        unsigned int nr_reclaimed;
@@ -6235,6 +6240,10 @@ int __alloc_contig_migrate_range(struct compact_control *cc,
                .nid = zone_to_nid(cc->zone),
                .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL,
        };
+       struct page *page;
+       unsigned long total_mapped = 0;
+       unsigned long total_migrated = 0;
+       unsigned long total_reclaimed = 0;
 
        lru_cache_disable();
 
@@ -6260,9 +6269,18 @@ int __alloc_contig_migrate_range(struct compact_control *cc,
                                                        &cc->migratepages);
                cc->nr_migratepages -= nr_reclaimed;
 
+               if (trace_mm_alloc_contig_migrate_range_info_enabled()) {
+                       total_reclaimed += nr_reclaimed;
+                       list_for_each_entry(page, &cc->migratepages, lru)
+                               total_mapped += page_mapcount(page);
+               }
+
                ret = migrate_pages(&cc->migratepages, alloc_migration_target,
                        NULL, (unsigned long)&mtc, cc->mode, MR_CONTIG_RANGE, NULL);
 
+               if (trace_mm_alloc_contig_migrate_range_info_enabled() && !ret)
+                       total_migrated += cc->nr_migratepages;
+
                /*
                 * On -ENOMEM, migrate_pages() bails out right away. It is pointless
                 * to retry again over this error, so do the same here.
@@ -6276,9 +6294,13 @@ int __alloc_contig_migrate_range(struct compact_control *cc,
                if (!(cc->gfp_mask & __GFP_NOWARN) && ret == -EBUSY)
                        alloc_contig_dump_pages(&cc->migratepages);
                putback_movable_pages(&cc->migratepages);
-               return ret;
        }
-       return 0;
+
+       trace_mm_alloc_contig_migrate_range_info(start, end, migratetype,
+                                                total_migrated,
+                                                total_reclaimed,
+                                                total_mapped);
+       return (ret < 0) ? ret : 0;
 }
 
 /**
@@ -6358,7 +6380,7 @@ int alloc_contig_range(unsigned long start, unsigned long end,
         * allocated.  So, if we fall through be sure to clear ret so that
         * -EBUSY is not accidentally used or returned to caller.
         */
-       ret = __alloc_contig_migrate_range(&cc, start, end);
+       ret = __alloc_contig_migrate_range(&cc, start, end, migratetype);
        if (ret && ret != -EBUSY)
                goto done;
        ret = 0;

diff --git a/mm/page_isolation.c b/mm/page_isolation.c
index cd0ea36682533ae7956a25ba50086c3584ec4bfa..a5c8fa4c2a75c35a68349e0e6a44e8f5b51435d5 100644 (file)
--- a/mm/page_isolation.c
+++ b/mm/page_isolation.c
@@ -434,7 +434,7 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags,
                                }
 
                                ret = __alloc_contig_migrate_range(&cc, head_pfn,
-                                                       head_pfn + nr_pages);
+                                                       head_pfn + nr_pages, page_mt);
 
                                /*
                                 * restore the page's migratetype so that it can