memcg: move preempt disable to callers of memcg_rstat_updated

Let's move the explicit preempt disable code to the callers of
memcg_rstat_updated and also remove the memcg_stats_lock and related
functions which ensures the callers of stats update functions have
disabled preemption because now the stats update functions are explicitly
disabling preemption.

Link: https://lkml.kernel.org/r/20250514184158.3471331-3-shakeel.butt@linux.dev
Signed-off-by: Shakeel Butt <shakeel.butt@linux.dev>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 01560e22dd0605ac994f6d80389eec5ef9cbf434..82b51e1b36f06e80632ab9ec1cc42b29e99068f7 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -555,48 +555,22 @@ static u64 flush_last_time;
 
 #define FLUSH_TIME (2UL*HZ)
 
-/*
- * Accessors to ensure that preemption is disabled on PREEMPT_RT because it can
- * not rely on this as part of an acquired spinlock_t lock. These functions are
- * never used in hardirq context on PREEMPT_RT and therefore disabling preemtion
- * is sufficient.
- */
-static void memcg_stats_lock(void)
-{
-       preempt_disable_nested();
-       VM_WARN_ON_IRQS_ENABLED();
-}
-
-static void __memcg_stats_lock(void)
-{
-       preempt_disable_nested();
-}
-
-static void memcg_stats_unlock(void)
-{
-       preempt_enable_nested();
-}
-
-
 static bool memcg_vmstats_needs_flush(struct memcg_vmstats *vmstats)
 {
        return atomic64_read(&vmstats->stats_updates) >
                MEMCG_CHARGE_BATCH * num_online_cpus();
 }
 
-static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val)
+static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val,
+                                      int cpu)
 {
        struct memcg_vmstats_percpu __percpu *statc_pcpu;
        struct memcg_vmstats_percpu *statc;
-       int cpu;
        unsigned int stats_updates;
 
        if (!val)
                return;
 
-       /* Don't assume callers have preemption disabled. */
-       cpu = get_cpu();
-
        cgroup_rstat_updated(memcg->css.cgroup, cpu);
        statc_pcpu = memcg->vmstats_percpu;
        for (; statc_pcpu; statc_pcpu = statc->parent_pcpu) {
@@ -617,7 +591,6 @@ static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val)
                stats_updates = this_cpu_xchg(statc_pcpu->stats_updates, 0);
                atomic64_add(stats_updates, &statc->vmstats->stats_updates);
        }
-       put_cpu();
 }
 
 static void __mem_cgroup_flush_stats(struct mem_cgroup *memcg, bool force)
@@ -715,6 +688,7 @@ void __mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx,
                       int val)
 {
        int i = memcg_stats_index(idx);
+       int cpu;
 
        if (mem_cgroup_disabled())
                return;
@@ -722,12 +696,14 @@ void __mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx,
        if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
                return;
 
-       memcg_stats_lock();
+       cpu = get_cpu();
+
        __this_cpu_add(memcg->vmstats_percpu->state[i], val);
        val = memcg_state_val_in_pages(idx, val);
-       memcg_rstat_updated(memcg, val);
+       memcg_rstat_updated(memcg, val, cpu);
        trace_mod_memcg_state(memcg, idx, val);
-       memcg_stats_unlock();
+
+       put_cpu();
 }
 
 #ifdef CONFIG_MEMCG_V1
@@ -756,6 +732,7 @@ static void __mod_memcg_lruvec_state(struct lruvec *lruvec,
        struct mem_cgroup_per_node *pn;
        struct mem_cgroup *memcg;
        int i = memcg_stats_index(idx);
+       int cpu;
 
        if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
                return;
@@ -763,24 +740,7 @@ static void __mod_memcg_lruvec_state(struct lruvec *lruvec,
        pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
        memcg = pn->memcg;
 
-       /*
-        * The caller from rmap relies on disabled preemption because they never
-        * update their counter from in-interrupt context. For these two
-        * counters we check that the update is never performed from an
-        * interrupt context while other caller need to have disabled interrupt.
-        */
-       __memcg_stats_lock();
-       if (IS_ENABLED(CONFIG_DEBUG_VM)) {
-               switch (idx) {
-               case NR_ANON_MAPPED:
-               case NR_FILE_MAPPED:
-               case NR_ANON_THPS:
-                       WARN_ON_ONCE(!in_task());
-                       break;
-               default:
-                       VM_WARN_ON_IRQS_ENABLED();
-               }
-       }
+       cpu = get_cpu();
 
        /* Update memcg */
        __this_cpu_add(memcg->vmstats_percpu->state[i], val);
@@ -789,9 +749,10 @@ static void __mod_memcg_lruvec_state(struct lruvec *lruvec,
        __this_cpu_add(pn->lruvec_stats_percpu->state[i], val);
 
        val = memcg_state_val_in_pages(idx, val);
-       memcg_rstat_updated(memcg, val);
+       memcg_rstat_updated(memcg, val, cpu);
        trace_mod_memcg_lruvec_state(memcg, idx, val);
-       memcg_stats_unlock();
+
+       put_cpu();
 }
 
 /**
@@ -871,6 +832,7 @@ void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
                          unsigned long count)
 {
        int i = memcg_events_index(idx);
+       int cpu;
 
        if (mem_cgroup_disabled())
                return;
@@ -878,11 +840,13 @@ void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
        if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
                return;
 
-       memcg_stats_lock();
+       cpu = get_cpu();
+
        __this_cpu_add(memcg->vmstats_percpu->events[i], count);
-       memcg_rstat_updated(memcg, count);
+       memcg_rstat_updated(memcg, count, cpu);
        trace_count_memcg_events(memcg, idx, count);
-       memcg_stats_unlock();
+
+       put_cpu();
 }
 
 unsigned long memcg_events(struct mem_cgroup *memcg, int event)