perf/core: Optimize perf_adjust_freq_unthr_context()

It was unnecessarily disabling and enabling PMUs for each event.  It
should be done at PMU level.  Add pmu_ctx->nr_freq counter to check it
at each PMU.  As PMU context has separate active lists for pinned group
and flexible group, factor out a new function to do the job.

Another minor optimization is that it can skip PMUs w/ CAP_NO_INTERRUPT
even if it needs to unthrottle sampling events.

Signed-off-by: Namhyung Kim <namhyung@kernel.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Mingwei Zhang <mizhang@google.com>
Reviewed-by: Ian Rogers <irogers@google.com>
Reviewed-by: Kan Liang <kan.liang@linux.intel.com>
Link: https://lore.kernel.org/r/20240207050545.2727923-1-namhyung@kernel.org

diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index d2a15c0c6f8a9ec767d93d6e52ab93cb2c12db28..3e33b366347a1f9a8f2b0ec41cc2d65fca4ec4c5 100644 (file)
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -883,6 +883,7 @@ struct perf_event_pmu_context {
 
        unsigned int                    nr_events;
        unsigned int                    nr_cgroups;
+       unsigned int                    nr_freq;
 
        atomic_t                        refcount; /* event <-> epc */
        struct rcu_head                 rcu_head;
@@ -897,6 +898,11 @@ struct perf_event_pmu_context {
        int                             rotate_necessary;
 };
 
+static inline bool perf_pmu_ctx_is_active(struct perf_event_pmu_context *epc)
+{
+       return !list_empty(&epc->flexible_active) || !list_empty(&epc->pinned_active);
+}
+
 struct perf_event_groups {
        struct rb_root  tree;
        u64             index;

diff --git a/kernel/events/core.c b/kernel/events/core.c
index 724e6d7e128f3766f89791861c258fb317297216..9566cfb273551c416eb77c1ad84517c0fa8ce4a3 100644 (file)
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -2302,8 +2302,10 @@ event_sched_out(struct perf_event *event, struct perf_event_context *ctx)
 
        if (!is_software_event(event))
                cpc->active_oncpu--;
-       if (event->attr.freq && event->attr.sample_freq)
+       if (event->attr.freq && event->attr.sample_freq) {
                ctx->nr_freq--;
+               epc->nr_freq--;
+       }
        if (event->attr.exclusive || !cpc->active_oncpu)
                cpc->exclusive = 0;
 
@@ -2558,9 +2560,10 @@ event_sched_in(struct perf_event *event, struct perf_event_context *ctx)
 
        if (!is_software_event(event))
                cpc->active_oncpu++;
-       if (event->attr.freq && event->attr.sample_freq)
+       if (event->attr.freq && event->attr.sample_freq) {
                ctx->nr_freq++;
-
+               epc->nr_freq++;
+       }
        if (event->attr.exclusive)
                cpc->exclusive = 1;
 
@@ -4123,30 +4126,14 @@ static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bo
        }
 }
 
-/*
- * combine freq adjustment with unthrottling to avoid two passes over the
- * events. At the same time, make sure, having freq events does not change
- * the rate of unthrottling as that would introduce bias.
- */
-static void
-perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
+static void perf_adjust_freq_unthr_events(struct list_head *event_list)
 {
        struct perf_event *event;
        struct hw_perf_event *hwc;
        u64 now, period = TICK_NSEC;
        s64 delta;
 
-       /*
-        * only need to iterate over all events iff:
-        * - context have events in frequency mode (needs freq adjust)
-        * - there are events to unthrottle on this cpu
-        */
-       if (!(ctx->nr_freq || unthrottle))
-               return;
-
-       raw_spin_lock(&ctx->lock);
-
-       list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
+       list_for_each_entry(event, event_list, active_list) {
                if (event->state != PERF_EVENT_STATE_ACTIVE)
                        continue;
 
@@ -4154,8 +4141,6 @@ perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
                if (!event_filter_match(event))
                        continue;
 
-               perf_pmu_disable(event->pmu);
-
                hwc = &event->hw;
 
                if (hwc->interrupts == MAX_INTERRUPTS) {
@@ -4165,7 +4150,7 @@ perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
                }
 
                if (!event->attr.freq || !event->attr.sample_freq)
-                       goto next;
+                       continue;
 
                /*
                 * stop the event and update event->count
@@ -4187,8 +4172,41 @@ perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
                        perf_adjust_period(event, period, delta, false);
 
                event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
-       next:
-               perf_pmu_enable(event->pmu);
+       }
+}
+
+/*
+ * combine freq adjustment with unthrottling to avoid two passes over the
+ * events. At the same time, make sure, having freq events does not change
+ * the rate of unthrottling as that would introduce bias.
+ */
+static void
+perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
+{
+       struct perf_event_pmu_context *pmu_ctx;
+
+       /*
+        * only need to iterate over all events iff:
+        * - context have events in frequency mode (needs freq adjust)
+        * - there are events to unthrottle on this cpu
+        */
+       if (!(ctx->nr_freq || unthrottle))
+               return;
+
+       raw_spin_lock(&ctx->lock);
+
+       list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
+               if (!(pmu_ctx->nr_freq || unthrottle))
+                       continue;
+               if (!perf_pmu_ctx_is_active(pmu_ctx))
+                       continue;
+               if (pmu_ctx->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT)
+                       continue;
+
+               perf_pmu_disable(pmu_ctx->pmu);
+               perf_adjust_freq_unthr_events(&pmu_ctx->pinned_active);
+               perf_adjust_freq_unthr_events(&pmu_ctx->flexible_active);
+               perf_pmu_enable(pmu_ctx->pmu);
        }
 
        raw_spin_unlock(&ctx->lock);