perf: Fix event leak upon exec and file release

The perf pending task work is never waited upon the matching event
release. In the case of a child event, released via free_event()
directly, this can potentially result in a leaked event, such as in the
following scenario that doesn't even require a weak IRQ work
implementation to trigger:

schedule()
   prepare_task_switch()
=======> <NMI>
      perf_event_overflow()
         event->pending_sigtrap = ...
         irq_work_queue(&event->pending_irq)
<======= </NMI>
      perf_event_task_sched_out()
          event_sched_out()
              event->pending_sigtrap = 0;
              atomic_long_inc_not_zero(&event->refcount)
              task_work_add(&event->pending_task)
   finish_lock_switch()
=======> <IRQ>
   perf_pending_irq()
      //do nothing, rely on pending task work
<======= </IRQ>

begin_new_exec()
   perf_event_exit_task()
      perf_event_exit_event()
         // If is child event
         free_event()
            WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1)
            // event is leaked

Similar scenarios can also happen with perf_event_remove_on_exec() or
simply against concurrent perf_event_release().

Fix this with synchonizing against the possibly remaining pending task
work while freeing the event, just like is done with remaining pending
IRQ work. This means that the pending task callback neither need nor
should hold a reference to the event, preventing it from ever beeing
freed.

Fixes: 517e6a301f34 ("perf: Fix perf_pending_task() UaF")
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: stable@vger.kernel.org
Link: https://lore.kernel.org/r/20240621091601.18227-5-frederic@kernel.org

diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index a5304ae8c654f040e611c34fdd1a7267931dff53..393fb13733b02447c6cca0886b9fcb5a2dc7c715 100644 (file)
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -786,6 +786,7 @@ struct perf_event {
        struct irq_work                 pending_irq;
        struct callback_head            pending_task;
        unsigned int                    pending_work;
+       struct rcuwait                  pending_work_wait;
 
        atomic_t                        event_limit;
 

diff --git a/kernel/events/core.c b/kernel/events/core.c
index 576400d220a5670a06359bc8dea560c5194a7f3d..32c7996a4c022ca49639fef83ce782b8e1fc231d 100644 (file)
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -2288,7 +2288,6 @@ event_sched_out(struct perf_event *event, struct perf_event_context *ctx)
                if (state != PERF_EVENT_STATE_OFF &&
                    !event->pending_work &&
                    !task_work_add(current, &event->pending_task, TWA_RESUME)) {
-                       WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount));
                        event->pending_work = 1;
                } else {
                        local_dec(&event->ctx->nr_pending);
@@ -5203,9 +5202,35 @@ static bool exclusive_event_installable(struct perf_event *event,
 static void perf_addr_filters_splice(struct perf_event *event,
                                       struct list_head *head);
 
+static void perf_pending_task_sync(struct perf_event *event)
+{
+       struct callback_head *head = &event->pending_task;
+
+       if (!event->pending_work)
+               return;
+       /*
+        * If the task is queued to the current task's queue, we
+        * obviously can't wait for it to complete. Simply cancel it.
+        */
+       if (task_work_cancel(current, head)) {
+               event->pending_work = 0;
+               local_dec(&event->ctx->nr_pending);
+               return;
+       }
+
+       /*
+        * All accesses related to the event are within the same
+        * non-preemptible section in perf_pending_task(). The RCU
+        * grace period before the event is freed will make sure all
+        * those accesses are complete by then.
+        */
+       rcuwait_wait_event(&event->pending_work_wait, !event->pending_work, TASK_UNINTERRUPTIBLE);
+}
+
 static void _free_event(struct perf_event *event)
 {
        irq_work_sync(&event->pending_irq);
+       perf_pending_task_sync(event);
 
        unaccount_event(event);
 
@@ -6817,24 +6842,28 @@ static void perf_pending_task(struct callback_head *head)
        struct perf_event *event = container_of(head, struct perf_event, pending_task);
        int rctx;
 
+       /*
+        * All accesses to the event must belong to the same implicit RCU read-side
+        * critical section as the ->pending_work reset. See comment in
+        * perf_pending_task_sync().
+        */
+       preempt_disable_notrace();
        /*
         * If we 'fail' here, that's OK, it means recursion is already disabled
         * and we won't recurse 'further'.
         */
-       preempt_disable_notrace();
        rctx = perf_swevent_get_recursion_context();
 
        if (event->pending_work) {
                event->pending_work = 0;
                perf_sigtrap(event);
                local_dec(&event->ctx->nr_pending);
+               rcuwait_wake_up(&event->pending_work_wait);
        }
 
        if (rctx >= 0)
                perf_swevent_put_recursion_context(rctx);
        preempt_enable_notrace();
-
-       put_event(event);
 }
 
 #ifdef CONFIG_GUEST_PERF_EVENTS
@@ -11948,6 +11977,7 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
        init_waitqueue_head(&event->waitq);
        init_irq_work(&event->pending_irq, perf_pending_irq);
        init_task_work(&event->pending_task, perf_pending_task);
+       rcuwait_init(&event->pending_work_wait);
 
        mutex_init(&event->mmap_mutex);
        raw_spin_lock_init(&event->addr_filters.lock);