net: split off __napi_busy_poll from napi_busy_poll

This splits off the key part of the napi_busy_poll function into its own
function, __napi_busy_poll, and changes the prefer_busy_poll bool to be
flag based to allow passing in more flags in the future.

This is done in preparation for an additional napi_busy_poll() function,
that doesn't take the rcu_read_lock(). The new function is introduced
in the next patch.

Signed-off-by: Stefan Roesch <shr@devkernel.io>
Link: https://lore.kernel.org/r/20230608163839.2891748-2-shr@devkernel.io
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/net/core/dev.c b/net/core/dev.c
index cb2dab0feee0abe758479a7a001342bf6613df08..1eaed657f2c2401d29c79c9057b85df1ec45c64b 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -6177,8 +6177,12 @@ static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
        clear_bit(NAPI_STATE_SCHED, &napi->state);
 }
 
-static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool prefer_busy_poll,
-                          u16 budget)
+enum {
+       NAPI_F_PREFER_BUSY_POLL = 1,
+};
+
+static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock,
+                          unsigned flags, u16 budget)
 {
        bool skip_schedule = false;
        unsigned long timeout;
@@ -6198,7 +6202,7 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool
 
        local_bh_disable();
 
-       if (prefer_busy_poll) {
+       if (flags & NAPI_F_PREFER_BUSY_POLL) {
                napi->defer_hard_irqs_count = READ_ONCE(napi->dev->napi_defer_hard_irqs);
                timeout = READ_ONCE(napi->dev->gro_flush_timeout);
                if (napi->defer_hard_irqs_count && timeout) {
@@ -6222,23 +6226,23 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool
        local_bh_enable();
 }
 
-void napi_busy_loop(unsigned int napi_id,
-                   bool (*loop_end)(void *, unsigned long),
-                   void *loop_end_arg, bool prefer_busy_poll, u16 budget)
+static void __napi_busy_loop(unsigned int napi_id,
+                     bool (*loop_end)(void *, unsigned long),
+                     void *loop_end_arg, unsigned flags, u16 budget)
 {
        unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
        int (*napi_poll)(struct napi_struct *napi, int budget);
        void *have_poll_lock = NULL;
        struct napi_struct *napi;
 
+       WARN_ON_ONCE(!rcu_read_lock_held());
+
 restart:
        napi_poll = NULL;
 
-       rcu_read_lock();
-
        napi = napi_by_id(napi_id);
        if (!napi)
-               goto out;
+               return;
 
        if (!IS_ENABLED(CONFIG_PREEMPT_RT))
                preempt_disable();
@@ -6254,14 +6258,14 @@ restart:
                         */
                        if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
                                   NAPIF_STATE_IN_BUSY_POLL)) {
-                               if (prefer_busy_poll)
+                               if (flags & NAPI_F_PREFER_BUSY_POLL)
                                        set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
                                goto count;
                        }
                        if (cmpxchg(&napi->state, val,
                                    val | NAPIF_STATE_IN_BUSY_POLL |
                                          NAPIF_STATE_SCHED) != val) {
-                               if (prefer_busy_poll)
+                               if (flags & NAPI_F_PREFER_BUSY_POLL)
                                        set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
                                goto count;
                        }
@@ -6282,11 +6286,12 @@ count:
 
                if (unlikely(need_resched())) {
                        if (napi_poll)
-                               busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
+                               busy_poll_stop(napi, have_poll_lock, flags, budget);
                        if (!IS_ENABLED(CONFIG_PREEMPT_RT))
                                preempt_enable();
                        rcu_read_unlock();
                        cond_resched();
+                       rcu_read_lock();
                        if (loop_end(loop_end_arg, start_time))
                                return;
                        goto restart;
@@ -6294,10 +6299,19 @@ count:
                cpu_relax();
        }
        if (napi_poll)
-               busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
+               busy_poll_stop(napi, have_poll_lock, flags, budget);
        if (!IS_ENABLED(CONFIG_PREEMPT_RT))
                preempt_enable();
-out:
+}
+
+void napi_busy_loop(unsigned int napi_id,
+                   bool (*loop_end)(void *, unsigned long),
+                   void *loop_end_arg, bool prefer_busy_poll, u16 budget)
+{
+       unsigned flags = prefer_busy_poll ? NAPI_F_PREFER_BUSY_POLL : 0;
+
+       rcu_read_lock();
+       __napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget);
        rcu_read_unlock();
 }
 EXPORT_SYMBOL(napi_busy_loop);