bpf: net_sched: Add a qdisc watchdog timer

Add a watchdog timer to bpf qdisc. The watchdog can be used to schedule
the execution of qdisc through kfunc, bpf_qdisc_schedule(). It can be
useful for building traffic shaping scheduling algorithm, where the time
the next packet will be dequeued is known.

The implementation relies on struct_ops gen_prologue/epilogue to patch bpf
programs provided by users. Operator specific prologue/epilogue kfuncs
are introduced instead of watchdog kfuncs so that it is easier to extend
prologue/epilogue in the future (writing C vs BPF bytecode).

Signed-off-by: Amery Hung <amery.hung@bytedance.com>
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Acked-by: Toke Høiland-Jørgensen <toke@redhat.com>
Link: https://patch.msgid.link/20250409214606.2000194-5-ameryhung@gmail.com

diff --git a/net/sched/bpf_qdisc.c b/net/sched/bpf_qdisc.c
index 5ef13cc6356024f1a6148e5e9651eabbef5e3415..0790f1248bd8282051632d6c59f6070034711a18 100644 (file)
--- a/net/sched/bpf_qdisc.c
+++ b/net/sched/bpf_qdisc.c
@@ -13,6 +13,10 @@
 
 static struct bpf_struct_ops bpf_Qdisc_ops;
 
+struct bpf_sched_data {
+       struct qdisc_watchdog watchdog;
+};
+
 struct bpf_sk_buff_ptr {
        struct sk_buff *skb;
 };
@@ -126,6 +130,56 @@ static int bpf_qdisc_btf_struct_access(struct bpf_verifier_log *log,
        return 0;
 }
 
+BTF_ID_LIST(bpf_qdisc_init_prologue_ids)
+BTF_ID(func, bpf_qdisc_init_prologue)
+
+static int bpf_qdisc_gen_prologue(struct bpf_insn *insn_buf, bool direct_write,
+                                 const struct bpf_prog *prog)
+{
+       struct bpf_insn *insn = insn_buf;
+
+       if (prog->aux->attach_st_ops_member_off != offsetof(struct Qdisc_ops, init))
+               return 0;
+
+       /* r6 = r1; // r6 will be "u64 *ctx". r1 is "u64 *ctx".
+        * r1 = r1[0]; // r1 will be "struct Qdisc *sch"
+        * r0 = bpf_qdisc_init_prologue(r1);
+        * r1 = r6; // r1 will be "u64 *ctx".
+        */
+       *insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
+       *insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0);
+       *insn++ = BPF_CALL_KFUNC(0, bpf_qdisc_init_prologue_ids[0]);
+       *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6);
+       *insn++ = prog->insnsi[0];
+
+       return insn - insn_buf;
+}
+
+BTF_ID_LIST(bpf_qdisc_reset_destroy_epilogue_ids)
+BTF_ID(func, bpf_qdisc_reset_destroy_epilogue)
+
+static int bpf_qdisc_gen_epilogue(struct bpf_insn *insn_buf, const struct bpf_prog *prog,
+                                 s16 ctx_stack_off)
+{
+       struct bpf_insn *insn = insn_buf;
+
+       if (prog->aux->attach_st_ops_member_off != offsetof(struct Qdisc_ops, reset) &&
+           prog->aux->attach_st_ops_member_off != offsetof(struct Qdisc_ops, destroy))
+               return 0;
+
+       /* r1 = stack[ctx_stack_off]; // r1 will be "u64 *ctx"
+        * r1 = r1[0]; // r1 will be "struct Qdisc *sch"
+        * r0 = bpf_qdisc_reset_destroy_epilogue(r1);
+        * BPF_EXIT;
+        */
+       *insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_FP, ctx_stack_off);
+       *insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0);
+       *insn++ = BPF_CALL_KFUNC(0, bpf_qdisc_reset_destroy_epilogue_ids[0]);
+       *insn++ = BPF_EXIT_INSN();
+
+       return insn - insn_buf;
+}
+
 __bpf_kfunc_start_defs();
 
 /* bpf_skb_get_hash - Get the flow hash of an skb.
@@ -154,6 +208,36 @@ __bpf_kfunc void bpf_qdisc_skb_drop(struct sk_buff *skb,
        __qdisc_drop(skb, (struct sk_buff **)to_free_list);
 }
 
+/* bpf_qdisc_watchdog_schedule - Schedule a qdisc to a later time using a timer.
+ * @sch: The qdisc to be scheduled.
+ * @expire: The expiry time of the timer.
+ * @delta_ns: The slack range of the timer.
+ */
+__bpf_kfunc void bpf_qdisc_watchdog_schedule(struct Qdisc *sch, u64 expire, u64 delta_ns)
+{
+       struct bpf_sched_data *q = qdisc_priv(sch);
+
+       qdisc_watchdog_schedule_range_ns(&q->watchdog, expire, delta_ns);
+}
+
+/* bpf_qdisc_init_prologue - Hidden kfunc called in prologue of .init. */
+__bpf_kfunc void bpf_qdisc_init_prologue(struct Qdisc *sch)
+{
+       struct bpf_sched_data *q = qdisc_priv(sch);
+
+       qdisc_watchdog_init(&q->watchdog, sch);
+}
+
+/* bpf_qdisc_reset_destroy_epilogue - Hidden kfunc called in epilogue of .reset
+ * and .destroy
+ */
+__bpf_kfunc void bpf_qdisc_reset_destroy_epilogue(struct Qdisc *sch)
+{
+       struct bpf_sched_data *q = qdisc_priv(sch);
+
+       qdisc_watchdog_cancel(&q->watchdog);
+}
+
 __bpf_kfunc_end_defs();
 
 BTF_KFUNCS_START(qdisc_kfunc_ids)
@@ -161,6 +245,9 @@ BTF_ID_FLAGS(func, bpf_skb_get_hash, KF_TRUSTED_ARGS)
 BTF_ID_FLAGS(func, bpf_kfree_skb, KF_RELEASE)
 BTF_ID_FLAGS(func, bpf_qdisc_skb_drop, KF_RELEASE)
 BTF_ID_FLAGS(func, bpf_dynptr_from_skb, KF_TRUSTED_ARGS)
+BTF_ID_FLAGS(func, bpf_qdisc_watchdog_schedule, KF_TRUSTED_ARGS)
+BTF_ID_FLAGS(func, bpf_qdisc_init_prologue, KF_TRUSTED_ARGS)
+BTF_ID_FLAGS(func, bpf_qdisc_reset_destroy_epilogue, KF_TRUSTED_ARGS)
 BTF_KFUNCS_END(qdisc_kfunc_ids)
 
 BTF_SET_START(qdisc_common_kfunc_set)
@@ -171,16 +258,22 @@ BTF_SET_END(qdisc_common_kfunc_set)
 
 BTF_SET_START(qdisc_enqueue_kfunc_set)
 BTF_ID(func, bpf_qdisc_skb_drop)
+BTF_ID(func, bpf_qdisc_watchdog_schedule)
 BTF_SET_END(qdisc_enqueue_kfunc_set)
 
+BTF_SET_START(qdisc_dequeue_kfunc_set)
+BTF_ID(func, bpf_qdisc_watchdog_schedule)
+BTF_SET_END(qdisc_dequeue_kfunc_set)
+
 enum qdisc_ops_kf_flags {
        QDISC_OPS_KF_COMMON             = 0,
        QDISC_OPS_KF_ENQUEUE            = 1 << 0,
+       QDISC_OPS_KF_DEQUEUE            = 1 << 1,
 };
 
 static const u32 qdisc_ops_context_flags[] = {
        [QDISC_OP_IDX(enqueue)]         = QDISC_OPS_KF_ENQUEUE,
-       [QDISC_OP_IDX(dequeue)]         = QDISC_OPS_KF_COMMON,
+       [QDISC_OP_IDX(dequeue)]         = QDISC_OPS_KF_DEQUEUE,
        [QDISC_OP_IDX(init)]            = QDISC_OPS_KF_COMMON,
        [QDISC_OP_IDX(reset)]           = QDISC_OPS_KF_COMMON,
        [QDISC_OP_IDX(destroy)]         = QDISC_OPS_KF_COMMON,
@@ -203,6 +296,10 @@ static int bpf_qdisc_kfunc_filter(const struct bpf_prog *prog, u32 kfunc_id)
            btf_id_set_contains(&qdisc_enqueue_kfunc_set, kfunc_id))
                return 0;
 
+       if ((flags & QDISC_OPS_KF_DEQUEUE) &&
+           btf_id_set_contains(&qdisc_dequeue_kfunc_set, kfunc_id))
+               return 0;
+
        if (btf_id_set_contains(&qdisc_common_kfunc_set, kfunc_id))
                return 0;
 
@@ -219,6 +316,8 @@ static const struct bpf_verifier_ops bpf_qdisc_verifier_ops = {
        .get_func_proto         = bpf_base_func_proto,
        .is_valid_access        = bpf_qdisc_is_valid_access,
        .btf_struct_access      = bpf_qdisc_btf_struct_access,
+       .gen_prologue           = bpf_qdisc_gen_prologue,
+       .gen_epilogue           = bpf_qdisc_gen_epilogue,
 };
 
 static int bpf_qdisc_init_member(const struct btf_type *t,
@@ -234,6 +333,11 @@ static int bpf_qdisc_init_member(const struct btf_type *t,
 
        moff = __btf_member_bit_offset(t, member) / 8;
        switch (moff) {
+       case offsetof(struct Qdisc_ops, priv_size):
+               if (uqdisc_ops->priv_size)
+                       return -EINVAL;
+               qdisc_ops->priv_size = sizeof(struct bpf_sched_data);
+               return 1;
        case offsetof(struct Qdisc_ops, peek):
                qdisc_ops->peek = qdisc_peek_dequeued;
                return 0;