net: hold netdev instance lock during ndo_bpf

Cover the paths that come via bpf system call and XSK bind.

Cc: Saeed Mahameed <saeed@kernel.org>
Signed-off-by: Stanislav Fomichev <sdf@fomichev.me>
Link: https://patch.msgid.link/20250305163732.2766420-10-sdf@fomichev.me
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index c61b128095881aaac4b657dd097930f9dd3adff5..ca9c09dab14e6344a51fa77e109d47ba90e3ee3f 100644 (file)
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -4277,6 +4277,7 @@ struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
 
 int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
 u8 dev_xdp_prog_count(struct net_device *dev);
+int netif_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf);
 int dev_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf);
 u8 dev_xdp_sb_prog_count(struct net_device *dev);
 u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode);

diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c
index 1a4fec330eaac8fd508b64677fccd26e8ca0f276..a10153c3be2df675af49f0ad9c70b64b83593f45 100644 (file)
--- a/kernel/bpf/offload.c
+++ b/kernel/bpf/offload.c
@@ -528,10 +528,10 @@ struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
                return ERR_PTR(-ENOMEM);
 
        bpf_map_init_from_attr(&offmap->map, attr);
-
        rtnl_lock();
-       down_write(&bpf_devs_lock);
        offmap->netdev = __dev_get_by_index(net, attr->map_ifindex);
+       netdev_lock_ops(offmap->netdev);
+       down_write(&bpf_devs_lock);
        err = bpf_dev_offload_check(offmap->netdev);
        if (err)
                goto err_unlock;
@@ -548,12 +548,14 @@ struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
 
        list_add_tail(&offmap->offloads, &ondev->maps);
        up_write(&bpf_devs_lock);
+       netdev_unlock_ops(offmap->netdev);
        rtnl_unlock();
 
        return &offmap->map;
 
 err_unlock:
        up_write(&bpf_devs_lock);
+       netdev_unlock_ops(offmap->netdev);
        rtnl_unlock();
        bpf_map_area_free(offmap);
        return ERR_PTR(err);

diff --git a/net/core/dev.c b/net/core/dev.c
index 121c0449f87f71509714b7cee0da80afeea0afe9..404047d4d94320b8c81a1632dad393c89d3e3d58 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -9852,7 +9852,7 @@ u8 dev_xdp_sb_prog_count(struct net_device *dev)
        return count;
 }
 
-int dev_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf)
+int netif_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf)
 {
        if (!dev->netdev_ops->ndo_bpf)
                return -EOPNOTSUPP;
@@ -9872,7 +9872,6 @@ int dev_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf)
 
        return dev->netdev_ops->ndo_bpf(dev, bpf);
 }
-EXPORT_SYMBOL_GPL(dev_xdp_propagate);
 
 u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode)
 {
@@ -9902,6 +9901,8 @@ static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,
        struct netdev_bpf xdp;
        int err;
 
+       netdev_ops_assert_locked(dev);
+
        if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED &&
            prog && !prog->aux->xdp_has_frags) {
                NL_SET_ERR_MSG(extack, "unable to install XDP to device using tcp-data-split");
@@ -10134,7 +10135,9 @@ static void bpf_xdp_link_release(struct bpf_link *link)
         * already NULL, in which case link was already auto-detached
         */
        if (xdp_link->dev) {
+               netdev_lock_ops(xdp_link->dev);
                WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link));
+               netdev_unlock_ops(xdp_link->dev);
                xdp_link->dev = NULL;
        }
 
@@ -10216,10 +10219,12 @@ static int bpf_xdp_link_update(struct bpf_link *link, struct bpf_prog *new_prog,
                goto out_unlock;
        }
 
+       netdev_lock_ops(xdp_link->dev);
        mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags);
        bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode);
        err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL,
                              xdp_link->flags, new_prog);
+       netdev_unlock_ops(xdp_link->dev);
        if (err)
                goto out_unlock;
 
@@ -11005,7 +11010,9 @@ int register_netdevice(struct net_device *dev)
        if (ret)
                goto err_uninit_notify;
 
+       netdev_lock_ops(dev);
        __netdev_update_features(dev);
+       netdev_unlock_ops(dev);
 
        /*
         *      Default initial state at registry is that the
@@ -11945,7 +11952,9 @@ void unregister_netdevice_many_notify(struct list_head *head,
                /* Shutdown queueing discipline. */
                dev_shutdown(dev);
                dev_tcx_uninstall(dev);
+               netdev_lock_ops(dev);
                dev_xdp_uninstall(dev);
+               netdev_unlock_ops(dev);
                bpf_dev_bound_netdev_unregister(dev);
                dev_memory_provider_uninstall(dev);
 

diff --git a/net/core/dev_api.c b/net/core/dev_api.c
index 7bd667b34b808886c4bd2ddc70008ef8bfa7f3da..98db990ce21ce2a4f2913a8ce5924eac148d4386 100644 (file)
--- a/net/core/dev_api.c
+++ b/net/core/dev_api.c
@@ -317,3 +317,15 @@ int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,
        return ret;
 }
 EXPORT_SYMBOL(dev_set_mac_address);
+
+int dev_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf)
+{
+       int ret;
+
+       netdev_lock_ops(dev);
+       ret = netif_xdp_propagate(dev, bpf);
+       netdev_unlock_ops(dev);
+
+       return ret;
+}
+EXPORT_SYMBOL_GPL(dev_xdp_propagate);

diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index 84bf9f1d4bf21caed1ae8463bbb39359e5d9719b..f864e5d70b40af831ebb4ca0a8f112f15876887b 100644 (file)
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -1181,6 +1181,8 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
                goto out_release;
        }
 
+       netdev_lock_ops(dev);
+
        if (!xs->rx && !xs->tx) {
                err = -EINVAL;
                goto out_unlock;
@@ -1315,6 +1317,7 @@ out_unlock:
                smp_wmb();
                WRITE_ONCE(xs->state, XSK_BOUND);
        }
+       netdev_unlock_ops(dev);
 out_release:
        mutex_unlock(&xs->mutex);
        rtnl_unlock();

diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c
index c263fb7a68dc35e737c030d62cd25aadf552766b..0e6ca568fdeeb9d62f207604212b95d1afdddbf4 100644 (file)
--- a/net/xdp/xsk_buff_pool.c
+++ b/net/xdp/xsk_buff_pool.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 
+#include <linux/netdevice.h>
 #include <net/xsk_buff_pool.h>
 #include <net/xdp_sock.h>
 #include <net/xdp_sock_drv.h>
@@ -219,6 +220,7 @@ int xp_assign_dev(struct xsk_buff_pool *pool,
        bpf.xsk.pool = pool;
        bpf.xsk.queue_id = queue_id;
 
+       netdev_ops_assert_locked(netdev);
        err = netdev->netdev_ops->ndo_bpf(netdev, &bpf);
        if (err)
                goto err_unreg_pool;