Merge tag 'mips_5.3' of git://git.kernel.org/pub/scm/linux/kernel/git/mips/linux

[linux-2.6-block.git] / net / xdp / xsk.c

diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index 74417a851ed549871694fb5e44ad574dfacc9321..d4d6f10aa93646d280c23eb6f28b89582cbb922f 100644 (file)
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -129,13 +129,17 @@ int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
        u64 addr;
        int err;
 
-       if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)
-               return -EINVAL;
+       spin_lock_bh(&xs->rx_lock);
+
+       if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) {
+               err = -EINVAL;
+               goto out_unlock;
+       }
 
        if (!xskq_peek_addr(xs->umem->fq, &addr) ||
            len > xs->umem->chunk_size_nohr - XDP_PACKET_HEADROOM) {
-               xs->rx_dropped++;
-               return -ENOSPC;
+               err = -ENOSPC;
+               goto out_drop;
        }
 
        addr += xs->umem->headroom;
@@ -144,13 +148,21 @@ int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
        memcpy(buffer, xdp->data_meta, len + metalen);
        addr += metalen;
        err = xskq_produce_batch_desc(xs->rx, addr, len);
-       if (!err) {
-               xskq_discard_addr(xs->umem->fq);
-               xsk_flush(xs);
-               return 0;
-       }
+       if (err)
+               goto out_drop;
+
+       xskq_discard_addr(xs->umem->fq);
+       xskq_produce_flush_desc(xs->rx);
+
+       spin_unlock_bh(&xs->rx_lock);
 
+       xs->sk.sk_data_ready(&xs->sk);
+       return 0;
+
+out_drop:
        xs->rx_dropped++;
+out_unlock:
+       spin_unlock_bh(&xs->rx_lock);
        return err;
 }
 
@@ -337,6 +349,22 @@ static int xsk_init_queue(u32 entries, struct xsk_queue **queue,
        return 0;
 }
 
+static void xsk_unbind_dev(struct xdp_sock *xs)
+{
+       struct net_device *dev = xs->dev;
+
+       if (!dev || xs->state != XSK_BOUND)
+               return;
+
+       xs->state = XSK_UNBOUND;
+
+       /* Wait for driver to stop using the xdp socket. */
+       xdp_del_sk_umem(xs->umem, xs);
+       xs->dev = NULL;
+       synchronize_net();
+       dev_put(dev);
+}
+
 static int xsk_release(struct socket *sock)
 {
        struct sock *sk = sock->sk;
@@ -356,15 +384,7 @@ static int xsk_release(struct socket *sock)
        sock_prot_inuse_add(net, sk->sk_prot, -1);
        local_bh_enable();
 
-       if (xs->dev) {
-               struct net_device *dev = xs->dev;
-
-               /* Wait for driver to stop using the xdp socket. */
-               xdp_del_sk_umem(xs->umem, xs);
-               xs->dev = NULL;
-               synchronize_net();
-               dev_put(dev);
-       }
+       xsk_unbind_dev(xs);
 
        xskq_destroy(xs->rx);
        xskq_destroy(xs->tx);
@@ -414,7 +434,7 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
                return -EINVAL;
 
        mutex_lock(&xs->mutex);
-       if (xs->dev) {
+       if (xs->state != XSK_READY) {
                err = -EBUSY;
                goto out_release;
        }
@@ -494,6 +514,8 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
 out_unlock:
        if (err)
                dev_put(dev);
+       else
+               xs->state = XSK_BOUND;
 out_release:
        mutex_unlock(&xs->mutex);
        return err;
@@ -522,6 +544,10 @@ static int xsk_setsockopt(struct socket *sock, int level, int optname,
                        return -EFAULT;
 
                mutex_lock(&xs->mutex);
+               if (xs->state != XSK_READY) {
+                       mutex_unlock(&xs->mutex);
+                       return -EBUSY;
+               }
                q = (optname == XDP_TX_RING) ? &xs->tx : &xs->rx;
                err = xsk_init_queue(entries, q, false);
                mutex_unlock(&xs->mutex);
@@ -536,7 +562,7 @@ static int xsk_setsockopt(struct socket *sock, int level, int optname,
                        return -EFAULT;
 
                mutex_lock(&xs->mutex);
-               if (xs->umem) {
+               if (xs->state != XSK_READY || xs->umem) {
                        mutex_unlock(&xs->mutex);
                        return -EBUSY;
                }
@@ -563,6 +589,10 @@ static int xsk_setsockopt(struct socket *sock, int level, int optname,
                        return -EFAULT;
 
                mutex_lock(&xs->mutex);
+               if (xs->state != XSK_READY) {
+                       mutex_unlock(&xs->mutex);
+                       return -EBUSY;
+               }
                if (!xs->umem) {
                        mutex_unlock(&xs->mutex);
                        return -EINVAL;
@@ -684,6 +714,9 @@ static int xsk_mmap(struct file *file, struct socket *sock,
        unsigned long pfn;
        struct page *qpg;
 
+       if (xs->state != XSK_READY)
+               return -EBUSY;
+
        if (offset == XDP_PGOFF_RX_RING) {
                q = READ_ONCE(xs->rx);
        } else if (offset == XDP_PGOFF_TX_RING) {
@@ -715,6 +748,38 @@ static int xsk_mmap(struct file *file, struct socket *sock,
                               size, vma->vm_page_prot);
 }
 
+static int xsk_notifier(struct notifier_block *this,
+                       unsigned long msg, void *ptr)
+{
+       struct net_device *dev = netdev_notifier_info_to_dev(ptr);
+       struct net *net = dev_net(dev);
+       struct sock *sk;
+
+       switch (msg) {
+       case NETDEV_UNREGISTER:
+               mutex_lock(&net->xdp.lock);
+               sk_for_each(sk, &net->xdp.list) {
+                       struct xdp_sock *xs = xdp_sk(sk);
+
+                       mutex_lock(&xs->mutex);
+                       if (xs->dev == dev) {
+                               sk->sk_err = ENETDOWN;
+                               if (!sock_flag(sk, SOCK_DEAD))
+                                       sk->sk_error_report(sk);
+
+                               xsk_unbind_dev(xs);
+
+                               /* Clear device references in umem. */
+                               xdp_umem_clear_dev(xs->umem);
+                       }
+                       mutex_unlock(&xs->mutex);
+               }
+               mutex_unlock(&net->xdp.lock);
+               break;
+       }
+       return NOTIFY_DONE;
+}
+
 static struct proto xsk_proto = {
        .name =         "XDP",
        .owner =        THIS_MODULE,
@@ -786,7 +851,9 @@ static int xsk_create(struct net *net, struct socket *sock, int protocol,
        sock_set_flag(sk, SOCK_RCU_FREE);
 
        xs = xdp_sk(sk);
+       xs->state = XSK_READY;
        mutex_init(&xs->mutex);
+       spin_lock_init(&xs->rx_lock);
        spin_lock_init(&xs->tx_completion_lock);
 
        mutex_lock(&net->xdp.lock);
@@ -806,6 +873,10 @@ static const struct net_proto_family xsk_family_ops = {
        .owner  = THIS_MODULE,
 };
 
+static struct notifier_block xsk_netdev_notifier = {
+       .notifier_call  = xsk_notifier,
+};
+
 static int __net_init xsk_net_init(struct net *net)
 {
        mutex_init(&net->xdp.lock);
@@ -838,8 +909,15 @@ static int __init xsk_init(void)
        err = register_pernet_subsys(&xsk_net_ops);
        if (err)
                goto out_sk;
+
+       err = register_netdevice_notifier(&xsk_netdev_notifier);
+       if (err)
+               goto out_pernet;
+
        return 0;
 
+out_pernet:
+       unregister_pernet_subsys(&xsk_net_ops);
 out_sk:
        sock_unregister(PF_XDP);
 out_proto: