[linux-2.6-block.git] / net / xdp / xsk.c

// SPDX-License-Identifier: GPL-2.0
/* XDP sockets
 *
 * AF_XDP sockets allows a channel between XDP programs and userspace
 * applications.
 * Copyright(c) 2018 Intel Corporation.
 *
 * Author(s): Björn Töpel <bjorn.topel@intel.com>
 *	      Magnus Karlsson <magnus.karlsson@intel.com>
 */

#define pr_fmt(fmt) "AF_XDP: %s: " fmt, __func__

#include <linux/if_xdp.h>
#include <linux/init.h>
#include <linux/sched/mm.h>
#include <linux/sched/signal.h>
#include <linux/sched/task.h>
#include <linux/socket.h>
#include <linux/file.h>
#include <linux/uaccess.h>
#include <linux/net.h>
#include <linux/netdevice.h>
#include <linux/rculist.h>
#include <net/xdp_sock.h>
#include <net/xdp.h>

#include "xsk_queue.h"
#include "xdp_umem.h"

#define TX_BATCH_SIZE 16

static struct xdp_sock *xdp_sk(struct sock *sk)
{
	return (struct xdp_sock *)sk;
}

bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs)
{
	return READ_ONCE(xs->rx) &&  READ_ONCE(xs->umem) &&
		READ_ONCE(xs->umem->fq);
}

u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr)
{
	return xskq_peek_addr(umem->fq, addr);
}
EXPORT_SYMBOL(xsk_umem_peek_addr);

void xsk_umem_discard_addr(struct xdp_umem *umem)
{
	xskq_discard_addr(umem->fq);
}
EXPORT_SYMBOL(xsk_umem_discard_addr);

static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
{
	void *to_buf, *from_buf;
	u32 metalen;
	u64 addr;
	int err;

	if (!xskq_peek_addr(xs->umem->fq, &addr) ||
	    len > xs->umem->chunk_size_nohr - XDP_PACKET_HEADROOM) {
		xs->rx_dropped++;
		return -ENOSPC;
	}

	addr += xs->umem->headroom;

	if (unlikely(xdp_data_meta_unsupported(xdp))) {
		from_buf = xdp->data;
		metalen = 0;
	} else {
		from_buf = xdp->data_meta;
		metalen = xdp->data - xdp->data_meta;
	}

	to_buf = xdp_umem_get_data(xs->umem, addr);
	memcpy(to_buf, from_buf, len + metalen);
	addr += metalen;
	err = xskq_produce_batch_desc(xs->rx, addr, len);
	if (!err) {
		xskq_discard_addr(xs->umem->fq);
		xdp_return_buff(xdp);
		return 0;
	}

	xs->rx_dropped++;
	return err;
}

static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
{
	int err = xskq_produce_batch_desc(xs->rx, (u64)xdp->handle, len);

	if (err)
		xs->rx_dropped++;

	return err;
}

int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
{
	u32 len;

	if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)
		return -EINVAL;

	len = xdp->data_end - xdp->data;

	return (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) ?
		__xsk_rcv_zc(xs, xdp, len) : __xsk_rcv(xs, xdp, len);
}

void xsk_flush(struct xdp_sock *xs)
{
	xskq_produce_flush_desc(xs->rx);
	xs->sk.sk_data_ready(&xs->sk);
}

int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
{
	u32 metalen = xdp->data - xdp->data_meta;
	u32 len = xdp->data_end - xdp->data;
	void *buffer;
	u64 addr;
	int err;

	if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)
		return -EINVAL;

	if (!xskq_peek_addr(xs->umem->fq, &addr) ||
	    len > xs->umem->chunk_size_nohr - XDP_PACKET_HEADROOM) {
		xs->rx_dropped++;
		return -ENOSPC;
	}

	addr += xs->umem->headroom;

	buffer = xdp_umem_get_data(xs->umem, addr);
	memcpy(buffer, xdp->data_meta, len + metalen);
	addr += metalen;
	err = xskq_produce_batch_desc(xs->rx, addr, len);
	if (!err) {
		xskq_discard_addr(xs->umem->fq);
		xsk_flush(xs);
		return 0;
	}

	xs->rx_dropped++;
	return err;
}

void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries)
{
	xskq_produce_flush_addr_n(umem->cq, nb_entries);
}
EXPORT_SYMBOL(xsk_umem_complete_tx);

void xsk_umem_consume_tx_done(struct xdp_umem *umem)
{
	struct xdp_sock *xs;

	rcu_read_lock();
	list_for_each_entry_rcu(xs, &umem->xsk_list, list) {
		xs->sk.sk_write_space(&xs->sk);
	}
	rcu_read_unlock();
}
EXPORT_SYMBOL(xsk_umem_consume_tx_done);

bool xsk_umem_consume_tx(struct xdp_umem *umem, dma_addr_t *dma, u32 *len)
{
	struct xdp_desc desc;
	struct xdp_sock *xs;

	rcu_read_lock();
	list_for_each_entry_rcu(xs, &umem->xsk_list, list) {
		if (!xskq_peek_desc(xs->tx, &desc))
			continue;

		if (xskq_produce_addr_lazy(umem->cq, desc.addr))
			goto out;

		*dma = xdp_umem_get_dma(umem, desc.addr);
		*len = desc.len;

		xskq_discard_desc(xs->tx);
		rcu_read_unlock();
		return true;
	}

out:
	rcu_read_unlock();
	return false;
}
EXPORT_SYMBOL(xsk_umem_consume_tx);

static int xsk_zc_xmit(struct sock *sk)
{
	struct xdp_sock *xs = xdp_sk(sk);
	struct net_device *dev = xs->dev;

	return dev->netdev_ops->ndo_xsk_async_xmit(dev, xs->queue_id);
}

static void xsk_destruct_skb(struct sk_buff *skb)
{
	u64 addr = (u64)(long)skb_shinfo(skb)->destructor_arg;
	struct xdp_sock *xs = xdp_sk(skb->sk);
	unsigned long flags;

	spin_lock_irqsave(&xs->tx_completion_lock, flags);
	WARN_ON_ONCE(xskq_produce_addr(xs->umem->cq, addr));
	spin_unlock_irqrestore(&xs->tx_completion_lock, flags);

	sock_wfree(skb);
}

static int xsk_generic_xmit(struct sock *sk, struct msghdr *m,
			    size_t total_len)
{
	u32 max_batch = TX_BATCH_SIZE;
	struct xdp_sock *xs = xdp_sk(sk);
	bool sent_frame = false;
	struct xdp_desc desc;
	struct sk_buff *skb;
	int err = 0;

	mutex_lock(&xs->mutex);

	while (xskq_peek_desc(xs->tx, &desc)) {
		char *buffer;
		u64 addr;
		u32 len;

		if (max_batch-- == 0) {
			err = -EAGAIN;
			goto out;
		}

		if (xskq_reserve_addr(xs->umem->cq))
			goto out;

		if (xs->queue_id >= xs->dev->real_num_tx_queues)
			goto out;

		len = desc.len;
		skb = sock_alloc_send_skb(sk, len, 1, &err);
		if (unlikely(!skb)) {
			err = -EAGAIN;
			goto out;
		}

		skb_put(skb, len);
		addr = desc.addr;
		buffer = xdp_umem_get_data(xs->umem, addr);
		err = skb_store_bits(skb, 0, buffer, len);
		if (unlikely(err)) {
			kfree_skb(skb);
			goto out;
		}

		skb->dev = xs->dev;
		skb->priority = sk->sk_priority;
		skb->mark = sk->sk_mark;
		skb_shinfo(skb)->destructor_arg = (void *)(long)addr;
		skb->destructor = xsk_destruct_skb;

		err = dev_direct_xmit(skb, xs->queue_id);
		xskq_discard_desc(xs->tx);
		/* Ignore NET_XMIT_CN as packet might have been sent */
		if (err == NET_XMIT_DROP || err == NETDEV_TX_BUSY) {
			/* SKB completed but not sent */
			err = -EBUSY;
			goto out;
		}

		sent_frame = true;
	}

out:
	if (sent_frame)
		sk->sk_write_space(sk);

	mutex_unlock(&xs->mutex);
	return err;
}

static int xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
{
	bool need_wait = !(m->msg_flags & MSG_DONTWAIT);
	struct sock *sk = sock->sk;
	struct xdp_sock *xs = xdp_sk(sk);

	if (unlikely(!xs->dev))
		return -ENXIO;
	if (unlikely(!(xs->dev->flags & IFF_UP)))
		return -ENETDOWN;
	if (unlikely(!xs->tx))
		return -ENOBUFS;
	if (need_wait)
		return -EOPNOTSUPP;

	return (xs->zc) ? xsk_zc_xmit(sk) : xsk_generic_xmit(sk, m, total_len);
}

static unsigned int xsk_poll(struct file *file, struct socket *sock,
			     struct poll_table_struct *wait)
{
	unsigned int mask = datagram_poll(file, sock, wait);
	struct sock *sk = sock->sk;
	struct xdp_sock *xs = xdp_sk(sk);

	if (xs->rx && !xskq_empty_desc(xs->rx))
		mask |= POLLIN | POLLRDNORM;
	if (xs->tx && !xskq_full_desc(xs->tx))
		mask |= POLLOUT | POLLWRNORM;

	return mask;
}

static int xsk_init_queue(u32 entries, struct xsk_queue **queue,
			  bool umem_queue)
{
	struct xsk_queue *q;

	if (entries == 0 || *queue || !is_power_of_2(entries))
		return -EINVAL;

	q = xskq_create(entries, umem_queue);
	if (!q)
		return -ENOMEM;

	/* Make sure queue is ready before it can be seen by others */
	smp_wmb();
	*queue = q;
	return 0;
}

static int xsk_release(struct socket *sock)
{
	struct sock *sk = sock->sk;
	struct xdp_sock *xs = xdp_sk(sk);
	struct net *net;

	if (!sk)
		return 0;

	net = sock_net(sk);

	local_bh_disable();
	sock_prot_inuse_add(net, sk->sk_prot, -1);
	local_bh_enable();

	if (xs->dev) {
		struct net_device *dev = xs->dev;

		/* Wait for driver to stop using the xdp socket. */
		xdp_del_sk_umem(xs->umem, xs);
		xs->dev = NULL;
		synchronize_net();
		dev_put(dev);
	}

	xskq_destroy(xs->rx);
	xskq_destroy(xs->tx);

	sock_orphan(sk);
	sock->sk = NULL;

	sk_refcnt_debug_release(sk);
	sock_put(sk);

	return 0;
}

static struct socket *xsk_lookup_xsk_from_fd(int fd)
{
	struct socket *sock;
	int err;

	sock = sockfd_lookup(fd, &err);
	if (!sock)
		return ERR_PTR(-ENOTSOCK);

	if (sock->sk->sk_family != PF_XDP) {
		sockfd_put(sock);
		return ERR_PTR(-ENOPROTOOPT);
	}

	return sock;
}

static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
{
	struct sockaddr_xdp *sxdp = (struct sockaddr_xdp *)addr;
	struct sock *sk = sock->sk;
	struct xdp_sock *xs = xdp_sk(sk);
	struct net_device *dev;
	u32 flags, qid;
	int err = 0;

	if (addr_len < sizeof(struct sockaddr_xdp))
		return -EINVAL;
	if (sxdp->sxdp_family != AF_XDP)
		return -EINVAL;

	mutex_lock(&xs->mutex);
	if (xs->dev) {
		err = -EBUSY;
		goto out_release;
	}

	dev = dev_get_by_index(sock_net(sk), sxdp->sxdp_ifindex);
	if (!dev) {
		err = -ENODEV;
		goto out_release;
	}

	if (!xs->rx && !xs->tx) {
		err = -EINVAL;
		goto out_unlock;
	}

	qid = sxdp->sxdp_queue_id;
	flags = sxdp->sxdp_flags;

	if (flags & XDP_SHARED_UMEM) {
		struct xdp_sock *umem_xs;
		struct socket *sock;

		if ((flags & XDP_COPY) || (flags & XDP_ZEROCOPY)) {
			/* Cannot specify flags for shared sockets. */
			err = -EINVAL;
			goto out_unlock;
		}

		if (xs->umem) {
			/* We have already our own. */
			err = -EINVAL;
			goto out_unlock;
		}

		sock = xsk_lookup_xsk_from_fd(sxdp->sxdp_shared_umem_fd);
		if (IS_ERR(sock)) {
			err = PTR_ERR(sock);
			goto out_unlock;
		}

		umem_xs = xdp_sk(sock->sk);
		if (!umem_xs->umem) {
			/* No umem to inherit. */
			err = -EBADF;
			sockfd_put(sock);
			goto out_unlock;
		} else if (umem_xs->dev != dev || umem_xs->queue_id != qid) {
			err = -EINVAL;
			sockfd_put(sock);
			goto out_unlock;
		}

		xdp_get_umem(umem_xs->umem);
		xs->umem = umem_xs->umem;
		sockfd_put(sock);
	} else if (!xs->umem || !xdp_umem_validate_queues(xs->umem)) {
		err = -EINVAL;
		goto out_unlock;
	} else {
		/* This xsk has its own umem. */
		xskq_set_umem(xs->umem->fq, xs->umem->size,
			      xs->umem->chunk_mask);
		xskq_set_umem(xs->umem->cq, xs->umem->size,
			      xs->umem->chunk_mask);

		err = xdp_umem_assign_dev(xs->umem, dev, qid, flags);
		if (err)
			goto out_unlock;
	}

	xs->dev = dev;
	xs->zc = xs->umem->zc;
	xs->queue_id = qid;
	xskq_set_umem(xs->rx, xs->umem->size, xs->umem->chunk_mask);
	xskq_set_umem(xs->tx, xs->umem->size, xs->umem->chunk_mask);
	xdp_add_sk_umem(xs->umem, xs);

out_unlock:
	if (err)
		dev_put(dev);
out_release:
	mutex_unlock(&xs->mutex);
	return err;
}

static int xsk_setsockopt(struct socket *sock, int level, int optname,
			  char __user *optval, unsigned int optlen)
{
	struct sock *sk = sock->sk;
	struct xdp_sock *xs = xdp_sk(sk);
	int err;

	if (level != SOL_XDP)
		return -ENOPROTOOPT;

	switch (optname) {
	case XDP_RX_RING:
	case XDP_TX_RING:
	{
		struct xsk_queue **q;
		int entries;

		if (optlen < sizeof(entries))
			return -EINVAL;
		if (copy_from_user(&entries, optval, sizeof(entries)))
			return -EFAULT;

		mutex_lock(&xs->mutex);
		q = (optname == XDP_TX_RING) ? &xs->tx : &xs->rx;
		err = xsk_init_queue(entries, q, false);
		mutex_unlock(&xs->mutex);
		return err;
	}
	case XDP_UMEM_REG:
	{
		struct xdp_umem_reg mr;
		struct xdp_umem *umem;

		if (copy_from_user(&mr, optval, sizeof(mr)))
			return -EFAULT;

		mutex_lock(&xs->mutex);
		if (xs->umem) {
			mutex_unlock(&xs->mutex);
			return -EBUSY;
		}

		umem = xdp_umem_create(&mr);
		if (IS_ERR(umem)) {
			mutex_unlock(&xs->mutex);
			return PTR_ERR(umem);
		}

		/* Make sure umem is ready before it can be seen by others */
		smp_wmb();
		xs->umem = umem;
		mutex_unlock(&xs->mutex);
		return 0;
	}
	case XDP_UMEM_FILL_RING:
	case XDP_UMEM_COMPLETION_RING:
	{
		struct xsk_queue **q;
		int entries;

		if (copy_from_user(&entries, optval, sizeof(entries)))
			return -EFAULT;

		mutex_lock(&xs->mutex);
		if (!xs->umem) {
			mutex_unlock(&xs->mutex);
			return -EINVAL;
		}

		q = (optname == XDP_UMEM_FILL_RING) ? &xs->umem->fq :
			&xs->umem->cq;
		err = xsk_init_queue(entries, q, true);
		mutex_unlock(&xs->mutex);
		return err;
	}
	default:
		break;
	}

	return -ENOPROTOOPT;
}

static int xsk_getsockopt(struct socket *sock, int level, int optname,
			  char __user *optval, int __user *optlen)
{
	struct sock *sk = sock->sk;
	struct xdp_sock *xs = xdp_sk(sk);
	int len;

	if (level != SOL_XDP)
		return -ENOPROTOOPT;

	if (get_user(len, optlen))
		return -EFAULT;
	if (len < 0)
		return -EINVAL;

	switch (optname) {
	case XDP_STATISTICS:
	{
		struct xdp_statistics stats;

		if (len < sizeof(stats))
			return -EINVAL;

		mutex_lock(&xs->mutex);
		stats.rx_dropped = xs->rx_dropped;
		stats.rx_invalid_descs = xskq_nb_invalid_descs(xs->rx);
		stats.tx_invalid_descs = xskq_nb_invalid_descs(xs->tx);
		mutex_unlock(&xs->mutex);

		if (copy_to_user(optval, &stats, sizeof(stats)))
			return -EFAULT;
		if (put_user(sizeof(stats), optlen))
			return -EFAULT;

		return 0;
	}
	case XDP_MMAP_OFFSETS:
	{
		struct xdp_mmap_offsets off;

		if (len < sizeof(off))
			return -EINVAL;

		off.rx.producer = offsetof(struct xdp_rxtx_ring, ptrs.producer);
		off.rx.consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer);
		off.rx.desc	= offsetof(struct xdp_rxtx_ring, desc);
		off.tx.producer = offsetof(struct xdp_rxtx_ring, ptrs.producer);
		off.tx.consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer);
		off.tx.desc	= offsetof(struct xdp_rxtx_ring, desc);

		off.fr.producer = offsetof(struct xdp_umem_ring, ptrs.producer);
		off.fr.consumer = offsetof(struct xdp_umem_ring, ptrs.consumer);
		off.fr.desc	= offsetof(struct xdp_umem_ring, desc);
		off.cr.producer = offsetof(struct xdp_umem_ring, ptrs.producer);
		off.cr.consumer = offsetof(struct xdp_umem_ring, ptrs.consumer);
		off.cr.desc	= offsetof(struct xdp_umem_ring, desc);

		len = sizeof(off);
		if (copy_to_user(optval, &off, len))
			return -EFAULT;
		if (put_user(len, optlen))
			return -EFAULT;

		return 0;
	}
	default:
		break;
	}

	return -EOPNOTSUPP;
}

static int xsk_mmap(struct file *file, struct socket *sock,
		    struct vm_area_struct *vma)
{
	loff_t offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
	unsigned long size = vma->vm_end - vma->vm_start;
	struct xdp_sock *xs = xdp_sk(sock->sk);
	struct xsk_queue *q = NULL;
	struct xdp_umem *umem;
	unsigned long pfn;
	struct page *qpg;

	if (offset == XDP_PGOFF_RX_RING) {
		q = READ_ONCE(xs->rx);
	} else if (offset == XDP_PGOFF_TX_RING) {
		q = READ_ONCE(xs->tx);
	} else {
		umem = READ_ONCE(xs->umem);
		if (!umem)
			return -EINVAL;

		if (offset == XDP_UMEM_PGOFF_FILL_RING)
			q = READ_ONCE(umem->fq);
		else if (offset == XDP_UMEM_PGOFF_COMPLETION_RING)
			q = READ_ONCE(umem->cq);
	}

	if (!q)
		return -EINVAL;

	qpg = virt_to_head_page(q->ring);
	if (size > (PAGE_SIZE << compound_order(qpg)))
		return -EINVAL;

	pfn = virt_to_phys(q->ring) >> PAGE_SHIFT;
	return remap_pfn_range(vma, vma->vm_start, pfn,
			       size, vma->vm_page_prot);
}

static struct proto xsk_proto = {
	.name =		"XDP",
	.owner =	THIS_MODULE,
	.obj_size =	sizeof(struct xdp_sock),
};

static const struct proto_ops xsk_proto_ops = {
	.family		= PF_XDP,
	.owner		= THIS_MODULE,
	.release	= xsk_release,
	.bind		= xsk_bind,
	.connect	= sock_no_connect,
	.socketpair	= sock_no_socketpair,
	.accept		= sock_no_accept,
	.getname	= sock_no_getname,
	.poll		= xsk_poll,
	.ioctl		= sock_no_ioctl,
	.listen		= sock_no_listen,
	.shutdown	= sock_no_shutdown,
	.setsockopt	= xsk_setsockopt,
	.getsockopt	= xsk_getsockopt,
	.sendmsg	= xsk_sendmsg,
	.recvmsg	= sock_no_recvmsg,
	.mmap		= xsk_mmap,
	.sendpage	= sock_no_sendpage,
};

static void xsk_destruct(struct sock *sk)
{
	struct xdp_sock *xs = xdp_sk(sk);

	if (!sock_flag(sk, SOCK_DEAD))
		return;

	xdp_put_umem(xs->umem);

	sk_refcnt_debug_dec(sk);
}

static int xsk_create(struct net *net, struct socket *sock, int protocol,
		      int kern)
{
	struct sock *sk;
	struct xdp_sock *xs;

	if (!ns_capable(net->user_ns, CAP_NET_RAW))
		return -EPERM;
	if (sock->type != SOCK_RAW)
		return -ESOCKTNOSUPPORT;

	if (protocol)
		return -EPROTONOSUPPORT;

	sock->state = SS_UNCONNECTED;

	sk = sk_alloc(net, PF_XDP, GFP_KERNEL, &xsk_proto, kern);
	if (!sk)
		return -ENOBUFS;

	sock->ops = &xsk_proto_ops;

	sock_init_data(sock, sk);

	sk->sk_family = PF_XDP;

	sk->sk_destruct = xsk_destruct;
	sk_refcnt_debug_inc(sk);

	sock_set_flag(sk, SOCK_RCU_FREE);

	xs = xdp_sk(sk);
	mutex_init(&xs->mutex);
	spin_lock_init(&xs->tx_completion_lock);

	local_bh_disable();
	sock_prot_inuse_add(net, &xsk_proto, 1);
	local_bh_enable();

	return 0;
}

static const struct net_proto_family xsk_family_ops = {
	.family = PF_XDP,
	.create = xsk_create,
	.owner	= THIS_MODULE,
};

static int __init xsk_init(void)
{
	int err;

	err = proto_register(&xsk_proto, 0 /* no slab */);
	if (err)
		goto out;

	err = sock_register(&xsk_family_ops);
	if (err)
		goto out_proto;

	return 0;

out_proto:
	proto_unregister(&xsk_proto);
out:
	return err;
}

fs_initcall(xsk_init);
Commit	Line	Data
c0c77d8f BT	1	// SPDX-License-Identifier: GPL-2.0
	2	/* XDP sockets
	3	*
	4	* AF_XDP sockets allows a channel between XDP programs and userspace
	5	* applications.
	6	* Copyright(c) 2018 Intel Corporation.
	7	*
c0c77d8f BT	8	* Author(s): Björn Töpel <bjorn.topel@intel.com>
	9	* Magnus Karlsson <magnus.karlsson@intel.com>
	10	*/
	11
	12	#define pr_fmt(fmt) "AF_XDP: %s: " fmt, __func__
	13
	14	#include <linux/if_xdp.h>
	15	#include <linux/init.h>
	16	#include <linux/sched/mm.h>
	17	#include <linux/sched/signal.h>
	18	#include <linux/sched/task.h>
	19	#include <linux/socket.h>
	20	#include <linux/file.h>
	21	#include <linux/uaccess.h>
	22	#include <linux/net.h>
	23	#include <linux/netdevice.h>
ac98d8aa	24	#include <linux/rculist.h>
c0c77d8f	25	#include <net/xdp_sock.h>
b9b6b68e	26	#include <net/xdp.h>
c0c77d8f	27
423f3832	28	#include "xsk_queue.h"
c0c77d8f BT	29	#include "xdp_umem.h"
c0c77d8f BT	30
35fcde7f MK	31	#define TX_BATCH_SIZE 16
35fcde7f MK	32
c0c77d8f BT	33	static struct xdp_sock xdp_sk(struct sock sk)
	34	{
	35	return (struct xdp_sock *)sk;
	36	}
	37
fbfc504a BT	38	bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs)
fbfc504a BT	39	{
173d3adb BT	40	return READ_ONCE(xs->rx) && READ_ONCE(xs->umem) &&
173d3adb BT	41	READ_ONCE(xs->umem->fq);
fbfc504a BT	42	}
fbfc504a BT	43
173d3adb BT	44	u64 xsk_umem_peek_addr(struct xdp_umem umem, u64 *addr)
	45	{
	46	return xskq_peek_addr(umem->fq, addr);
	47	}
	48	EXPORT_SYMBOL(xsk_umem_peek_addr);
	49
	50	void xsk_umem_discard_addr(struct xdp_umem *umem)
	51	{
	52	xskq_discard_addr(umem->fq);
	53	}
	54	EXPORT_SYMBOL(xsk_umem_discard_addr);
	55
	56	static int __xsk_rcv(struct xdp_sock xs, struct xdp_buff xdp, u32 len)
c497176c	57	{
18baed26 BT	58	void to_buf, from_buf;
18baed26 BT	59	u32 metalen;
bbff2f32	60	u64 addr;
4e64c835	61	int err;
c497176c	62
bbff2f32	63	if (!xskq_peek_addr(xs->umem->fq, &addr) \|\|
18baed26	64	len > xs->umem->chunk_size_nohr - XDP_PACKET_HEADROOM) {
a509a955	65	xs->rx_dropped++;
c497176c	66	return -ENOSPC;
a509a955	67	}
c497176c	68
bbff2f32 BT	69	addr += xs->umem->headroom;
bbff2f32 BT	70
18baed26 BT	71	if (unlikely(xdp_data_meta_unsupported(xdp))) {
	72	from_buf = xdp->data;
	73	metalen = 0;
	74	} else {
	75	from_buf = xdp->data_meta;
	76	metalen = xdp->data - xdp->data_meta;
	77	}
	78
	79	to_buf = xdp_umem_get_data(xs->umem, addr);
	80	memcpy(to_buf, from_buf, len + metalen);
	81	addr += metalen;
bbff2f32	82	err = xskq_produce_batch_desc(xs->rx, addr, len);
173d3adb	83	if (!err) {
bbff2f32	84	xskq_discard_addr(xs->umem->fq);
173d3adb BT	85	xdp_return_buff(xdp);
	86	return 0;
	87	}
c497176c	88
173d3adb	89	xs->rx_dropped++;
c497176c BT	90	return err;
	91	}
	92
173d3adb	93	static int __xsk_rcv_zc(struct xdp_sock xs, struct xdp_buff xdp, u32 len)
c497176c	94	{
173d3adb	95	int err = xskq_produce_batch_desc(xs->rx, (u64)xdp->handle, len);
c497176c	96
2d55d614	97	if (err)
173d3adb	98	xs->rx_dropped++;
c497176c BT	99
	100	return err;
	101	}
	102
173d3adb BT	103	int xsk_rcv(struct xdp_sock xs, struct xdp_buff xdp)
	104	{
	105	u32 len;
	106
	107	if (xs->dev != xdp->rxq->dev \|\| xs->queue_id != xdp->rxq->queue_index)
	108	return -EINVAL;
	109
	110	len = xdp->data_end - xdp->data;
	111
	112	return (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) ?
	113	__xsk_rcv_zc(xs, xdp, len) : __xsk_rcv(xs, xdp, len);
	114	}
	115
c497176c BT	116	void xsk_flush(struct xdp_sock *xs)
	117	{
	118	xskq_produce_flush_desc(xs->rx);
	119	xs->sk.sk_data_ready(&xs->sk);
	120	}
	121
	122	int xsk_generic_rcv(struct xdp_sock xs, struct xdp_buff xdp)
	123	{
18baed26	124	u32 metalen = xdp->data - xdp->data_meta;
173d3adb BT	125	u32 len = xdp->data_end - xdp->data;
	126	void *buffer;
	127	u64 addr;
c497176c BT	128	int err;
c497176c BT	129
5d902372 BT	130	if (xs->dev != xdp->rxq->dev \|\| xs->queue_id != xdp->rxq->queue_index)
	131	return -EINVAL;
	132
173d3adb	133	if (!xskq_peek_addr(xs->umem->fq, &addr) \|\|
18baed26	134	len > xs->umem->chunk_size_nohr - XDP_PACKET_HEADROOM) {
173d3adb BT	135	xs->rx_dropped++;
	136	return -ENOSPC;
	137	}
	138
	139	addr += xs->umem->headroom;
	140
	141	buffer = xdp_umem_get_data(xs->umem, addr);
18baed26 BT	142	memcpy(buffer, xdp->data_meta, len + metalen);
18baed26 BT	143	addr += metalen;
173d3adb BT	144	err = xskq_produce_batch_desc(xs->rx, addr, len);
	145	if (!err) {
	146	xskq_discard_addr(xs->umem->fq);
c497176c	147	xsk_flush(xs);
173d3adb BT	148	return 0;
173d3adb BT	149	}
c497176c	150
173d3adb	151	xs->rx_dropped++;
c497176c BT	152	return err;
	153	}
	154
ac98d8aa MK	155	void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries)
	156	{
	157	xskq_produce_flush_addr_n(umem->cq, nb_entries);
	158	}
	159	EXPORT_SYMBOL(xsk_umem_complete_tx);
	160
	161	void xsk_umem_consume_tx_done(struct xdp_umem *umem)
	162	{
	163	struct xdp_sock *xs;
	164
	165	rcu_read_lock();
	166	list_for_each_entry_rcu(xs, &umem->xsk_list, list) {
	167	xs->sk.sk_write_space(&xs->sk);
	168	}
	169	rcu_read_unlock();
	170	}
	171	EXPORT_SYMBOL(xsk_umem_consume_tx_done);
	172
	173	bool xsk_umem_consume_tx(struct xdp_umem umem, dma_addr_t dma, u32 *len)
	174	{
	175	struct xdp_desc desc;
	176	struct xdp_sock *xs;
	177
	178	rcu_read_lock();
	179	list_for_each_entry_rcu(xs, &umem->xsk_list, list) {
	180	if (!xskq_peek_desc(xs->tx, &desc))
	181	continue;
	182
	183	if (xskq_produce_addr_lazy(umem->cq, desc.addr))
	184	goto out;
	185
	186	*dma = xdp_umem_get_dma(umem, desc.addr);
	187	*len = desc.len;
	188
	189	xskq_discard_desc(xs->tx);
	190	rcu_read_unlock();
	191	return true;
	192	}
	193
	194	out:
	195	rcu_read_unlock();
	196	return false;
	197	}
	198	EXPORT_SYMBOL(xsk_umem_consume_tx);
	199
	200	static int xsk_zc_xmit(struct sock *sk)
	201	{
	202	struct xdp_sock *xs = xdp_sk(sk);
	203	struct net_device *dev = xs->dev;
	204
	205	return dev->netdev_ops->ndo_xsk_async_xmit(dev, xs->queue_id);
	206	}
	207
35fcde7f MK	208	static void xsk_destruct_skb(struct sk_buff *skb)
35fcde7f MK	209	{
bbff2f32	210	u64 addr = (u64)(long)skb_shinfo(skb)->destructor_arg;
35fcde7f	211	struct xdp_sock *xs = xdp_sk(skb->sk);
a9744f7c	212	unsigned long flags;
35fcde7f	213
a9744f7c	214	spin_lock_irqsave(&xs->tx_completion_lock, flags);
bbff2f32	215	WARN_ON_ONCE(xskq_produce_addr(xs->umem->cq, addr));
a9744f7c	216	spin_unlock_irqrestore(&xs->tx_completion_lock, flags);
35fcde7f MK	217
	218	sock_wfree(skb);
	219	}
	220
	221	static int xsk_generic_xmit(struct sock sk, struct msghdr m,
	222	size_t total_len)
	223	{
35fcde7f MK	224	u32 max_batch = TX_BATCH_SIZE;
	225	struct xdp_sock *xs = xdp_sk(sk);
	226	bool sent_frame = false;
	227	struct xdp_desc desc;
	228	struct sk_buff *skb;
	229	int err = 0;
	230
35fcde7f MK	231	mutex_lock(&xs->mutex);
	232
	233	while (xskq_peek_desc(xs->tx, &desc)) {
	234	char *buffer;
bbff2f32 BT	235	u64 addr;
bbff2f32 BT	236	u32 len;
35fcde7f MK	237
	238	if (max_batch-- == 0) {
	239	err = -EAGAIN;
	240	goto out;
	241	}
	242
9684f5e7	243	if (xskq_reserve_addr(xs->umem->cq))
35fcde7f	244	goto out;
35fcde7f	245
509d7648	246	if (xs->queue_id >= xs->dev->real_num_tx_queues)
2e59dd5e	247	goto out;
2e59dd5e	248
09210c4b	249	len = desc.len;
ac98d8aa	250	skb = sock_alloc_send_skb(sk, len, 1, &err);
35fcde7f MK	251	if (unlikely(!skb)) {
	252	err = -EAGAIN;
	253	goto out;
	254	}
	255
	256	skb_put(skb, len);
bbff2f32 BT	257	addr = desc.addr;
bbff2f32 BT	258	buffer = xdp_umem_get_data(xs->umem, addr);
35fcde7f MK	259	err = skb_store_bits(skb, 0, buffer, len);
	260	if (unlikely(err)) {
	261	kfree_skb(skb);
	262	goto out;
	263	}
	264
	265	skb->dev = xs->dev;
	266	skb->priority = sk->sk_priority;
	267	skb->mark = sk->sk_mark;
bbff2f32	268	skb_shinfo(skb)->destructor_arg = (void *)(long)addr;
35fcde7f MK	269	skb->destructor = xsk_destruct_skb;
	270
	271	err = dev_direct_xmit(skb, xs->queue_id);
fe588685	272	xskq_discard_desc(xs->tx);
35fcde7f MK	273	/* Ignore NET_XMIT_CN as packet might have been sent */
35fcde7f MK	274	if (err == NET_XMIT_DROP \|\| err == NETDEV_TX_BUSY) {
fe588685 MK	275	/* SKB completed but not sent */
fe588685 MK	276	err = -EBUSY;
35fcde7f MK	277	goto out;
	278	}
	279
	280	sent_frame = true;
35fcde7f MK	281	}
	282
	283	out:
	284	if (sent_frame)
	285	sk->sk_write_space(sk);
	286
	287	mutex_unlock(&xs->mutex);
	288	return err;
	289	}
	290
	291	static int xsk_sendmsg(struct socket sock, struct msghdr m, size_t total_len)
	292	{
ac98d8aa	293	bool need_wait = !(m->msg_flags & MSG_DONTWAIT);
35fcde7f MK	294	struct sock *sk = sock->sk;
	295	struct xdp_sock *xs = xdp_sk(sk);
	296
	297	if (unlikely(!xs->dev))
	298	return -ENXIO;
	299	if (unlikely(!(xs->dev->flags & IFF_UP)))
	300	return -ENETDOWN;
6efb4436 MK	301	if (unlikely(!xs->tx))
6efb4436 MK	302	return -ENOBUFS;
ac98d8aa MK	303	if (need_wait)
ac98d8aa MK	304	return -EOPNOTSUPP;
35fcde7f	305
ac98d8aa	306	return (xs->zc) ? xsk_zc_xmit(sk) : xsk_generic_xmit(sk, m, total_len);
35fcde7f MK	307	}
35fcde7f MK	308
a11e1d43 LT	309	static unsigned int xsk_poll(struct file file, struct socket sock,
a11e1d43 LT	310	struct poll_table_struct *wait)
c497176c	311	{
a11e1d43	312	unsigned int mask = datagram_poll(file, sock, wait);
c497176c BT	313	struct sock *sk = sock->sk;
	314	struct xdp_sock *xs = xdp_sk(sk);
	315
	316	if (xs->rx && !xskq_empty_desc(xs->rx))
	317	mask \|= POLLIN \| POLLRDNORM;
35fcde7f MK	318	if (xs->tx && !xskq_full_desc(xs->tx))
35fcde7f MK	319	mask \|= POLLOUT \| POLLWRNORM;
c497176c BT	320
	321	return mask;
	322	}
	323
b9b6b68e BT	324	static int xsk_init_queue(u32 entries, struct xsk_queue **queue,
b9b6b68e BT	325	bool umem_queue)
423f3832 MK	326	{
	327	struct xsk_queue *q;
	328
	329	if (entries == 0 \|\| *queue \|\| !is_power_of_2(entries))
	330	return -EINVAL;
	331
b9b6b68e	332	q = xskq_create(entries, umem_queue);
423f3832 MK	333	if (!q)
	334	return -ENOMEM;
	335
37b07693 BT	336	/* Make sure queue is ready before it can be seen by others */
37b07693 BT	337	smp_wmb();
423f3832 MK	338	*queue = q;
	339	return 0;
	340	}
	341
c0c77d8f BT	342	static int xsk_release(struct socket *sock)
	343	{
	344	struct sock *sk = sock->sk;
965a9909	345	struct xdp_sock *xs = xdp_sk(sk);
c0c77d8f BT	346	struct net *net;
	347
	348	if (!sk)
	349	return 0;
	350
	351	net = sock_net(sk);
	352
	353	local_bh_disable();
	354	sock_prot_inuse_add(net, sk->sk_prot, -1);
	355	local_bh_enable();
	356
965a9909	357	if (xs->dev) {
541d7fdd BT	358	struct net_device *dev = xs->dev;
541d7fdd BT	359
959b71db	360	/* Wait for driver to stop using the xdp socket. */
541d7fdd	361	xdp_del_sk_umem(xs->umem, xs);
965a9909	362	xs->dev = NULL;
541d7fdd BT	363	synchronize_net();
541d7fdd BT	364	dev_put(dev);
965a9909 MK	365	}
965a9909 MK	366
541d7fdd BT	367	xskq_destroy(xs->rx);
	368	xskq_destroy(xs->tx);
	369
c0c77d8f BT	370	sock_orphan(sk);
	371	sock->sk = NULL;
	372
	373	sk_refcnt_debug_release(sk);
	374	sock_put(sk);
	375
	376	return 0;
	377	}
	378
965a9909 MK	379	static struct socket *xsk_lookup_xsk_from_fd(int fd)
	380	{
	381	struct socket *sock;
	382	int err;
	383
	384	sock = sockfd_lookup(fd, &err);
	385	if (!sock)
	386	return ERR_PTR(-ENOTSOCK);
	387
	388	if (sock->sk->sk_family != PF_XDP) {
	389	sockfd_put(sock);
	390	return ERR_PTR(-ENOPROTOOPT);
	391	}
	392
	393	return sock;
	394	}
	395
	396	static int xsk_bind(struct socket sock, struct sockaddr addr, int addr_len)
	397	{
	398	struct sockaddr_xdp sxdp = (struct sockaddr_xdp )addr;
	399	struct sock *sk = sock->sk;
965a9909	400	struct xdp_sock *xs = xdp_sk(sk);
959b71db	401	struct net_device *dev;
173d3adb	402	u32 flags, qid;
965a9909 MK	403	int err = 0;
	404
	405	if (addr_len < sizeof(struct sockaddr_xdp))
	406	return -EINVAL;
	407	if (sxdp->sxdp_family != AF_XDP)
	408	return -EINVAL;
	409
	410	mutex_lock(&xs->mutex);
959b71db BT	411	if (xs->dev) {
	412	err = -EBUSY;
	413	goto out_release;
	414	}
	415
965a9909 MK	416	dev = dev_get_by_index(sock_net(sk), sxdp->sxdp_ifindex);
	417	if (!dev) {
	418	err = -ENODEV;
	419	goto out_release;
	420	}
	421
f6145903	422	if (!xs->rx && !xs->tx) {
965a9909 MK	423	err = -EINVAL;
	424	goto out_unlock;
	425	}
	426
173d3adb	427	qid = sxdp->sxdp_queue_id;
173d3adb BT	428	flags = sxdp->sxdp_flags;
	429
	430	if (flags & XDP_SHARED_UMEM) {
965a9909 MK	431	struct xdp_sock *umem_xs;
	432	struct socket *sock;
	433
173d3adb BT	434	if ((flags & XDP_COPY) \|\| (flags & XDP_ZEROCOPY)) {
	435	/* Cannot specify flags for shared sockets. */
	436	err = -EINVAL;
	437	goto out_unlock;
	438	}
	439
965a9909 MK	440	if (xs->umem) {
	441	/* We have already our own. */
	442	err = -EINVAL;
	443	goto out_unlock;
	444	}
	445
	446	sock = xsk_lookup_xsk_from_fd(sxdp->sxdp_shared_umem_fd);
	447	if (IS_ERR(sock)) {
	448	err = PTR_ERR(sock);
	449	goto out_unlock;
	450	}
	451
	452	umem_xs = xdp_sk(sock->sk);
	453	if (!umem_xs->umem) {
	454	/* No umem to inherit. */
	455	err = -EBADF;
	456	sockfd_put(sock);
	457	goto out_unlock;
173d3adb	458	} else if (umem_xs->dev != dev \|\| umem_xs->queue_id != qid) {
965a9909 MK	459	err = -EINVAL;
	460	sockfd_put(sock);
	461	goto out_unlock;
	462	}
	463
	464	xdp_get_umem(umem_xs->umem);
965a9909 MK	465	xs->umem = umem_xs->umem;
	466	sockfd_put(sock);
	467	} else if (!xs->umem \|\| !xdp_umem_validate_queues(xs->umem)) {
	468	err = -EINVAL;
	469	goto out_unlock;
c497176c BT	470	} else {
c497176c BT	471	/* This xsk has its own umem. */
93ee30f3 MK	472	xskq_set_umem(xs->umem->fq, xs->umem->size,
	473	xs->umem->chunk_mask);
	474	xskq_set_umem(xs->umem->cq, xs->umem->size,
	475	xs->umem->chunk_mask);
173d3adb BT	476
	477	err = xdp_umem_assign_dev(xs->umem, dev, qid, flags);
	478	if (err)
	479	goto out_unlock;
965a9909 MK	480	}
965a9909 MK	481
965a9909	482	xs->dev = dev;
ac98d8aa MK	483	xs->zc = xs->umem->zc;
ac98d8aa MK	484	xs->queue_id = qid;
93ee30f3 MK	485	xskq_set_umem(xs->rx, xs->umem->size, xs->umem->chunk_mask);
93ee30f3 MK	486	xskq_set_umem(xs->tx, xs->umem->size, xs->umem->chunk_mask);
ac98d8aa	487	xdp_add_sk_umem(xs->umem, xs);
965a9909 MK	488
	489	out_unlock:
	490	if (err)
	491	dev_put(dev);
	492	out_release:
	493	mutex_unlock(&xs->mutex);
	494	return err;
	495	}
	496
c0c77d8f BT	497	static int xsk_setsockopt(struct socket *sock, int level, int optname,
	498	char __user *optval, unsigned int optlen)
	499	{
	500	struct sock *sk = sock->sk;
	501	struct xdp_sock *xs = xdp_sk(sk);
	502	int err;
	503
	504	if (level != SOL_XDP)
	505	return -ENOPROTOOPT;
	506
	507	switch (optname) {
b9b6b68e	508	case XDP_RX_RING:
f6145903	509	case XDP_TX_RING:
b9b6b68e BT	510	{
	511	struct xsk_queue **q;
	512	int entries;
	513
	514	if (optlen < sizeof(entries))
	515	return -EINVAL;
	516	if (copy_from_user(&entries, optval, sizeof(entries)))
	517	return -EFAULT;
	518
	519	mutex_lock(&xs->mutex);
f6145903	520	q = (optname == XDP_TX_RING) ? &xs->tx : &xs->rx;
b9b6b68e BT	521	err = xsk_init_queue(entries, q, false);
	522	mutex_unlock(&xs->mutex);
	523	return err;
	524	}
c0c77d8f BT	525	case XDP_UMEM_REG:
	526	{
	527	struct xdp_umem_reg mr;
	528	struct xdp_umem *umem;
	529
c0c77d8f BT	530	if (copy_from_user(&mr, optval, sizeof(mr)))
	531	return -EFAULT;
	532
	533	mutex_lock(&xs->mutex);
a49049ea BT	534	if (xs->umem) {
	535	mutex_unlock(&xs->mutex);
	536	return -EBUSY;
	537	}
c0c77d8f	538
a49049ea BT	539	umem = xdp_umem_create(&mr);
a49049ea BT	540	if (IS_ERR(umem)) {
c0c77d8f	541	mutex_unlock(&xs->mutex);
a49049ea	542	return PTR_ERR(umem);
c0c77d8f BT	543	}
	544
	545	/* Make sure umem is ready before it can be seen by others */
	546	smp_wmb();
c0c77d8f BT	547	xs->umem = umem;
	548	mutex_unlock(&xs->mutex);
	549	return 0;
	550	}
423f3832	551	case XDP_UMEM_FILL_RING:
fe230832	552	case XDP_UMEM_COMPLETION_RING:
423f3832 MK	553	{
	554	struct xsk_queue **q;
	555	int entries;
	556
423f3832 MK	557	if (copy_from_user(&entries, optval, sizeof(entries)))
	558	return -EFAULT;
	559
	560	mutex_lock(&xs->mutex);
a49049ea BT	561	if (!xs->umem) {
	562	mutex_unlock(&xs->mutex);
	563	return -EINVAL;
	564	}
	565
fe230832 MK	566	q = (optname == XDP_UMEM_FILL_RING) ? &xs->umem->fq :
fe230832 MK	567	&xs->umem->cq;
b9b6b68e	568	err = xsk_init_queue(entries, q, true);
423f3832 MK	569	mutex_unlock(&xs->mutex);
	570	return err;
	571	}
c0c77d8f BT	572	default:
	573	break;
	574	}
	575
	576	return -ENOPROTOOPT;
	577	}
	578
af75d9e0 MK	579	static int xsk_getsockopt(struct socket *sock, int level, int optname,
	580	char __user optval, int __user optlen)
	581	{
	582	struct sock *sk = sock->sk;
	583	struct xdp_sock *xs = xdp_sk(sk);
	584	int len;
	585
	586	if (level != SOL_XDP)
	587	return -ENOPROTOOPT;
	588
	589	if (get_user(len, optlen))
	590	return -EFAULT;
	591	if (len < 0)
	592	return -EINVAL;
	593
	594	switch (optname) {
	595	case XDP_STATISTICS:
	596	{
	597	struct xdp_statistics stats;
	598
	599	if (len < sizeof(stats))
	600	return -EINVAL;
	601
	602	mutex_lock(&xs->mutex);
	603	stats.rx_dropped = xs->rx_dropped;
	604	stats.rx_invalid_descs = xskq_nb_invalid_descs(xs->rx);
	605	stats.tx_invalid_descs = xskq_nb_invalid_descs(xs->tx);
	606	mutex_unlock(&xs->mutex);
	607
	608	if (copy_to_user(optval, &stats, sizeof(stats)))
	609	return -EFAULT;
	610	if (put_user(sizeof(stats), optlen))
	611	return -EFAULT;
	612
	613	return 0;
	614	}
b3a9e0be BT	615	case XDP_MMAP_OFFSETS:
	616	{
	617	struct xdp_mmap_offsets off;
	618
	619	if (len < sizeof(off))
	620	return -EINVAL;
	621
	622	off.rx.producer = offsetof(struct xdp_rxtx_ring, ptrs.producer);
	623	off.rx.consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer);
	624	off.rx.desc = offsetof(struct xdp_rxtx_ring, desc);
	625	off.tx.producer = offsetof(struct xdp_rxtx_ring, ptrs.producer);
	626	off.tx.consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer);
	627	off.tx.desc = offsetof(struct xdp_rxtx_ring, desc);
	628
	629	off.fr.producer = offsetof(struct xdp_umem_ring, ptrs.producer);
	630	off.fr.consumer = offsetof(struct xdp_umem_ring, ptrs.consumer);
	631	off.fr.desc = offsetof(struct xdp_umem_ring, desc);
	632	off.cr.producer = offsetof(struct xdp_umem_ring, ptrs.producer);
	633	off.cr.consumer = offsetof(struct xdp_umem_ring, ptrs.consumer);
	634	off.cr.desc = offsetof(struct xdp_umem_ring, desc);
	635
	636	len = sizeof(off);
	637	if (copy_to_user(optval, &off, len))
	638	return -EFAULT;
	639	if (put_user(len, optlen))
	640	return -EFAULT;
	641
	642	return 0;
	643	}
af75d9e0 MK	644	default:
	645	break;
	646	}
	647
	648	return -EOPNOTSUPP;
	649	}
	650
423f3832 MK	651	static int xsk_mmap(struct file file, struct socket sock,
	652	struct vm_area_struct *vma)
	653	{
a5a16e43	654	loff_t offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
423f3832 MK	655	unsigned long size = vma->vm_end - vma->vm_start;
	656	struct xdp_sock *xs = xdp_sk(sock->sk);
	657	struct xsk_queue *q = NULL;
37b07693	658	struct xdp_umem *umem;
423f3832 MK	659	unsigned long pfn;
	660	struct page *qpg;
	661
b9b6b68e	662	if (offset == XDP_PGOFF_RX_RING) {
37b07693	663	q = READ_ONCE(xs->rx);
f6145903	664	} else if (offset == XDP_PGOFF_TX_RING) {
37b07693	665	q = READ_ONCE(xs->tx);
b9b6b68e	666	} else {
37b07693 BT	667	umem = READ_ONCE(xs->umem);
37b07693 BT	668	if (!umem)
b9b6b68e	669	return -EINVAL;
423f3832	670
b9b6b68e	671	if (offset == XDP_UMEM_PGOFF_FILL_RING)
37b07693	672	q = READ_ONCE(umem->fq);
fe230832	673	else if (offset == XDP_UMEM_PGOFF_COMPLETION_RING)
37b07693	674	q = READ_ONCE(umem->cq);
b9b6b68e	675	}
423f3832 MK	676
	677	if (!q)
	678	return -EINVAL;
	679
	680	qpg = virt_to_head_page(q->ring);
	681	if (size > (PAGE_SIZE << compound_order(qpg)))
	682	return -EINVAL;
	683
	684	pfn = virt_to_phys(q->ring) >> PAGE_SHIFT;
	685	return remap_pfn_range(vma, vma->vm_start, pfn,
	686	size, vma->vm_page_prot);
	687	}
	688
c0c77d8f BT	689	static struct proto xsk_proto = {
	690	.name = "XDP",
	691	.owner = THIS_MODULE,
	692	.obj_size = sizeof(struct xdp_sock),
	693	};
	694
	695	static const struct proto_ops xsk_proto_ops = {
c2f4374b BT	696	.family = PF_XDP,
	697	.owner = THIS_MODULE,
	698	.release = xsk_release,
	699	.bind = xsk_bind,
	700	.connect = sock_no_connect,
	701	.socketpair = sock_no_socketpair,
	702	.accept = sock_no_accept,
	703	.getname = sock_no_getname,
a11e1d43	704	.poll = xsk_poll,
c2f4374b BT	705	.ioctl = sock_no_ioctl,
	706	.listen = sock_no_listen,
	707	.shutdown = sock_no_shutdown,
	708	.setsockopt = xsk_setsockopt,
	709	.getsockopt = xsk_getsockopt,
	710	.sendmsg = xsk_sendmsg,
	711	.recvmsg = sock_no_recvmsg,
	712	.mmap = xsk_mmap,
	713	.sendpage = sock_no_sendpage,
c0c77d8f BT	714	};
	715
	716	static void xsk_destruct(struct sock *sk)
	717	{
	718	struct xdp_sock *xs = xdp_sk(sk);
	719
	720	if (!sock_flag(sk, SOCK_DEAD))
	721	return;
	722
	723	xdp_put_umem(xs->umem);
	724
	725	sk_refcnt_debug_dec(sk);
	726	}
	727
	728	static int xsk_create(struct net net, struct socket sock, int protocol,
	729	int kern)
	730	{
	731	struct sock *sk;
	732	struct xdp_sock *xs;
	733
	734	if (!ns_capable(net->user_ns, CAP_NET_RAW))
	735	return -EPERM;
	736	if (sock->type != SOCK_RAW)
	737	return -ESOCKTNOSUPPORT;
	738
	739	if (protocol)
	740	return -EPROTONOSUPPORT;
	741
	742	sock->state = SS_UNCONNECTED;
	743
	744	sk = sk_alloc(net, PF_XDP, GFP_KERNEL, &xsk_proto, kern);
	745	if (!sk)
	746	return -ENOBUFS;
	747
	748	sock->ops = &xsk_proto_ops;
	749
	750	sock_init_data(sock, sk);
	751
	752	sk->sk_family = PF_XDP;
	753
	754	sk->sk_destruct = xsk_destruct;
	755	sk_refcnt_debug_inc(sk);
	756
cee27167 BT	757	sock_set_flag(sk, SOCK_RCU_FREE);
cee27167 BT	758
c0c77d8f BT	759	xs = xdp_sk(sk);
c0c77d8f BT	760	mutex_init(&xs->mutex);
a9744f7c	761	spin_lock_init(&xs->tx_completion_lock);
c0c77d8f BT	762
	763	local_bh_disable();
	764	sock_prot_inuse_add(net, &xsk_proto, 1);
	765	local_bh_enable();
	766
	767	return 0;
	768	}
	769
	770	static const struct net_proto_family xsk_family_ops = {
	771	.family = PF_XDP,
	772	.create = xsk_create,
	773	.owner = THIS_MODULE,
	774	};
	775
	776	static int __init xsk_init(void)
	777	{
	778	int err;
	779
	780	err = proto_register(&xsk_proto, 0 /* no slab */);
	781	if (err)
	782	goto out;
	783
	784	err = sock_register(&xsk_family_ops);
	785	if (err)
	786	goto out_proto;
	787
	788	return 0;
	789
	790	out_proto:
	791	proto_unregister(&xsk_proto);
	792	out:
	793	return err;
	794	}
	795
	796	fs_initcall(xsk_init);