[linux-block.git] / net / xdp / xdp_umem.c

// SPDX-License-Identifier: GPL-2.0
/* XDP user-space packet buffer
 * Copyright(c) 2018 Intel Corporation.
 */

#include <linux/init.h>
#include <linux/sched/mm.h>
#include <linux/sched/signal.h>
#include <linux/sched/task.h>
#include <linux/uaccess.h>
#include <linux/slab.h>
#include <linux/bpf.h>
#include <linux/mm.h>
#include <linux/netdevice.h>
#include <linux/rtnetlink.h>
#include <linux/idr.h>
#include <linux/vmalloc.h>

#include "xdp_umem.h"
#include "xsk_queue.h"

#define XDP_UMEM_MIN_CHUNK_SIZE 2048

static DEFINE_IDA(umem_ida);

void xdp_add_sk_umem(struct xdp_umem *umem, struct xdp_sock *xs)
{
	unsigned long flags;

	if (!xs->tx)
		return;

	spin_lock_irqsave(&umem->xsk_tx_list_lock, flags);
	list_add_rcu(&xs->list, &umem->xsk_tx_list);
	spin_unlock_irqrestore(&umem->xsk_tx_list_lock, flags);
}

void xdp_del_sk_umem(struct xdp_umem *umem, struct xdp_sock *xs)
{
	unsigned long flags;

	if (!xs->tx)
		return;

	spin_lock_irqsave(&umem->xsk_tx_list_lock, flags);
	list_del_rcu(&xs->list);
	spin_unlock_irqrestore(&umem->xsk_tx_list_lock, flags);
}

/* The umem is stored both in the _rx struct and the _tx struct as we do
 * not know if the device has more tx queues than rx, or the opposite.
 * This might also change during run time.
 */
static int xdp_reg_umem_at_qid(struct net_device *dev, struct xdp_umem *umem,
			       u16 queue_id)
{
	if (queue_id >= max_t(unsigned int,
			      dev->real_num_rx_queues,
			      dev->real_num_tx_queues))
		return -EINVAL;

	if (queue_id < dev->real_num_rx_queues)
		dev->_rx[queue_id].umem = umem;
	if (queue_id < dev->real_num_tx_queues)
		dev->_tx[queue_id].umem = umem;

	return 0;
}

struct xdp_umem *xdp_get_umem_from_qid(struct net_device *dev,
				       u16 queue_id)
{
	if (queue_id < dev->real_num_rx_queues)
		return dev->_rx[queue_id].umem;
	if (queue_id < dev->real_num_tx_queues)
		return dev->_tx[queue_id].umem;

	return NULL;
}
EXPORT_SYMBOL(xdp_get_umem_from_qid);

static void xdp_clear_umem_at_qid(struct net_device *dev, u16 queue_id)
{
	if (queue_id < dev->real_num_rx_queues)
		dev->_rx[queue_id].umem = NULL;
	if (queue_id < dev->real_num_tx_queues)
		dev->_tx[queue_id].umem = NULL;
}

int xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev,
			u16 queue_id, u16 flags)
{
	bool force_zc, force_copy;
	struct netdev_bpf bpf;
	int err = 0;

	ASSERT_RTNL();

	force_zc = flags & XDP_ZEROCOPY;
	force_copy = flags & XDP_COPY;

	if (force_zc && force_copy)
		return -EINVAL;

	if (xdp_get_umem_from_qid(dev, queue_id))
		return -EBUSY;

	err = xdp_reg_umem_at_qid(dev, umem, queue_id);
	if (err)
		return err;

	umem->dev = dev;
	umem->queue_id = queue_id;

	if (flags & XDP_USE_NEED_WAKEUP) {
		umem->flags |= XDP_UMEM_USES_NEED_WAKEUP;
		/* Tx needs to be explicitly woken up the first time.
		 * Also for supporting drivers that do not implement this
		 * feature. They will always have to call sendto().
		 */
		xsk_set_tx_need_wakeup(umem);
	}

	dev_hold(dev);

	if (force_copy)
		/* For copy-mode, we are done. */
		return 0;

	if (!dev->netdev_ops->ndo_bpf || !dev->netdev_ops->ndo_xsk_wakeup) {
		err = -EOPNOTSUPP;
		goto err_unreg_umem;
	}

	bpf.command = XDP_SETUP_XSK_UMEM;
	bpf.xsk.umem = umem;
	bpf.xsk.queue_id = queue_id;

	err = dev->netdev_ops->ndo_bpf(dev, &bpf);
	if (err)
		goto err_unreg_umem;

	umem->zc = true;
	return 0;

err_unreg_umem:
	if (!force_zc)
		err = 0; /* fallback to copy mode */
	if (err)
		xdp_clear_umem_at_qid(dev, queue_id);
	return err;
}

void xdp_umem_clear_dev(struct xdp_umem *umem)
{
	struct netdev_bpf bpf;
	int err;

	ASSERT_RTNL();

	if (!umem->dev)
		return;

	if (umem->zc) {
		bpf.command = XDP_SETUP_XSK_UMEM;
		bpf.xsk.umem = NULL;
		bpf.xsk.queue_id = umem->queue_id;

		err = umem->dev->netdev_ops->ndo_bpf(umem->dev, &bpf);

		if (err)
			WARN(1, "failed to disable umem!\n");
	}

	xdp_clear_umem_at_qid(umem->dev, umem->queue_id);

	dev_put(umem->dev);
	umem->dev = NULL;
	umem->zc = false;
}

static void xdp_umem_unpin_pages(struct xdp_umem *umem)
{
	unpin_user_pages_dirty_lock(umem->pgs, umem->npgs, true);

	kfree(umem->pgs);
	umem->pgs = NULL;
}

static void xdp_umem_unaccount_pages(struct xdp_umem *umem)
{
	if (umem->user) {
		atomic_long_sub(umem->npgs, &umem->user->locked_vm);
		free_uid(umem->user);
	}
}

static void xdp_umem_release(struct xdp_umem *umem)
{
	rtnl_lock();
	xdp_umem_clear_dev(umem);
	rtnl_unlock();

	ida_simple_remove(&umem_ida, umem->id);

	if (umem->fq) {
		xskq_destroy(umem->fq);
		umem->fq = NULL;
	}

	if (umem->cq) {
		xskq_destroy(umem->cq);
		umem->cq = NULL;
	}

	xp_destroy(umem->pool);
	xdp_umem_unpin_pages(umem);

	xdp_umem_unaccount_pages(umem);
	kfree(umem);
}

static void xdp_umem_release_deferred(struct work_struct *work)
{
	struct xdp_umem *umem = container_of(work, struct xdp_umem, work);

	xdp_umem_release(umem);
}

void xdp_get_umem(struct xdp_umem *umem)
{
	refcount_inc(&umem->users);
}

void xdp_put_umem(struct xdp_umem *umem)
{
	if (!umem)
		return;

	if (refcount_dec_and_test(&umem->users)) {
		INIT_WORK(&umem->work, xdp_umem_release_deferred);
		schedule_work(&umem->work);
	}
}

static int xdp_umem_pin_pages(struct xdp_umem *umem, unsigned long address)
{
	unsigned int gup_flags = FOLL_WRITE;
	long npgs;
	int err;

	umem->pgs = kcalloc(umem->npgs, sizeof(*umem->pgs),
			    GFP_KERNEL | __GFP_NOWARN);
	if (!umem->pgs)
		return -ENOMEM;

	mmap_read_lock(current->mm);
	npgs = pin_user_pages(address, umem->npgs,
			      gup_flags | FOLL_LONGTERM, &umem->pgs[0], NULL);
	mmap_read_unlock(current->mm);

	if (npgs != umem->npgs) {
		if (npgs >= 0) {
			umem->npgs = npgs;
			err = -ENOMEM;
			goto out_pin;
		}
		err = npgs;
		goto out_pgs;
	}
	return 0;

out_pin:
	xdp_umem_unpin_pages(umem);
out_pgs:
	kfree(umem->pgs);
	umem->pgs = NULL;
	return err;
}

static int xdp_umem_account_pages(struct xdp_umem *umem)
{
	unsigned long lock_limit, new_npgs, old_npgs;

	if (capable(CAP_IPC_LOCK))
		return 0;

	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
	umem->user = get_uid(current_user());

	do {
		old_npgs = atomic_long_read(&umem->user->locked_vm);
		new_npgs = old_npgs + umem->npgs;
		if (new_npgs > lock_limit) {
			free_uid(umem->user);
			umem->user = NULL;
			return -ENOBUFS;
		}
	} while (atomic_long_cmpxchg(&umem->user->locked_vm, old_npgs,
				     new_npgs) != old_npgs);
	return 0;
}

static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
{
	bool unaligned_chunks = mr->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG;
	u32 chunk_size = mr->chunk_size, headroom = mr->headroom;
	u64 npgs, addr = mr->addr, size = mr->len;
	unsigned int chunks, chunks_per_page;
	int err;

	if (chunk_size < XDP_UMEM_MIN_CHUNK_SIZE || chunk_size > PAGE_SIZE) {
		/* Strictly speaking we could support this, if:
		 * - huge pages, or*
		 * - using an IOMMU, or
		 * - making sure the memory area is consecutive
		 * but for now, we simply say "computer says no".
		 */
		return -EINVAL;
	}

	if (mr->flags & ~(XDP_UMEM_UNALIGNED_CHUNK_FLAG |
			XDP_UMEM_USES_NEED_WAKEUP))
		return -EINVAL;

	if (!unaligned_chunks && !is_power_of_2(chunk_size))
		return -EINVAL;

	if (!PAGE_ALIGNED(addr)) {
		/* Memory area has to be page size aligned. For
		 * simplicity, this might change.
		 */
		return -EINVAL;
	}

	if ((addr + size) < addr)
		return -EINVAL;

	npgs = size >> PAGE_SHIFT;
	if (npgs > U32_MAX)
		return -EINVAL;

	chunks = (unsigned int)div_u64(size, chunk_size);
	if (chunks == 0)
		return -EINVAL;

	if (!unaligned_chunks) {
		chunks_per_page = PAGE_SIZE / chunk_size;
		if (chunks < chunks_per_page || chunks % chunks_per_page)
			return -EINVAL;
	}

	if (headroom >= chunk_size - XDP_PACKET_HEADROOM)
		return -EINVAL;

	umem->size = size;
	umem->headroom = headroom;
	umem->chunk_size = chunk_size;
	umem->npgs = (u32)npgs;
	umem->pgs = NULL;
	umem->user = NULL;
	umem->flags = mr->flags;
	INIT_LIST_HEAD(&umem->xsk_tx_list);
	spin_lock_init(&umem->xsk_tx_list_lock);

	refcount_set(&umem->users, 1);

	err = xdp_umem_account_pages(umem);
	if (err)
		return err;

	err = xdp_umem_pin_pages(umem, (unsigned long)addr);
	if (err)
		goto out_account;

	umem->pool = xp_create(umem->pgs, umem->npgs, chunks, chunk_size,
			       headroom, size, unaligned_chunks);
	if (!umem->pool) {
		err = -ENOMEM;
		goto out_pin;
	}
	return 0;

out_pin:
	xdp_umem_unpin_pages(umem);
out_account:
	xdp_umem_unaccount_pages(umem);
	return err;
}

struct xdp_umem *xdp_umem_create(struct xdp_umem_reg *mr)
{
	struct xdp_umem *umem;
	int err;

	umem = kzalloc(sizeof(*umem), GFP_KERNEL);
	if (!umem)
		return ERR_PTR(-ENOMEM);

	err = ida_simple_get(&umem_ida, 0, 0, GFP_KERNEL);
	if (err < 0) {
		kfree(umem);
		return ERR_PTR(err);
	}
	umem->id = err;

	err = xdp_umem_reg(umem, mr);
	if (err) {
		ida_simple_remove(&umem_ida, umem->id);
		kfree(umem);
		return ERR_PTR(err);
	}

	return umem;
}

bool xdp_umem_validate_queues(struct xdp_umem *umem)
{
	return umem->fq && umem->cq;
}
Commit	Line	Data
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* XDP user-space packet buffer
	3	* Copyright(c) 2018 Intel Corporation.
	4	*/
	5
	6	#include <linux/init.h>
	7	#include <linux/sched/mm.h>
	8	#include <linux/sched/signal.h>
	9	#include <linux/sched/task.h>
	10	#include <linux/uaccess.h>
	11	#include <linux/slab.h>
	12	#include <linux/bpf.h>
	13	#include <linux/mm.h>
	14	#include <linux/netdevice.h>
	15	#include <linux/rtnetlink.h>
	16	#include <linux/idr.h>
	17	#include <linux/vmalloc.h>
	18
	19	#include "xdp_umem.h"
	20	#include "xsk_queue.h"
	21
	22	#define XDP_UMEM_MIN_CHUNK_SIZE 2048
	23
	24	static DEFINE_IDA(umem_ida);
	25
	26	void xdp_add_sk_umem(struct xdp_umem umem, struct xdp_sock xs)
	27	{
	28	unsigned long flags;
	29
	30	if (!xs->tx)
	31	return;
	32
	33	spin_lock_irqsave(&umem->xsk_tx_list_lock, flags);
	34	list_add_rcu(&xs->list, &umem->xsk_tx_list);
	35	spin_unlock_irqrestore(&umem->xsk_tx_list_lock, flags);
	36	}
	37
	38	void xdp_del_sk_umem(struct xdp_umem umem, struct xdp_sock xs)
	39	{
	40	unsigned long flags;
	41
	42	if (!xs->tx)
	43	return;
	44
	45	spin_lock_irqsave(&umem->xsk_tx_list_lock, flags);
	46	list_del_rcu(&xs->list);
	47	spin_unlock_irqrestore(&umem->xsk_tx_list_lock, flags);
	48	}
	49
	50	/* The umem is stored both in the _rx struct and the _tx struct as we do
	51	* not know if the device has more tx queues than rx, or the opposite.
	52	* This might also change during run time.
	53	*/
	54	static int xdp_reg_umem_at_qid(struct net_device dev, struct xdp_umem umem,
	55	u16 queue_id)
	56	{
	57	if (queue_id >= max_t(unsigned int,
	58	dev->real_num_rx_queues,
	59	dev->real_num_tx_queues))
	60	return -EINVAL;
	61
	62	if (queue_id < dev->real_num_rx_queues)
	63	dev->_rx[queue_id].umem = umem;
	64	if (queue_id < dev->real_num_tx_queues)
	65	dev->_tx[queue_id].umem = umem;
	66
	67	return 0;
	68	}
	69
	70	struct xdp_umem xdp_get_umem_from_qid(struct net_device dev,
	71	u16 queue_id)
	72	{
	73	if (queue_id < dev->real_num_rx_queues)
	74	return dev->_rx[queue_id].umem;
	75	if (queue_id < dev->real_num_tx_queues)
	76	return dev->_tx[queue_id].umem;
	77
	78	return NULL;
	79	}
	80	EXPORT_SYMBOL(xdp_get_umem_from_qid);
	81
	82	static void xdp_clear_umem_at_qid(struct net_device *dev, u16 queue_id)
	83	{
	84	if (queue_id < dev->real_num_rx_queues)
	85	dev->_rx[queue_id].umem = NULL;
	86	if (queue_id < dev->real_num_tx_queues)
	87	dev->_tx[queue_id].umem = NULL;
	88	}
	89
	90	int xdp_umem_assign_dev(struct xdp_umem umem, struct net_device dev,
	91	u16 queue_id, u16 flags)
	92	{
	93	bool force_zc, force_copy;
	94	struct netdev_bpf bpf;
	95	int err = 0;
	96
	97	ASSERT_RTNL();
	98
	99	force_zc = flags & XDP_ZEROCOPY;
	100	force_copy = flags & XDP_COPY;
	101
	102	if (force_zc && force_copy)
	103	return -EINVAL;
	104
	105	if (xdp_get_umem_from_qid(dev, queue_id))
	106	return -EBUSY;
	107
	108	err = xdp_reg_umem_at_qid(dev, umem, queue_id);
	109	if (err)
	110	return err;
	111
	112	umem->dev = dev;
	113	umem->queue_id = queue_id;
	114
	115	if (flags & XDP_USE_NEED_WAKEUP) {
	116	umem->flags \|= XDP_UMEM_USES_NEED_WAKEUP;
	117	/* Tx needs to be explicitly woken up the first time.
	118	* Also for supporting drivers that do not implement this
	119	* feature. They will always have to call sendto().
	120	*/
	121	xsk_set_tx_need_wakeup(umem);
	122	}
	123
	124	dev_hold(dev);
	125
	126	if (force_copy)
	127	/* For copy-mode, we are done. */
	128	return 0;
	129
	130	if (!dev->netdev_ops->ndo_bpf \|\| !dev->netdev_ops->ndo_xsk_wakeup) {
	131	err = -EOPNOTSUPP;
	132	goto err_unreg_umem;
	133	}
	134
	135	bpf.command = XDP_SETUP_XSK_UMEM;
	136	bpf.xsk.umem = umem;
	137	bpf.xsk.queue_id = queue_id;
	138
	139	err = dev->netdev_ops->ndo_bpf(dev, &bpf);
	140	if (err)
	141	goto err_unreg_umem;
	142
	143	umem->zc = true;
	144	return 0;
	145
	146	err_unreg_umem:
	147	if (!force_zc)
	148	err = 0; /* fallback to copy mode */
	149	if (err)
	150	xdp_clear_umem_at_qid(dev, queue_id);
	151	return err;
	152	}
	153
	154	void xdp_umem_clear_dev(struct xdp_umem *umem)
	155	{
	156	struct netdev_bpf bpf;
	157	int err;
	158
	159	ASSERT_RTNL();
	160
	161	if (!umem->dev)
	162	return;
	163
	164	if (umem->zc) {
	165	bpf.command = XDP_SETUP_XSK_UMEM;
	166	bpf.xsk.umem = NULL;
	167	bpf.xsk.queue_id = umem->queue_id;
	168
	169	err = umem->dev->netdev_ops->ndo_bpf(umem->dev, &bpf);
	170
	171	if (err)
	172	WARN(1, "failed to disable umem!\n");
	173	}
	174
	175	xdp_clear_umem_at_qid(umem->dev, umem->queue_id);
	176
	177	dev_put(umem->dev);
	178	umem->dev = NULL;
	179	umem->zc = false;
	180	}
	181
	182	static void xdp_umem_unpin_pages(struct xdp_umem *umem)
	183	{
	184	unpin_user_pages_dirty_lock(umem->pgs, umem->npgs, true);
	185
	186	kfree(umem->pgs);
	187	umem->pgs = NULL;
	188	}
	189
	190	static void xdp_umem_unaccount_pages(struct xdp_umem *umem)
	191	{
	192	if (umem->user) {
	193	atomic_long_sub(umem->npgs, &umem->user->locked_vm);
	194	free_uid(umem->user);
	195	}
	196	}
	197
	198	static void xdp_umem_release(struct xdp_umem *umem)
	199	{
	200	rtnl_lock();
	201	xdp_umem_clear_dev(umem);
	202	rtnl_unlock();
	203
	204	ida_simple_remove(&umem_ida, umem->id);
	205
	206	if (umem->fq) {
	207	xskq_destroy(umem->fq);
	208	umem->fq = NULL;
	209	}
	210
	211	if (umem->cq) {
	212	xskq_destroy(umem->cq);
	213	umem->cq = NULL;
	214	}
	215
	216	xp_destroy(umem->pool);
	217	xdp_umem_unpin_pages(umem);
	218
	219	xdp_umem_unaccount_pages(umem);
	220	kfree(umem);
	221	}
	222
	223	static void xdp_umem_release_deferred(struct work_struct *work)
	224	{
	225	struct xdp_umem *umem = container_of(work, struct xdp_umem, work);
	226
	227	xdp_umem_release(umem);
	228	}
	229
	230	void xdp_get_umem(struct xdp_umem *umem)
	231	{
	232	refcount_inc(&umem->users);
	233	}
	234
	235	void xdp_put_umem(struct xdp_umem *umem)
	236	{
	237	if (!umem)
	238	return;
	239
	240	if (refcount_dec_and_test(&umem->users)) {
	241	INIT_WORK(&umem->work, xdp_umem_release_deferred);
	242	schedule_work(&umem->work);
	243	}
	244	}
	245
	246	static int xdp_umem_pin_pages(struct xdp_umem *umem, unsigned long address)
	247	{
	248	unsigned int gup_flags = FOLL_WRITE;
	249	long npgs;
	250	int err;
	251
	252	umem->pgs = kcalloc(umem->npgs, sizeof(*umem->pgs),
	253	GFP_KERNEL \| __GFP_NOWARN);
	254	if (!umem->pgs)
	255	return -ENOMEM;
	256
	257	mmap_read_lock(current->mm);
	258	npgs = pin_user_pages(address, umem->npgs,
	259	gup_flags \| FOLL_LONGTERM, &umem->pgs[0], NULL);
	260	mmap_read_unlock(current->mm);
	261
	262	if (npgs != umem->npgs) {
	263	if (npgs >= 0) {
	264	umem->npgs = npgs;
	265	err = -ENOMEM;
	266	goto out_pin;
	267	}
	268	err = npgs;
	269	goto out_pgs;
	270	}
	271	return 0;
	272
	273	out_pin:
	274	xdp_umem_unpin_pages(umem);
	275	out_pgs:
	276	kfree(umem->pgs);
	277	umem->pgs = NULL;
	278	return err;
	279	}
	280
	281	static int xdp_umem_account_pages(struct xdp_umem *umem)
	282	{
	283	unsigned long lock_limit, new_npgs, old_npgs;
	284
	285	if (capable(CAP_IPC_LOCK))
	286	return 0;
	287
	288	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
	289	umem->user = get_uid(current_user());
	290
	291	do {
	292	old_npgs = atomic_long_read(&umem->user->locked_vm);
	293	new_npgs = old_npgs + umem->npgs;
	294	if (new_npgs > lock_limit) {
	295	free_uid(umem->user);
	296	umem->user = NULL;
	297	return -ENOBUFS;
	298	}
	299	} while (atomic_long_cmpxchg(&umem->user->locked_vm, old_npgs,
	300	new_npgs) != old_npgs);
	301	return 0;
	302	}
	303
	304	static int xdp_umem_reg(struct xdp_umem umem, struct xdp_umem_reg mr)
	305	{
	306	bool unaligned_chunks = mr->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG;
	307	u32 chunk_size = mr->chunk_size, headroom = mr->headroom;
	308	u64 npgs, addr = mr->addr, size = mr->len;
	309	unsigned int chunks, chunks_per_page;
	310	int err;
	311
	312	if (chunk_size < XDP_UMEM_MIN_CHUNK_SIZE \|\| chunk_size > PAGE_SIZE) {
	313	/* Strictly speaking we could support this, if:
	314	* - huge pages, or*
	315	* - using an IOMMU, or
	316	* - making sure the memory area is consecutive
	317	* but for now, we simply say "computer says no".
	318	*/
	319	return -EINVAL;
	320	}
	321
	322	if (mr->flags & ~(XDP_UMEM_UNALIGNED_CHUNK_FLAG \|
	323	XDP_UMEM_USES_NEED_WAKEUP))
	324	return -EINVAL;
	325
	326	if (!unaligned_chunks && !is_power_of_2(chunk_size))
	327	return -EINVAL;
	328
	329	if (!PAGE_ALIGNED(addr)) {
	330	/* Memory area has to be page size aligned. For
	331	* simplicity, this might change.
	332	*/
	333	return -EINVAL;
	334	}
	335
	336	if ((addr + size) < addr)
	337	return -EINVAL;
	338
	339	npgs = size >> PAGE_SHIFT;
	340	if (npgs > U32_MAX)
	341	return -EINVAL;
	342
	343	chunks = (unsigned int)div_u64(size, chunk_size);
	344	if (chunks == 0)
	345	return -EINVAL;
	346
	347	if (!unaligned_chunks) {
	348	chunks_per_page = PAGE_SIZE / chunk_size;
	349	if (chunks < chunks_per_page \|\| chunks % chunks_per_page)
	350	return -EINVAL;
	351	}
	352
	353	if (headroom >= chunk_size - XDP_PACKET_HEADROOM)
	354	return -EINVAL;
	355
	356	umem->size = size;
	357	umem->headroom = headroom;
	358	umem->chunk_size = chunk_size;
	359	umem->npgs = (u32)npgs;
	360	umem->pgs = NULL;
	361	umem->user = NULL;
	362	umem->flags = mr->flags;
	363	INIT_LIST_HEAD(&umem->xsk_tx_list);
	364	spin_lock_init(&umem->xsk_tx_list_lock);
	365
	366	refcount_set(&umem->users, 1);
	367
	368	err = xdp_umem_account_pages(umem);
	369	if (err)
	370	return err;
	371
	372	err = xdp_umem_pin_pages(umem, (unsigned long)addr);
	373	if (err)
	374	goto out_account;
	375
	376	umem->pool = xp_create(umem->pgs, umem->npgs, chunks, chunk_size,
	377	headroom, size, unaligned_chunks);
	378	if (!umem->pool) {
	379	err = -ENOMEM;
	380	goto out_pin;
	381	}
	382	return 0;
	383
	384	out_pin:
	385	xdp_umem_unpin_pages(umem);
	386	out_account:
	387	xdp_umem_unaccount_pages(umem);
	388	return err;
	389	}
	390
	391	struct xdp_umem xdp_umem_create(struct xdp_umem_reg mr)
	392	{
	393	struct xdp_umem *umem;
	394	int err;
	395
	396	umem = kzalloc(sizeof(*umem), GFP_KERNEL);
	397	if (!umem)
	398	return ERR_PTR(-ENOMEM);
	399
	400	err = ida_simple_get(&umem_ida, 0, 0, GFP_KERNEL);
	401	if (err < 0) {
	402	kfree(umem);
	403	return ERR_PTR(err);
	404	}
	405	umem->id = err;
	406
	407	err = xdp_umem_reg(umem, mr);
	408	if (err) {
	409	ida_simple_remove(&umem_ida, umem->id);
	410	kfree(umem);
	411	return ERR_PTR(err);
	412	}
	413
	414	return umem;
	415	}
	416
	417	bool xdp_umem_validate_queues(struct xdp_umem *umem)
	418	{
	419	return umem->fq && umem->cq;
	420	}