xsk: support use vaddr as ring

When we try to start AF_XDP on some machines with long running time, due
to the machine's memory fragmentation problem, there is no sufficient
contiguous physical memory that will cause the start failure.

If the size of the queue is 8 * 1024, then the size of the desc[] is
8 * 1024 * 8 = 16 * PAGE, but we also add struct xdp_ring size, so it is
16page+. This is necessary to apply for a 4-order memory. If there are a
lot of queues, it is difficult to these machine with long running time.

Here, that we actually waste 15 pages. 4-Order memory is 32 pages, but
we only use 17 pages.

This patch replaces __get_free_pages() by vmalloc() to allocate memory
to solve these problems.

Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
Acked-by: Magnus Karlsson <magnus.karlsson@intel.com>
Reviewed-by: Alexander Lobakin <aleksander.lobakin@intel.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index a245c1b4a21b1aa22ce516f1fcde09a1c68ebd4a..63c82e8bcd8e7c6d084dd52db98fbcbace2e2dcf 100644 (file)
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -1294,8 +1294,6 @@ static int xsk_mmap(struct file *file, struct socket *sock,
        unsigned long size = vma->vm_end - vma->vm_start;
        struct xdp_sock *xs = xdp_sk(sock->sk);
        struct xsk_queue *q = NULL;
-       unsigned long pfn;
-       struct page *qpg;
 
        if (READ_ONCE(xs->state) != XSK_READY)
                return -EBUSY;
@@ -1318,13 +1316,10 @@ static int xsk_mmap(struct file *file, struct socket *sock,
 
        /* Matches the smp_wmb() in xsk_init_queue */
        smp_rmb();
-       qpg = virt_to_head_page(q->ring);
-       if (size > page_size(qpg))
+       if (size > q->ring_vmalloc_size)
                return -EINVAL;
 
-       pfn = virt_to_phys(q->ring) >> PAGE_SHIFT;
-       return remap_pfn_range(vma, vma->vm_start, pfn,
-                              size, vma->vm_page_prot);
+       return remap_vmalloc_range(vma, q->ring, 0);
 }
 
 static int xsk_notifier(struct notifier_block *this,

diff --git a/net/xdp/xsk_queue.c b/net/xdp/xsk_queue.c
index 6cf9586e5027a64c55dde9aa561e1cc662db51c6..f8905400ee07a0f69095349084c44037dd15e0bf 100644 (file)
--- a/net/xdp/xsk_queue.c
+++ b/net/xdp/xsk_queue.c
@@ -6,6 +6,7 @@
 #include <linux/log2.h>
 #include <linux/slab.h>
 #include <linux/overflow.h>
+#include <linux/vmalloc.h>
 #include <net/xdp_sock_drv.h>
 
 #include "xsk_queue.h"
@@ -23,7 +24,6 @@ static size_t xskq_get_ring_size(struct xsk_queue *q, bool umem_queue)
 struct xsk_queue *xskq_create(u32 nentries, bool umem_queue)
 {
        struct xsk_queue *q;
-       gfp_t gfp_flags;
        size_t size;
 
        q = kzalloc(sizeof(*q), GFP_KERNEL);
@@ -33,17 +33,16 @@ struct xsk_queue *xskq_create(u32 nentries, bool umem_queue)
        q->nentries = nentries;
        q->ring_mask = nentries - 1;
 
-       gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN |
-                   __GFP_COMP  | __GFP_NORETRY;
        size = xskq_get_ring_size(q, umem_queue);
+       size = PAGE_ALIGN(size);
 
-       q->ring = (struct xdp_ring *)__get_free_pages(gfp_flags,
-                                                     get_order(size));
+       q->ring = vmalloc_user(size);
        if (!q->ring) {
                kfree(q);
                return NULL;
        }
 
+       q->ring_vmalloc_size = size;
        return q;
 }
 
@@ -52,6 +51,6 @@ void xskq_destroy(struct xsk_queue *q)
        if (!q)
                return;
 
-       page_frag_free(q->ring);
+       vfree(q->ring);
        kfree(q);
 }

diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h
index c6fb6b763658255873e1263337b09c980ce88079..bfb2a7e50c2617a3ff000d8bc3ef858e73ba3393 100644 (file)
--- a/net/xdp/xsk_queue.h
+++ b/net/xdp/xsk_queue.h
@@ -45,6 +45,7 @@ struct xsk_queue {
        struct xdp_ring *ring;
        u64 invalid_descs;
        u64 queue_empty_descs;
+       size_t ring_vmalloc_size;
 };
 
 /* The structure of the shared state of the rings are a simple