From: Jens Axboe <axboe@kernel.dk>
Date: Mon, 14 Aug 2017 19:18:36 +0000 (-0600)
Subject: aio: enable cpu_cache_alloc frontend for aio_kiocb allocations
X-Git-Url: https://git.kernel.dk/?a=commitdiff_plain;h=refs%2Fheads%2Fcpu-alloc-cache;p=linux-2.6-block.git

aio: enable cpu_cache_alloc frontend for aio_kiocb allocations

Signed-off-by: Jens Axboe <axboe@kernel.dk>
---

diff --git a/fs/aio.c b/fs/aio.c
index dcad3a66748c..35fee27173f8 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -18,6 +18,7 @@
 #include <linux/export.h>
 #include <linux/syscalls.h>
 #include <linux/backing-dev.h>
+#include <linux/pcpu_cache.h>
 #include <linux/uio.h>
 
 #include <linux/sched/signal.h>
@@ -202,6 +203,9 @@ static struct vfsmount *aio_mnt;
 static const struct file_operations aio_ring_fops;
 static const struct address_space_operations aio_ctx_aops;
 
+static struct pcpu_alloc_cache alloc_cache;
+static DEFINE_PER_CPU(struct pcpu_cache, req_alloc_cache);
+
 static struct file *aio_private_file(struct kioctx *ctx, loff_t nr_pages)
 {
 	struct qstr this = QSTR_INIT("[aio]", 5);
@@ -265,6 +269,8 @@ static int __init aio_setup(void)
 	kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC);
 	kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC);
 
+	pcpu_cache_init(&alloc_cache, &req_alloc_cache, kiocb_cachep);
+
 	pr_debug("sizeof(struct page) = %zu\n", sizeof(struct page));
 
 	return 0;
@@ -1017,7 +1023,7 @@ static inline struct aio_kiocb *aio_get_req(struct kioctx *ctx)
 			return NULL;
 	}
 
-	req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL|__GFP_ZERO);
+	req = pcpu_cache_alloc(&alloc_cache, GFP_KERNEL|__GFP_ZERO);
 	if (unlikely(!req))
 		goto out_put;
 
@@ -1036,7 +1042,7 @@ static void kiocb_free(struct aio_kiocb *req)
 		fput(req->common.ki_filp);
 	if (req->ki_eventfd != NULL)
 		eventfd_ctx_put(req->ki_eventfd);
-	kmem_cache_free(kiocb_cachep, req);
+	pcpu_cache_free(&alloc_cache, req);
 }
 
 static struct kioctx *lookup_ioctx(unsigned long ctx_id)