summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJens Axboe <axboe@kernel.dk>2016-04-19 22:50:23 -0400
committerJens Axboe <axboe@kernel.dk>2017-08-15 08:42:16 -0600
commit5e83ee1aea3572a443c72272bac6f1abc1f26e24 (patch)
tree287f62335eb913e865d73afd5f2c79dfaf85d3f3
parente26361868c17ce2fdcf3cad2bb86a2d978d3d816 (diff)
direct-io: enable cpu_cache_alloc frontend for dio allocations
Signed-off-by: Jens Axboe <axboe@kernel.dk>
-rw-r--r--fs/direct-io.c15
1 files changed, 10 insertions, 5 deletions
diff --git a/fs/direct-io.c b/fs/direct-io.c
index 08cf27811e5a..dec834f7f218 100644
--- a/fs/direct-io.c
+++ b/fs/direct-io.c
@@ -37,6 +37,7 @@
#include <linux/uio.h>
#include <linux/atomic.h>
#include <linux/prefetch.h>
+#include <linux/pcpu_cache.h>
/*
* How many user pages to map in one call to get_user_pages(). This determines
@@ -44,6 +45,9 @@
*/
#define DIO_PAGES 64
+static struct pcpu_alloc_cache alloc_cache;
+static DEFINE_PER_CPU(struct pcpu_cache, dio_alloc_cache);
+
/*
* This code generally works in units of "dio_blocks". A dio_block is
* somewhere between the hard sector size and the filesystem block size. it
@@ -283,7 +287,7 @@ static ssize_t dio_complete(struct dio *dio, ssize_t ret, bool is_async)
dio->iocb->ki_complete(dio->iocb, ret, 0);
}
- kmem_cache_free(dio_cache, dio);
+ pcpu_cache_free(&alloc_cache, dio);
return ret;
}
@@ -1142,7 +1146,7 @@ do_blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
if (iov_iter_rw(iter) == READ && !iov_iter_count(iter))
return 0;
- dio = kmem_cache_alloc(dio_cache, GFP_KERNEL);
+ dio = pcpu_cache_alloc(&alloc_cache, GFP_KERNEL);
retval = -ENOMEM;
if (!dio)
goto out;
@@ -1166,7 +1170,7 @@ do_blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
end - 1);
if (retval) {
inode_unlock(inode);
- kmem_cache_free(dio_cache, dio);
+ pcpu_cache_free(&alloc_cache, dio);
goto out;
}
}
@@ -1177,7 +1181,7 @@ do_blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
if (iov_iter_rw(iter) == READ && offset >= dio->i_size) {
if (dio->flags & DIO_LOCKING)
inode_unlock(inode);
- kmem_cache_free(dio_cache, dio);
+ pcpu_cache_free(&alloc_cache, dio);
retval = 0;
goto out;
}
@@ -1219,7 +1223,7 @@ do_blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
* We grab i_mutex only for reads so we don't have
* to release it here
*/
- kmem_cache_free(dio_cache, dio);
+ pcpu_cache_free(&alloc_cache, dio);
goto out;
}
}
@@ -1357,6 +1361,7 @@ EXPORT_SYMBOL(__blockdev_direct_IO);
static __init int dio_init(void)
{
dio_cache = KMEM_CACHE(dio, SLAB_PANIC);
+ pcpu_cache_init(&alloc_cache, &dio_alloc_cache, dio_cache);
return 0;
}
module_init(dio_init)