block: add a request cache freelist in the io_context

Instead of always directly freeing a request, add a freelist
in the io_context. This is done if we are under the full limit
of the queue, with the idea being that a busy disk should then
never touch the memory allocator.

We need to augment this with some way of responding to memory
pressure.

Signed-off-by: Jens Axboe <jaxboe@fusionio.com>

4 files changed, 84 insertions, 9 deletions

diff --git a/block/blk-core.c b/block/blk-core.c
index 1eaec60f95b5..cbaa33e9bf64 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -660,8 +660,6 @@ static inline void blk_free_request(struct request_queue *q,struct request *rq)
 		atomic_dec(&q->elvpriv);
 	}
 
-	blk_slab_free_request(rq);
-
 	if (ioc) {
 		bool do_wakeup = false;
 		unsigned long flags;
@@ -670,6 +668,10 @@ static inline void blk_free_request(struct request_queue *q,struct request *rq)
 		ioc->count[is_sync]--;
 		if (waitqueue_active(&ioc->wait[is_sync]))
 			do_wakeup = true;
+
+		if (blk_ioc_cache_request(q, ioc, rq))
+			rq = NULL;
+
 		spin_unlock_irqrestore(&ioc->lock, flags);
 
 		if (do_wakeup)
@@ -677,16 +679,26 @@ static inline void blk_free_request(struct request_queue *q,struct request *rq)
 
 		put_io_context(ioc);
 	}
+
+	/*
+	 * rq will only be NULL if we decided to move it into the ioc cache
+	 */
+	if (rq)
+		blk_slab_free_request(rq);
 }
 
-static struct request *blk_alloc_request(struct request_queue *q, int flags,
+static struct request *blk_alloc_request(struct request_queue *q,
+					 struct io_context *ioc, int flags,
 					 int priv, gfp_t gfp_mask)
 {
-	struct request *rq;
+	struct request *rq = NULL;
 
-	if (!(flags & REQ_MEMPOOL))
-		rq = kmem_cache_alloc_node(request_cachep, gfp_mask, q->node);
-	else
+	if (!(flags & REQ_MEMPOOL)) {
+		rq = blk_ioc_get_cached_request(ioc);
+		if (!rq)
+			rq = kmem_cache_alloc_node(request_cachep, gfp_mask,
+							q->node);
+	} else
 		rq = mempool_alloc(q->rq_pool, gfp_mask);
 
 	if (!rq)
@@ -736,7 +748,7 @@ static struct request *get_request(struct request_queue *q, int rw_flags,
 	if (blk_queue_io_stat(q))
 		rw_flags |= REQ_IO_STAT;
 
-	rq = blk_alloc_request(q, rw_flags, priv, gfp_mask);
+	rq = blk_alloc_request(q, ioc, rw_flags, priv, gfp_mask);
 	if (!rq) {
 		/*
 		 * If there's no ioc or it doesn't have requests in
@@ -746,7 +758,7 @@ static struct request *get_request(struct request_queue *q, int rw_flags,
 		if ((gfp_mask & __GFP_WAIT) &&
 		    (!ioc || !(ioc->count[0] + ioc->count[1]))) {
 			rw_flags |= REQ_MEMPOOL;
-			rq = blk_alloc_request(q, rw_flags, priv, gfp_mask);
+			rq = blk_alloc_request(q, ioc,rw_flags, priv, gfp_mask);
 		}
 
 		if (!rq) {
diff --git a/block/blk-ioc.c b/block/blk-ioc.c
index 8807d5297bb5..357d0b8e9bee 100644
--- a/block/blk-ioc.c
+++ b/block/blk-ioc.c
@@ -16,6 +16,47 @@
  */
 static struct kmem_cache *iocontext_cachep;
 
+struct request *blk_ioc_get_cached_request(struct io_context *ioc)
+{
+	struct request *rq = NULL;
+
+	/*
+	 * See if there are available cached requests
+	 */
+	if (!list_empty_careful(&ioc->free_list)) {
+		spin_lock_irq(&ioc->lock);
+		if (!list_empty(&ioc->free_list)) {
+			rq = list_entry(ioc->free_list.next, struct request,
+					queuelist);
+			list_del_init(&rq->queuelist);
+		}
+		spin_unlock_irq(&ioc->lock);
+	}
+
+	return rq;
+}
+
+bool blk_ioc_cache_request(struct request_queue *q, struct io_context *ioc,
+			   struct request *rq)
+{
+	/*
+	 * Don't cache something that came from the emergency pool
+	 */
+	if (rq->cmd_flags & REQ_MEMPOOL)
+		return false;
+
+	/*
+	 * cache this request for reuse
+	 */
+	if ((rq->cmd_flags & REQ_ALLOCED) &&
+	    (ioc->free_requests < 2 * q->nr_requests)) {
+		list_add(&rq->queuelist, &ioc->free_list);
+		return true;
+	}
+
+	return false;
+}
+
 static void cfq_dtor(struct io_context *ioc)
 {
 	if (!hlist_empty(&ioc->cic_list)) {
@@ -43,6 +84,20 @@ int put_io_context(struct io_context *ioc)
 		cfq_dtor(ioc);
 		rcu_read_unlock();
 
+		/*
+		 * free cached requests, if any. this is safe without the
+		 * lock, since we are the only ones that are still holding
+		 * a reference to it and others cannot grab a ref to it now.
+		 */
+		while (!list_empty(&ioc->free_list)) {
+			struct request *rq;
+
+			rq = list_entry(ioc->free_list.next, struct request,
+					queuelist);
+			list_del(&rq->queuelist);
+			blk_slab_free_request(rq);
+		}
+
 		kmem_cache_free(iocontext_cachep, ioc);
 		return 1;
 	}
@@ -93,6 +148,8 @@ struct io_context *alloc_io_context(gfp_t gfp_flags, int node)
 		ret->ioprio = 0;
 		init_waitqueue_head(&ret->wait[BLK_RW_SYNC]);
 		init_waitqueue_head(&ret->wait[BLK_RW_ASYNC]);
+		INIT_LIST_HEAD(&ret->free_list);
+		ret->free_requests = 0;
 		ret->count[BLK_RW_SYNC] = ret->count[BLK_RW_ASYNC] = 0;
 		INIT_RADIX_TREE(&ret->radix_root, GFP_ATOMIC | __GFP_HIGH);
 		INIT_HLIST_HEAD(&ret->cic_list);
diff --git a/block/blk.h b/block/blk.h
index 87d25c835ca6..30187f2500ed 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -105,12 +105,16 @@ int attempt_back_merge(struct request_queue *q, struct request *rq);
 int attempt_front_merge(struct request_queue *q, struct request *rq);
 void blk_recalc_rq_segments(struct request *rq);
 void blk_rq_set_mixed_merge(struct request *rq);
+void blk_slab_free_request(struct request *rq);
 
 int blk_dev_init(void);
 
 void elv_quiesce_start(struct request_queue *q);
 void elv_quiesce_end(struct request_queue *q);
 
+struct request *blk_ioc_get_cached_request(struct io_context *ioc);
+bool blk_ioc_cache_request(struct request_queue *q, struct io_context *ioc, struct request *rq);
+
 static inline int blk_cpu_to_group(int cpu)
 {
 	int group = NR_CPUS;
diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h
index 8a3dc6c05fae..1e51568ce68a 100644
--- a/include/linux/iocontext.h
+++ b/include/linux/iocontext.h
@@ -49,6 +49,8 @@ struct io_context {
 	 */
 	int count[2];
 	wait_queue_head_t wait[2];
+	struct list_head free_list;
+	unsigned int free_requests;
 
 	struct radix_tree_root radix_root;
 	struct hlist_head cic_list;