From: Jens Axboe <jens.axboe@oracle.com>
Date: Sat, 26 Sep 2009 15:27:23 +0000 (+0200)
Subject: aio: more buffered commits
X-Git-Url: https://git.kernel.dk/?a=commitdiff_plain;h=ffb36f0fcd24b06323d8a7cdc15f22a3cc3dc1f5;p=linux-block.git

aio: more buffered commits

Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
---

diff --git a/fs/aio.c b/fs/aio.c
index 0d5e06c1ffaa..36ee51a6d761 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -696,12 +696,12 @@ static ssize_t aio_run_iocb(struct kiocb *iocb)
 	 * context.
 	 */
 	BUG_ON(current->io_wait != NULL);
-	current->io_wait = &iocb->ki_wait;
+	current->io_wait = &iocb->ki_wq;
 	ret = retry(iocb);
 	current->io_wait = NULL;
 
 	if (ret != -EIOCBRETRY && ret != -EIOCBQUEUED) {
-		BUG_ON(!list_empty(&iocb->ki_wait.task_list));
+		BUG_ON(!list_empty(&iocb->ki_wq.wait.task_list));
 		aio_complete(iocb, ret, 0);
 	}
 out:
@@ -714,6 +714,7 @@ out:
 		 * this is where we let go so that a subsequent
 		 * "kick" can start the next iteration
 		 */
+		iocb->ki_retry = retry;
 
 		/* will make __queue_kicked_iocb succeed from here on */
 		INIT_LIST_HEAD(&iocb->ki_run_list);
@@ -860,7 +861,7 @@ static void try_queue_kicked_iocb(struct kiocb *iocb)
 	 * than retry has happened before we could queue the iocb.  This also
 	 * means that the retry could have completed and freed our iocb, no
 	 * good. */
-	BUG_ON((!list_empty(&iocb->ki_wait.task_list)));
+	BUG_ON((!list_empty(&iocb->ki_wq.wait.task_list)));
 
 	spin_lock_irqsave(&ctx->ctx_lock, flags);
 	/* set this inside the lock so that we can't race with aio_run_iocb()
@@ -1525,9 +1526,16 @@ static ssize_t aio_setup_iocb(struct kiocb *kiocb)
  * are nested inside ioctx lock (i.e. ctx->wait)
  */
 static int aio_wake_function(wait_queue_t *wait, unsigned mode,
-			     int sync, void *key)
+			     int sync, void *arg)
 {
-	struct kiocb *iocb = container_of(wait, struct kiocb, ki_wait);
+	struct wait_bit_key *key = arg;
+	struct wait_bit_queue *wb
+		= container_of(wait, struct wait_bit_queue, wait);
+	struct kiocb *iocb = container_of(wb, struct kiocb, ki_wq);
+
+	if (wb->key.flags != key->flags || wb->key.bit_nr != key->bit_nr ||
+			test_bit(key->bit_nr, key->flags))
+		return 0;
 
 	list_del_init(&wait->task_list);
 	kick_iocb(iocb);
@@ -1595,8 +1603,8 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
 	req->ki_buf = (char __user *)(unsigned long)iocb->aio_buf;
 	req->ki_left = req->ki_nbytes = iocb->aio_nbytes;
 	req->ki_opcode = iocb->aio_lio_opcode;
-	init_waitqueue_func_entry(&req->ki_wait, aio_wake_function);
-	INIT_LIST_HEAD(&req->ki_wait.task_list);
+	init_waitqueue_func_entry(&req->ki_wq.wait, aio_wake_function);
+	INIT_LIST_HEAD(&req->ki_wq.wait.task_list);
 
 	ret = aio_setup_iocb(req);
 
diff --git a/include/linux/aio.h b/include/linux/aio.h
index aea219d7d8d1..c25c92495c5d 100644
--- a/include/linux/aio.h
+++ b/include/linux/aio.h
@@ -102,7 +102,7 @@ struct kiocb {
 	} ki_obj;
 
 	__u64			ki_user_data;	/* user's data for completion */
-	wait_queue_t		ki_wait;
+	struct wait_bit_queue	ki_wq;
 	loff_t			ki_pos;
 
 	void			*private;
@@ -140,7 +140,7 @@ struct kiocb {
 		(x)->ki_dtor = NULL;			\
 		(x)->ki_obj.tsk = tsk;			\
 		(x)->ki_user_data = 0;                  \
-		init_wait((&(x)->ki_wait));             \
+		init_wait((&(x)->ki_wq.wait));		\
 	} while (0)
 
 #define AIO_RING_MAGIC			0xa10a10a1
@@ -223,8 +223,6 @@ struct mm_struct;
 static inline void exit_aio(struct mm_struct *mm) { }
 #endif /* CONFIG_AIO */
 
-#define io_wait_to_kiocb(wait) container_of(wait, struct kiocb, ki_wait)
-
 static inline struct kiocb *list_kiocb(struct list_head *h)
 {
 	return list_entry(h, struct kiocb, ki_list);
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index 065cb319b3db..6f0b88c9a3e0 100644
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -344,8 +344,14 @@ static inline void lock_page_nosync(struct page *page)
 
 static inline int lock_page_wq(struct page *page, struct wait_bit_queue *wq)
 {
-	if (!trylock_page(page))
+	if (!trylock_page(page)) {
+		DEFINE_WAIT_BIT(wq_stack, &page->flags, PG_locked);
+		
+		if (!wq)
+			wq = &wq_stack;
+
 		return __lock_page_wq(page, wq);
+	}
 
 	return 0;
 }
diff --git a/include/linux/sched.h b/include/linux/sched.h
index e902d15fc708..d8ee9728d2f3 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1455,7 +1455,7 @@ struct task_struct {
 	struct backing_dev_info *backing_dev_info;
 
 	struct io_context *io_context;
-	wait_queue_t *io_wait;
+	struct wait_bit_queue *io_wait;
 
 	unsigned long ptrace_message;
 	siginfo_t *last_siginfo; /* For ptrace use.  */
diff --git a/mm/filemap.c b/mm/filemap.c
index 9e2fc115166b..1669d644a544 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -615,6 +615,11 @@ EXPORT_SYMBOL(end_page_writeback);
 
 int __lock_page_wq(struct page *page, struct wait_bit_queue *wq)
 {
+	if (wq) {
+		wq->key.flags = &page->flags;
+		wq->key.bit_nr = PG_locked;
+	}
+
 	return __wait_on_bit_lock(page_waitqueue(page), wq, sync_page_no_sched,
 							TASK_UNINTERRUPTIBLE);
 }
@@ -1136,7 +1141,7 @@ page_ok:
 
 page_not_up_to_date:
 		/* Get exclusive access to the page ... */
-		error = lock_page_killable(page);
+		error = lock_page_wq(page, current->io_wait);
 		if (unlikely(error))
 			goto readpage_error;
 
@@ -1167,7 +1172,7 @@ readpage:
 		}
 
 		if (!PageUptodate(page)) {
-			error = lock_page_killable(page);
+			error = lock_page_wq(page, current->io_wait);
 			if (unlikely(error))
 				goto readpage_error;
 			if (!PageUptodate(page)) {