aio: more buffered commits

Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

diff --git a/fs/aio.c b/fs/aio.c
index 0d5e06c1ffaa6377e3a64c561c9e86d498071dbf..36ee51a6d76157e4e53c339130f996eae4bd84d8 100644 (file)
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -696,12 +696,12 @@ static ssize_t aio_run_iocb(struct kiocb *iocb)
         * context.
         */
        BUG_ON(current->io_wait != NULL);
-       current->io_wait = &iocb->ki_wait;
+       current->io_wait = &iocb->ki_wq;
        ret = retry(iocb);
        current->io_wait = NULL;
 
        if (ret != -EIOCBRETRY && ret != -EIOCBQUEUED) {
-               BUG_ON(!list_empty(&iocb->ki_wait.task_list));
+               BUG_ON(!list_empty(&iocb->ki_wq.wait.task_list));
                aio_complete(iocb, ret, 0);
        }
 out:
@@ -714,6 +714,7 @@ out:
                 * this is where we let go so that a subsequent
                 * "kick" can start the next iteration
                 */
+               iocb->ki_retry = retry;
 
                /* will make __queue_kicked_iocb succeed from here on */
                INIT_LIST_HEAD(&iocb->ki_run_list);
@@ -860,7 +861,7 @@ static void try_queue_kicked_iocb(struct kiocb *iocb)
         * than retry has happened before we could queue the iocb.  This also
         * means that the retry could have completed and freed our iocb, no
         * good. */
-       BUG_ON((!list_empty(&iocb->ki_wait.task_list)));
+       BUG_ON((!list_empty(&iocb->ki_wq.wait.task_list)));
 
        spin_lock_irqsave(&ctx->ctx_lock, flags);
        /* set this inside the lock so that we can't race with aio_run_iocb()
@@ -1525,9 +1526,16 @@ static ssize_t aio_setup_iocb(struct kiocb *kiocb)
  * are nested inside ioctx lock (i.e. ctx->wait)
  */
 static int aio_wake_function(wait_queue_t *wait, unsigned mode,
-                            int sync, void *key)
+                            int sync, void *arg)
 {
-       struct kiocb *iocb = container_of(wait, struct kiocb, ki_wait);
+       struct wait_bit_key *key = arg;
+       struct wait_bit_queue *wb
+               = container_of(wait, struct wait_bit_queue, wait);
+       struct kiocb *iocb = container_of(wb, struct kiocb, ki_wq);
+
+       if (wb->key.flags != key->flags || wb->key.bit_nr != key->bit_nr ||
+                       test_bit(key->bit_nr, key->flags))
+               return 0;
 
        list_del_init(&wait->task_list);
        kick_iocb(iocb);
@@ -1595,8 +1603,8 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
        req->ki_buf = (char __user *)(unsigned long)iocb->aio_buf;
        req->ki_left = req->ki_nbytes = iocb->aio_nbytes;
        req->ki_opcode = iocb->aio_lio_opcode;
-       init_waitqueue_func_entry(&req->ki_wait, aio_wake_function);
-       INIT_LIST_HEAD(&req->ki_wait.task_list);
+       init_waitqueue_func_entry(&req->ki_wq.wait, aio_wake_function);
+       INIT_LIST_HEAD(&req->ki_wq.wait.task_list);
 
        ret = aio_setup_iocb(req);
 

diff --git a/include/linux/aio.h b/include/linux/aio.h
index aea219d7d8d1afec462b546df8c3412c915eecf2..c25c92495c5dc56a7cbcdd9d1009a1478730e64e 100644 (file)
--- a/include/linux/aio.h
+++ b/include/linux/aio.h
@@ -102,7 +102,7 @@ struct kiocb {
        } ki_obj;
 
        __u64                   ki_user_data;   /* user's data for completion */
-       wait_queue_t            ki_wait;
+       struct wait_bit_queue   ki_wq;
        loff_t                  ki_pos;
 
        void                    *private;
@@ -140,7 +140,7 @@ struct kiocb {
                (x)->ki_dtor = NULL;                    \
                (x)->ki_obj.tsk = tsk;                  \
                (x)->ki_user_data = 0;                  \
-               init_wait((&(x)->ki_wait));             \
+               init_wait((&(x)->ki_wq.wait));          \
        } while (0)
 
 #define AIO_RING_MAGIC                 0xa10a10a1
@@ -223,8 +223,6 @@ struct mm_struct;
 static inline void exit_aio(struct mm_struct *mm) { }
 #endif /* CONFIG_AIO */
 
-#define io_wait_to_kiocb(wait) container_of(wait, struct kiocb, ki_wait)
-
 static inline struct kiocb *list_kiocb(struct list_head *h)
 {
        return list_entry(h, struct kiocb, ki_list);

diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index 065cb319b3db954eb0b2b73a1394dcfc252a85b8..6f0b88c9a3e06c8fdf8e97a96c225d61c1a6d936 100644 (file)
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -344,8 +344,14 @@ static inline void lock_page_nosync(struct page *page)
 
 static inline int lock_page_wq(struct page *page, struct wait_bit_queue *wq)
 {
-       if (!trylock_page(page))
+       if (!trylock_page(page)) {
+               DEFINE_WAIT_BIT(wq_stack, &page->flags, PG_locked);
+               
+               if (!wq)
+                       wq = &wq_stack;
+
                return __lock_page_wq(page, wq);
+       }
 
        return 0;
 }

diff --git a/include/linux/sched.h b/include/linux/sched.h
index e902d15fc708b498741b0781f70f2862c3af35f5..d8ee9728d2f38fddf7b72c745d3b8c7e981b93db 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1455,7 +1455,7 @@ struct task_struct {
        struct backing_dev_info *backing_dev_info;
 
        struct io_context *io_context;
-       wait_queue_t *io_wait;
+       struct wait_bit_queue *io_wait;
 
        unsigned long ptrace_message;
        siginfo_t *last_siginfo; /* For ptrace use.  */

diff --git a/mm/filemap.c b/mm/filemap.c
index 9e2fc115166bccaa7e511c4a293c24da32d75295..1669d644a54458cf415ed1d3c288fb8e986d47a8 100644 (file)
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -615,6 +615,11 @@ EXPORT_SYMBOL(end_page_writeback);
 
 int __lock_page_wq(struct page *page, struct wait_bit_queue *wq)
 {
+       if (wq) {
+               wq->key.flags = &page->flags;
+               wq->key.bit_nr = PG_locked;
+       }
+
        return __wait_on_bit_lock(page_waitqueue(page), wq, sync_page_no_sched,
                                                        TASK_UNINTERRUPTIBLE);
 }
@@ -1136,7 +1141,7 @@ page_ok:
 
 page_not_up_to_date:
                /* Get exclusive access to the page ... */
-               error = lock_page_killable(page);
+               error = lock_page_wq(page, current->io_wait);
                if (unlikely(error))
                        goto readpage_error;
 
@@ -1167,7 +1172,7 @@ readpage:
                }
 
                if (!PageUptodate(page)) {
-                       error = lock_page_killable(page);
+                       error = lock_page_wq(page, current->io_wait);
                        if (unlikely(error))
                                goto readpage_error;
                        if (!PageUptodate(page)) {