aio: fix io_getevents documentation
[linux-2.6-block.git] / fs / aio.c
index 263ebce940c08470719ac59d7979fe18d9bdad7c..3fcdd73b6f1b17e2dc71533ab78c6f8c406639de 100644 (file)
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -58,18 +58,6 @@ struct aio_ring {
 }; /* 128 bytes + ring size */
 
 #define AIO_RING_PAGES 8
-struct aio_ring_info {
-       unsigned long           mmap_base;
-       unsigned long           mmap_size;
-
-       struct page             **ring_pages;
-       struct mutex            ring_lock;
-       long                    nr_pages;
-
-       unsigned                nr, tail;
-
-       struct page             *internal_pages[AIO_RING_PAGES];
-};
 
 struct kioctx {
        atomic_t                users;
@@ -79,28 +67,47 @@ struct kioctx {
        unsigned long           user_id;
        struct hlist_node       list;
 
-       wait_queue_head_t       wait;
-
-       spinlock_t              ctx_lock;
-
-       atomic_t                reqs_active;
-       struct list_head        active_reqs;    /* used for cancellation */
-
        /*
         * This is what userspace passed to io_setup(), it's not used for
         * anything but counting against the global max_reqs quota.
         *
-        * The real limit is ring->nr - 1, which will be larger (see
+        * The real limit is nr_events - 1, which will be larger (see
         * aio_setup_ring())
         */
        unsigned                max_reqs;
 
-       struct aio_ring_info    ring_info;
+       /* Size of ringbuffer, in units of struct io_event */
+       unsigned                nr_events;
 
-       spinlock_t              completion_lock;
+       unsigned long           mmap_base;
+       unsigned long           mmap_size;
+
+       struct page             **ring_pages;
+       long                    nr_pages;
 
        struct rcu_head         rcu_head;
        struct work_struct      rcu_work;
+
+       struct {
+               atomic_t        reqs_active;
+       } ____cacheline_aligned_in_smp;
+
+       struct {
+               spinlock_t      ctx_lock;
+               struct list_head active_reqs;   /* used for cancellation */
+       } ____cacheline_aligned_in_smp;
+
+       struct {
+               struct mutex    ring_lock;
+               wait_queue_head_t wait;
+       } ____cacheline_aligned_in_smp;
+
+       struct {
+               unsigned        tail;
+               spinlock_t      completion_lock;
+       } ____cacheline_aligned_in_smp;
+
+       struct page             *internal_pages[AIO_RING_PAGES];
 };
 
 /*------ sysctl variables----*/
@@ -129,26 +136,21 @@ __initcall(aio_setup);
 
 static void aio_free_ring(struct kioctx *ctx)
 {
-       struct aio_ring_info *info = &ctx->ring_info;
        long i;
 
-       for (i=0; i<info->nr_pages; i++)
-               put_page(info->ring_pages[i]);
+       for (i = 0; i < ctx->nr_pages; i++)
+               put_page(ctx->ring_pages[i]);
 
-       if (info->mmap_size) {
-               vm_munmap(info->mmap_base, info->mmap_size);
-       }
+       if (ctx->mmap_size)
+               vm_munmap(ctx->mmap_base, ctx->mmap_size);
 
-       if (info->ring_pages && info->ring_pages != info->internal_pages)
-               kfree(info->ring_pages);
-       info->ring_pages = NULL;
-       info->nr = 0;
+       if (ctx->ring_pages && ctx->ring_pages != ctx->internal_pages)
+               kfree(ctx->ring_pages);
 }
 
 static int aio_setup_ring(struct kioctx *ctx)
 {
        struct aio_ring *ring;
-       struct aio_ring_info *info = &ctx->ring_info;
        unsigned nr_events = ctx->max_reqs;
        struct mm_struct *mm = current->mm;
        unsigned long size, populate;
@@ -166,45 +168,44 @@ static int aio_setup_ring(struct kioctx *ctx)
 
        nr_events = (PAGE_SIZE * nr_pages - sizeof(struct aio_ring)) / sizeof(struct io_event);
 
-       info->nr = 0;
-       info->ring_pages = info->internal_pages;
+       ctx->nr_events = 0;
+       ctx->ring_pages = ctx->internal_pages;
        if (nr_pages > AIO_RING_PAGES) {
-               info->ring_pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);
-               if (!info->ring_pages)
+               ctx->ring_pages = kcalloc(nr_pages, sizeof(struct page *),
+                                         GFP_KERNEL);
+               if (!ctx->ring_pages)
                        return -ENOMEM;
        }
 
-       info->mmap_size = nr_pages * PAGE_SIZE;
-       pr_debug("attempting mmap of %lu bytes\n", info->mmap_size);
+       ctx->mmap_size = nr_pages * PAGE_SIZE;
+       pr_debug("attempting mmap of %lu bytes\n", ctx->mmap_size);
        down_write(&mm->mmap_sem);
-       info->mmap_base = do_mmap_pgoff(NULL, 0, info->mmap_size, 
-                                       PROT_READ|PROT_WRITE,
-                                       MAP_ANONYMOUS|MAP_PRIVATE, 0,
-                                       &populate);
-       if (IS_ERR((void *)info->mmap_base)) {
+       ctx->mmap_base = do_mmap_pgoff(NULL, 0, ctx->mmap_size,
+                                      PROT_READ|PROT_WRITE,
+                                      MAP_ANONYMOUS|MAP_PRIVATE, 0, &populate);
+       if (IS_ERR((void *)ctx->mmap_base)) {
                up_write(&mm->mmap_sem);
-               info->mmap_size = 0;
+               ctx->mmap_size = 0;
                aio_free_ring(ctx);
                return -EAGAIN;
        }
 
-       pr_debug("mmap address: 0x%08lx\n", info->mmap_base);
-       info->nr_pages = get_user_pages(current, mm, info->mmap_base, nr_pages,
-                                       1, 0, info->ring_pages, NULL);
+       pr_debug("mmap address: 0x%08lx\n", ctx->mmap_base);
+       ctx->nr_pages = get_user_pages(current, mm, ctx->mmap_base, nr_pages,
+                                      1, 0, ctx->ring_pages, NULL);
        up_write(&mm->mmap_sem);
 
-       if (unlikely(info->nr_pages != nr_pages)) {
+       if (unlikely(ctx->nr_pages != nr_pages)) {
                aio_free_ring(ctx);
                return -EAGAIN;
        }
        if (populate)
-               mm_populate(info->mmap_base, populate);
+               mm_populate(ctx->mmap_base, populate);
 
-       ctx->user_id = info->mmap_base;
+       ctx->user_id = ctx->mmap_base;
+       ctx->nr_events = nr_events; /* trusted copy */
 
-       info->nr = nr_events;           /* trusted copy */
-
-       ring = kmap_atomic(info->ring_pages[0]);
+       ring = kmap_atomic(ctx->ring_pages[0]);
        ring->nr = nr_events;   /* user copy */
        ring->id = ctx->user_id;
        ring->head = ring->tail = 0;
@@ -213,7 +214,7 @@ static int aio_setup_ring(struct kioctx *ctx)
        ring->incompat_features = AIO_RING_INCOMPAT_FEATURES;
        ring->header_length = sizeof(struct aio_ring);
        kunmap_atomic(ring);
-       flush_dcache_page(info->ring_pages[0]);
+       flush_dcache_page(ctx->ring_pages[0]);
 
        return 0;
 }
@@ -284,7 +285,6 @@ static void free_ioctx_rcu(struct rcu_head *head)
  */
 static void free_ioctx(struct kioctx *ctx)
 {
-       struct aio_ring_info *info = &ctx->ring_info;
        struct aio_ring *ring;
        struct io_event res;
        struct kiocb *req;
@@ -302,18 +302,18 @@ static void free_ioctx(struct kioctx *ctx)
 
        spin_unlock_irq(&ctx->ctx_lock);
 
-       ring = kmap_atomic(info->ring_pages[0]);
+       ring = kmap_atomic(ctx->ring_pages[0]);
        head = ring->head;
        kunmap_atomic(ring);
 
        while (atomic_read(&ctx->reqs_active) > 0) {
-               wait_event(ctx->wait, head != info->tail);
+               wait_event(ctx->wait, head != ctx->tail);
 
-               avail = (head <= info->tail ? info->tail : info->nr) - head;
+               avail = (head <= ctx->tail ? ctx->tail : ctx->nr_events) - head;
 
                atomic_sub(avail, &ctx->reqs_active);
                head += avail;
-               head %= info->nr;
+               head %= ctx->nr_events;
        }
 
        WARN_ON(atomic_read(&ctx->reqs_active) < 0);
@@ -372,7 +372,7 @@ static struct kioctx *ioctx_alloc(unsigned nr_events)
        atomic_set(&ctx->dead, 0);
        spin_lock_init(&ctx->ctx_lock);
        spin_lock_init(&ctx->completion_lock);
-       mutex_init(&ctx->ring_info.ring_lock);
+       mutex_init(&ctx->ring_lock);
        init_waitqueue_head(&ctx->wait);
 
        INIT_LIST_HEAD(&ctx->active_reqs);
@@ -396,7 +396,7 @@ static struct kioctx *ioctx_alloc(unsigned nr_events)
        spin_unlock(&mm->ioctx_lock);
 
        pr_debug("allocated ioctx %p[%ld]: mm=%p mask=0x%x\n",
-               ctx, ctx->user_id, mm, ctx->ring_info.nr);
+                ctx, ctx->user_id, mm, ctx->nr_events);
        return ctx;
 
 out_cleanup:
@@ -491,7 +491,7 @@ void exit_aio(struct mm_struct *mm)
                 * just set it to 0; aio_free_ring() is the only
                 * place that uses ->mmap_size, so it's safe.
                 */
-               ctx->ring_info.mmap_size = 0;
+               ctx->mmap_size = 0;
 
                if (!atomic_xchg(&ctx->dead, 1)) {
                        hlist_del_rcu(&ctx->list);
@@ -514,10 +514,10 @@ static inline struct kiocb *aio_get_req(struct kioctx *ctx)
 {
        struct kiocb *req;
 
-       if (atomic_read(&ctx->reqs_active) >= ctx->ring_info.nr)
+       if (atomic_read(&ctx->reqs_active) >= ctx->nr_events)
                return NULL;
 
-       if (atomic_inc_return(&ctx->reqs_active) > ctx->ring_info.nr - 1)
+       if (atomic_inc_return(&ctx->reqs_active) > ctx->nr_events - 1)
                goto out_put;
 
        req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL|__GFP_ZERO);
@@ -578,7 +578,6 @@ static struct kioctx *lookup_ioctx(unsigned long ctx_id)
 void aio_complete(struct kiocb *iocb, long res, long res2)
 {
        struct kioctx   *ctx = iocb->ki_ctx;
-       struct aio_ring_info    *info;
        struct aio_ring *ring;
        struct io_event *ev_page, *event;
        unsigned long   flags;
@@ -599,8 +598,6 @@ void aio_complete(struct kiocb *iocb, long res, long res2)
                return;
        }
 
-       info = &ctx->ring_info;
-
        /*
         * Take rcu_read_lock() in case the kioctx is being destroyed, as we
         * need to issue a wakeup after decrementing reqs_active.
@@ -633,13 +630,13 @@ void aio_complete(struct kiocb *iocb, long res, long res2)
         */
        spin_lock_irqsave(&ctx->completion_lock, flags);
 
-       tail = info->tail;
+       tail = ctx->tail;
        pos = tail + AIO_EVENTS_OFFSET;
 
-       if (++tail >= info->nr)
+       if (++tail >= ctx->nr_events)
                tail = 0;
 
-       ev_page = kmap_atomic(info->ring_pages[pos / AIO_EVENTS_PER_PAGE]);
+       ev_page = kmap_atomic(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]);
        event = ev_page + pos % AIO_EVENTS_PER_PAGE;
 
        event->obj = (u64)(unsigned long)iocb->ki_obj.user;
@@ -648,7 +645,7 @@ void aio_complete(struct kiocb *iocb, long res, long res2)
        event->res2 = res2;
 
        kunmap_atomic(ev_page);
-       flush_dcache_page(info->ring_pages[pos / AIO_EVENTS_PER_PAGE]);
+       flush_dcache_page(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]);
 
        pr_debug("%p[%u]: %p: %p %Lx %lx %lx\n",
                 ctx, tail, iocb, iocb->ki_obj.user, iocb->ki_user_data,
@@ -659,12 +656,12 @@ void aio_complete(struct kiocb *iocb, long res, long res2)
         */
        smp_wmb();      /* make event visible before updating tail */
 
-       info->tail = tail;
+       ctx->tail = tail;
 
-       ring = kmap_atomic(info->ring_pages[0]);
+       ring = kmap_atomic(ctx->ring_pages[0]);
        ring->tail = tail;
        kunmap_atomic(ring);
-       flush_dcache_page(info->ring_pages[0]);
+       flush_dcache_page(ctx->ring_pages[0]);
 
        spin_unlock_irqrestore(&ctx->completion_lock, flags);
 
@@ -704,21 +701,20 @@ EXPORT_SYMBOL(aio_complete);
 static long aio_read_events_ring(struct kioctx *ctx,
                                 struct io_event __user *event, long nr)
 {
-       struct aio_ring_info *info = &ctx->ring_info;
        struct aio_ring *ring;
        unsigned head, pos;
        long ret = 0;
        int copy_ret;
 
-       mutex_lock(&info->ring_lock);
+       mutex_lock(&ctx->ring_lock);
 
-       ring = kmap_atomic(info->ring_pages[0]);
+       ring = kmap_atomic(ctx->ring_pages[0]);
        head = ring->head;
        kunmap_atomic(ring);
 
-       pr_debug("h%u t%u m%u\n", head, info->tail, info->nr);
+       pr_debug("h%u t%u m%u\n", head, ctx->tail, ctx->nr_events);
 
-       if (head == info->tail)
+       if (head == ctx->tail)
                goto out;
 
        while (ret < nr) {
@@ -726,8 +722,8 @@ static long aio_read_events_ring(struct kioctx *ctx,
                struct io_event *ev;
                struct page *page;
 
-               avail = (head <= info->tail ? info->tail : info->nr) - head;
-               if (head == info->tail)
+               avail = (head <= ctx->tail ? ctx->tail : ctx->nr_events) - head;
+               if (head == ctx->tail)
                        break;
 
                avail = min(avail, nr - ret);
@@ -735,7 +731,7 @@ static long aio_read_events_ring(struct kioctx *ctx,
                            ((head + AIO_EVENTS_OFFSET) % AIO_EVENTS_PER_PAGE));
 
                pos = head + AIO_EVENTS_OFFSET;
-               page = info->ring_pages[pos / AIO_EVENTS_PER_PAGE];
+               page = ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE];
                pos %= AIO_EVENTS_PER_PAGE;
 
                ev = kmap(page);
@@ -750,19 +746,19 @@ static long aio_read_events_ring(struct kioctx *ctx,
 
                ret += avail;
                head += avail;
-               head %= info->nr;
+               head %= ctx->nr_events;
        }
 
-       ring = kmap_atomic(info->ring_pages[0]);
+       ring = kmap_atomic(ctx->ring_pages[0]);
        ring->head = head;
        kunmap_atomic(ring);
-       flush_dcache_page(info->ring_pages[0]);
+       flush_dcache_page(ctx->ring_pages[0]);
 
-       pr_debug("%li  h%u t%u\n", ret, head, info->tail);
+       pr_debug("%li  h%u t%u\n", ret, head, ctx->tail);
 
        atomic_sub(ret, &ctx->reqs_active);
 out:
-       mutex_unlock(&info->ring_lock);
+       mutex_unlock(&ctx->ring_lock);
 
        return ret;
 }
@@ -907,30 +903,21 @@ static void aio_advance_iovec(struct kiocb *iocb, ssize_t ret)
        BUG_ON(ret > 0 && iocb->ki_left == 0);
 }
 
-static ssize_t aio_rw_vect_retry(struct kiocb *iocb)
+typedef ssize_t (aio_rw_op)(struct kiocb *, const struct iovec *,
+                           unsigned long, loff_t);
+
+static ssize_t aio_rw_vect_retry(struct kiocb *iocb, int rw, aio_rw_op *rw_op)
 {
        struct file *file = iocb->ki_filp;
        struct address_space *mapping = file->f_mapping;
        struct inode *inode = mapping->host;
-       ssize_t (*rw_op)(struct kiocb *, const struct iovec *,
-                        unsigned long, loff_t);
        ssize_t ret = 0;
-       unsigned short opcode;
-
-       if ((iocb->ki_opcode == IOCB_CMD_PREADV) ||
-               (iocb->ki_opcode == IOCB_CMD_PREAD)) {
-               rw_op = file->f_op->aio_read;
-               opcode = IOCB_CMD_PREADV;
-       } else {
-               rw_op = file->f_op->aio_write;
-               opcode = IOCB_CMD_PWRITEV;
-       }
 
        /* This matches the pread()/pwrite() logic */
        if (iocb->ki_pos < 0)
                return -EINVAL;
 
-       if (opcode == IOCB_CMD_PWRITEV)
+       if (rw == WRITE)
                file_start_write(file);
        do {
                ret = rw_op(iocb, &iocb->ki_iovec[iocb->ki_cur_seg],
@@ -942,9 +929,9 @@ static ssize_t aio_rw_vect_retry(struct kiocb *iocb)
        /* retry all partial writes.  retry partial reads as long as its a
         * regular file. */
        } while (ret > 0 && iocb->ki_left > 0 &&
-                (opcode == IOCB_CMD_PWRITEV ||
+                (rw == WRITE ||
                  (!S_ISFIFO(inode->i_mode) && !S_ISSOCK(inode->i_mode))));
-       if (opcode == IOCB_CMD_PWRITEV)
+       if (rw == WRITE)
                file_end_write(file);
 
        /* This means we must have transferred all that we could */
@@ -954,7 +941,7 @@ static ssize_t aio_rw_vect_retry(struct kiocb *iocb)
 
        /* If we managed to write some out we return that, rather than
         * the eventual error. */
-       if (opcode == IOCB_CMD_PWRITEV
+       if (rw == WRITE
            && ret < 0 && ret != -EIOCBQUEUED
            && iocb->ki_nbytes - iocb->ki_left)
                ret = iocb->ki_nbytes - iocb->ki_left;
@@ -962,73 +949,41 @@ static ssize_t aio_rw_vect_retry(struct kiocb *iocb)
        return ret;
 }
 
-static ssize_t aio_fdsync(struct kiocb *iocb)
-{
-       struct file *file = iocb->ki_filp;
-       ssize_t ret = -EINVAL;
-
-       if (file->f_op->aio_fsync)
-               ret = file->f_op->aio_fsync(iocb, 1);
-       return ret;
-}
-
-static ssize_t aio_fsync(struct kiocb *iocb)
-{
-       struct file *file = iocb->ki_filp;
-       ssize_t ret = -EINVAL;
-
-       if (file->f_op->aio_fsync)
-               ret = file->f_op->aio_fsync(iocb, 0);
-       return ret;
-}
-
-static ssize_t aio_setup_vectored_rw(int type, struct kiocb *kiocb, bool compat)
+static ssize_t aio_setup_vectored_rw(int rw, struct kiocb *kiocb, bool compat)
 {
        ssize_t ret;
 
+       kiocb->ki_nr_segs = kiocb->ki_nbytes;
+
 #ifdef CONFIG_COMPAT
        if (compat)
-               ret = compat_rw_copy_check_uvector(type,
+               ret = compat_rw_copy_check_uvector(rw,
                                (struct compat_iovec __user *)kiocb->ki_buf,
-                               kiocb->ki_nbytes, 1, &kiocb->ki_inline_vec,
+                               kiocb->ki_nr_segs, 1, &kiocb->ki_inline_vec,
                                &kiocb->ki_iovec);
        else
 #endif
-               ret = rw_copy_check_uvector(type,
+               ret = rw_copy_check_uvector(rw,
                                (struct iovec __user *)kiocb->ki_buf,
-                               kiocb->ki_nbytes, 1, &kiocb->ki_inline_vec,
+                               kiocb->ki_nr_segs, 1, &kiocb->ki_inline_vec,
                                &kiocb->ki_iovec);
        if (ret < 0)
-               goto out;
-
-       ret = rw_verify_area(type, kiocb->ki_filp, &kiocb->ki_pos, ret);
-       if (ret < 0)
-               goto out;
+               return ret;
 
-       kiocb->ki_nr_segs = kiocb->ki_nbytes;
-       kiocb->ki_cur_seg = 0;
-       /* ki_nbytes/left now reflect bytes instead of segs */
+       /* ki_nbytes now reflect bytes instead of segs */
        kiocb->ki_nbytes = ret;
-       kiocb->ki_left = ret;
-
-       ret = 0;
-out:
-       return ret;
+       return 0;
 }
 
-static ssize_t aio_setup_single_vector(int type, struct file * file, struct kiocb *kiocb)
+static ssize_t aio_setup_single_vector(int rw, struct kiocb *kiocb)
 {
-       int bytes;
-
-       bytes = rw_verify_area(type, file, &kiocb->ki_pos, kiocb->ki_left);
-       if (bytes < 0)
-               return bytes;
+       if (unlikely(!access_ok(!rw, kiocb->ki_buf, kiocb->ki_nbytes)))
+               return -EFAULT;
 
        kiocb->ki_iovec = &kiocb->ki_inline_vec;
        kiocb->ki_iovec->iov_base = kiocb->ki_buf;
-       kiocb->ki_iovec->iov_len = bytes;
+       kiocb->ki_iovec->iov_len = kiocb->ki_nbytes;
        kiocb->ki_nr_segs = 1;
-       kiocb->ki_cur_seg = 0;
        return 0;
 }
 
@@ -1037,81 +992,82 @@ static ssize_t aio_setup_single_vector(int type, struct file * file, struct kioc
  *     Performs the initial checks and aio retry method
  *     setup for the kiocb at the time of io submission.
  */
-static ssize_t aio_setup_iocb(struct kiocb *kiocb, bool compat)
+static ssize_t aio_run_iocb(struct kiocb *req, bool compat)
 {
-       struct file *file = kiocb->ki_filp;
-       ssize_t ret = 0;
+       struct file *file = req->ki_filp;
+       ssize_t ret;
+       int rw;
+       fmode_t mode;
+       aio_rw_op *rw_op;
 
-       switch (kiocb->ki_opcode) {
+       switch (req->ki_opcode) {
        case IOCB_CMD_PREAD:
-               ret = -EBADF;
-               if (unlikely(!(file->f_mode & FMODE_READ)))
-                       break;
-               ret = -EFAULT;
-               if (unlikely(!access_ok(VERIFY_WRITE, kiocb->ki_buf,
-                       kiocb->ki_left)))
-                       break;
-               ret = aio_setup_single_vector(READ, file, kiocb);
-               if (ret)
-                       break;
-               ret = -EINVAL;
-               if (file->f_op->aio_read)
-                       kiocb->ki_retry = aio_rw_vect_retry;
-               break;
-       case IOCB_CMD_PWRITE:
-               ret = -EBADF;
-               if (unlikely(!(file->f_mode & FMODE_WRITE)))
-                       break;
-               ret = -EFAULT;
-               if (unlikely(!access_ok(VERIFY_READ, kiocb->ki_buf,
-                       kiocb->ki_left)))
-                       break;
-               ret = aio_setup_single_vector(WRITE, file, kiocb);
-               if (ret)
-                       break;
-               ret = -EINVAL;
-               if (file->f_op->aio_write)
-                       kiocb->ki_retry = aio_rw_vect_retry;
-               break;
        case IOCB_CMD_PREADV:
-               ret = -EBADF;
-               if (unlikely(!(file->f_mode & FMODE_READ)))
-                       break;
-               ret = aio_setup_vectored_rw(READ, kiocb, compat);
-               if (ret)
-                       break;
-               ret = -EINVAL;
-               if (file->f_op->aio_read)
-                       kiocb->ki_retry = aio_rw_vect_retry;
-               break;
+               mode    = FMODE_READ;
+               rw      = READ;
+               rw_op   = file->f_op->aio_read;
+               goto rw_common;
+
+       case IOCB_CMD_PWRITE:
        case IOCB_CMD_PWRITEV:
-               ret = -EBADF;
-               if (unlikely(!(file->f_mode & FMODE_WRITE)))
-                       break;
-               ret = aio_setup_vectored_rw(WRITE, kiocb, compat);
+               mode    = FMODE_WRITE;
+               rw      = WRITE;
+               rw_op   = file->f_op->aio_write;
+               goto rw_common;
+rw_common:
+               if (unlikely(!(file->f_mode & mode)))
+                       return -EBADF;
+
+               if (!rw_op)
+                       return -EINVAL;
+
+               ret = (req->ki_opcode == IOCB_CMD_PREADV ||
+                      req->ki_opcode == IOCB_CMD_PWRITEV)
+                       ? aio_setup_vectored_rw(rw, req, compat)
+                       : aio_setup_single_vector(rw, req);
                if (ret)
-                       break;
-               ret = -EINVAL;
-               if (file->f_op->aio_write)
-                       kiocb->ki_retry = aio_rw_vect_retry;
+                       return ret;
+
+               ret = rw_verify_area(rw, file, &req->ki_pos, req->ki_nbytes);
+               if (ret < 0)
+                       return ret;
+
+               req->ki_nbytes = ret;
+               req->ki_left = ret;
+
+               ret = aio_rw_vect_retry(req, rw, rw_op);
                break;
+
        case IOCB_CMD_FDSYNC:
-               ret = -EINVAL;
-               if (file->f_op->aio_fsync)
-                       kiocb->ki_retry = aio_fdsync;
+               if (!file->f_op->aio_fsync)
+                       return -EINVAL;
+
+               ret = file->f_op->aio_fsync(req, 1);
                break;
+
        case IOCB_CMD_FSYNC:
-               ret = -EINVAL;
-               if (file->f_op->aio_fsync)
-                       kiocb->ki_retry = aio_fsync;
+               if (!file->f_op->aio_fsync)
+                       return -EINVAL;
+
+               ret = file->f_op->aio_fsync(req, 0);
                break;
+
        default:
                pr_debug("EINVAL: no operation provided\n");
-               ret = -EINVAL;
+               return -EINVAL;
        }
 
-       if (!kiocb->ki_retry)
-               return ret;
+       if (ret != -EIOCBQUEUED) {
+               /*
+                * There's no easy way to restart the syscall since other AIO's
+                * may be already running. Just fail this IO with EINTR.
+                */
+               if (unlikely(ret == -ERESTARTSYS || ret == -ERESTARTNOINTR ||
+                            ret == -ERESTARTNOHAND ||
+                            ret == -ERESTART_RESTARTBLOCK))
+                       ret = -EINTR;
+               aio_complete(req, ret, 0);
+       }
 
        return 0;
 }
@@ -1138,7 +1094,7 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
                return -EINVAL;
        }
 
-       req = aio_get_req(ctx);  /* returns with 2 references to req */
+       req = aio_get_req(ctx);
        if (unlikely(!req))
                return -EAGAIN;
 
@@ -1163,7 +1119,7 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
                }
        }
 
-       ret = put_user(req->ki_key, &user_iocb->aio_key);
+       ret = put_user(KIOCB_KEY, &user_iocb->aio_key);
        if (unlikely(ret)) {
                pr_debug("EFAULT: aio_key\n");
                goto out_put_req;
@@ -1177,26 +1133,12 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
        req->ki_left = req->ki_nbytes = iocb->aio_nbytes;
        req->ki_opcode = iocb->aio_lio_opcode;
 
-       ret = aio_setup_iocb(req, compat);
+       ret = aio_run_iocb(req, compat);
        if (ret)
                goto out_put_req;
 
-       ret = req->ki_retry(req);
-       if (ret != -EIOCBQUEUED) {
-               /*
-                * There's no easy way to restart the syscall since other AIO's
-                * may be already running. Just fail this IO with EINTR.
-                */
-               if (unlikely(ret == -ERESTARTSYS || ret == -ERESTARTNOINTR ||
-                            ret == -ERESTARTNOHAND ||
-                            ret == -ERESTART_RESTARTBLOCK))
-                       ret = -EINTR;
-               aio_complete(req, ret, 0);
-       }
-
        aio_put_req(req);       /* drop extra ref to req */
        return 0;
-
 out_put_req:
        atomic_dec(&ctx->reqs_active);
        aio_put_req(req);       /* drop extra ref to req */
@@ -1285,10 +1227,13 @@ static struct kiocb *lookup_kiocb(struct kioctx *ctx, struct iocb __user *iocb,
 
        assert_spin_locked(&ctx->ctx_lock);
 
+       if (key != KIOCB_KEY)
+               return NULL;
+
        /* TODO: use a hash or array, this sucks. */
        list_for_each(pos, &ctx->active_reqs) {
                struct kiocb *kiocb = list_kiocb(pos);
-               if (kiocb->ki_obj.user == iocb && kiocb->ki_key == key)
+               if (kiocb->ki_obj.user == iocb)
                        return kiocb;
        }
        return NULL;
@@ -1354,8 +1299,7 @@ SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb,
  *     < min_nr if the timeout specified by timeout has elapsed
  *     before sufficient events are available, where timeout == NULL
  *     specifies an infinite timeout. Note that the timeout pointed to by
- *     timeout is relative and will be updated if not NULL and the
- *     operation blocks. Will fail with -ENOSYS if not implemented.
+ *     timeout is relative.  Will fail with -ENOSYS if not implemented.
  */
 SYSCALL_DEFINE5(io_getevents, aio_context_t, ctx_id,
                long, min_nr,