io_uring: combine REQ_F_NOWAIT_{READ,WRITE} flags

Merge REQ_F_NOWAIT_READ and REQ_F_NOWAIT_WRITE into one flag, i.e.
REQ_F_SUPPORT_NOWAIT. First it gets rid of dependence on CONFIG_64BIT
but also simplifies the code.

One thing to consider is when we don't have ->{read,write}_iter and go
through loop_rw_iter(). Just fail it with -EAGAIN if we expect nowait
behaviour but not sure whether it supports it.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/f832a20e5186c2e79c6519280c238f559a1d2bbc.1634425438.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 8a5b20eafccf27e35254606940f4bcfe177bebb9..d7bf2002ab0ce01bed1468e9b4bdcd7e06540f3d 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -732,8 +732,7 @@ enum {
        REQ_F_ARM_LTIMEOUT_BIT,
        REQ_F_ASYNC_DATA_BIT,
        /* keep async read/write and isreg together and in order */
-       REQ_F_NOWAIT_READ_BIT,
-       REQ_F_NOWAIT_WRITE_BIT,
+       REQ_F_SUPPORT_NOWAIT_BIT,
        REQ_F_ISREG_BIT,
 
        /* not a real bit, just to check we're not overflowing the space */
@@ -774,10 +773,8 @@ enum {
        REQ_F_COMPLETE_INLINE   = BIT(REQ_F_COMPLETE_INLINE_BIT),
        /* caller should reissue async */
        REQ_F_REISSUE           = BIT(REQ_F_REISSUE_BIT),
-       /* supports async reads */
-       REQ_F_NOWAIT_READ       = BIT(REQ_F_NOWAIT_READ_BIT),
-       /* supports async writes */
-       REQ_F_NOWAIT_WRITE      = BIT(REQ_F_NOWAIT_WRITE_BIT),
+       /* supports async reads/writes */
+       REQ_F_SUPPORT_NOWAIT    = BIT(REQ_F_SUPPORT_NOWAIT_BIT),
        /* regular file */
        REQ_F_ISREG             = BIT(REQ_F_ISREG_BIT),
        /* has creds assigned */
@@ -1390,18 +1387,13 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq)
        return false;
 }
 
-#define FFS_ASYNC_READ         0x1UL
-#define FFS_ASYNC_WRITE                0x2UL
-#ifdef CONFIG_64BIT
-#define FFS_ISREG              0x4UL
-#else
-#define FFS_ISREG              0x0UL
-#endif
-#define FFS_MASK               ~(FFS_ASYNC_READ|FFS_ASYNC_WRITE|FFS_ISREG)
+#define FFS_NOWAIT             0x1UL
+#define FFS_ISREG              0x2UL
+#define FFS_MASK               ~(FFS_NOWAIT|FFS_ISREG)
 
 static inline bool io_req_ffs_set(struct io_kiocb *req)
 {
-       return IS_ENABLED(CONFIG_64BIT) && (req->flags & REQ_F_FIXED_FILE);
+       return req->flags & REQ_F_FIXED_FILE;
 }
 
 static inline void io_req_track_inflight(struct io_kiocb *req)
@@ -2772,7 +2764,7 @@ static bool io_bdev_nowait(struct block_device *bdev)
  * any file. For now, just ensure that anything potentially problematic is done
  * inline.
  */
-static bool __io_file_supports_nowait(struct file *file, int rw)
+static bool __io_file_supports_nowait(struct file *file)
 {
        umode_t mode = file_inode(file)->i_mode;
 
@@ -2795,24 +2787,14 @@ static bool __io_file_supports_nowait(struct file *file, int rw)
        /* any ->read/write should understand O_NONBLOCK */
        if (file->f_flags & O_NONBLOCK)
                return true;
-
-       if (!(file->f_mode & FMODE_NOWAIT))
-               return false;
-
-       if (rw == READ)
-               return file->f_op->read_iter != NULL;
-
-       return file->f_op->write_iter != NULL;
+       return file->f_mode & FMODE_NOWAIT;
 }
 
-static bool io_file_supports_nowait(struct io_kiocb *req, int rw)
+static inline bool io_file_supports_nowait(struct io_kiocb *req)
 {
-       if (rw == READ && (req->flags & REQ_F_NOWAIT_READ))
+       if (likely(req->flags & REQ_F_SUPPORT_NOWAIT))
                return true;
-       else if (rw == WRITE && (req->flags & REQ_F_NOWAIT_WRITE))
-               return true;
-
-       return __io_file_supports_nowait(req->file, rw);
+       return __io_file_supports_nowait(req->file);
 }
 
 static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
@@ -2844,7 +2826,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
         * reliably. If not, or it IOCB_NOWAIT is set, don't retry.
         */
        if ((kiocb->ki_flags & IOCB_NOWAIT) ||
-           ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req, rw)))
+           ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req)))
                req->flags |= REQ_F_NOWAIT;
 
        ioprio = READ_ONCE(sqe->ioprio);
@@ -3235,7 +3217,8 @@ static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter)
         */
        if (kiocb->ki_flags & IOCB_HIPRI)
                return -EOPNOTSUPP;
-       if (kiocb->ki_flags & IOCB_NOWAIT)
+       if ((kiocb->ki_flags & IOCB_NOWAIT) &&
+           !(kiocb->ki_filp->f_flags & O_NONBLOCK))
                return -EAGAIN;
 
        while (iov_iter_count(iter)) {
@@ -3475,7 +3458,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags)
 
        if (force_nonblock) {
                /* If the file doesn't support async, just async punt */
-               if (unlikely(!io_file_supports_nowait(req, READ))) {
+               if (unlikely(!io_file_supports_nowait(req))) {
                        ret = io_setup_async_rw(req, iovec, s, true);
                        return ret ?: -EAGAIN;
                }
@@ -3599,7 +3582,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags)
 
        if (force_nonblock) {
                /* If the file doesn't support async, just async punt */
-               if (unlikely(!io_file_supports_nowait(req, WRITE)))
+               if (unlikely(!io_file_supports_nowait(req)))
                        goto copy_iov;
 
                /* file path doesn't support NOWAIT for non-direct_IO */
@@ -3631,7 +3614,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags)
        }
        kiocb->ki_flags |= IOCB_WRITE;
 
-       if (req->file->f_op->write_iter)
+       if (likely(req->file->f_op->write_iter))
                ret2 = call_write_iter(req->file, kiocb, &s->iter);
        else if (req->file->f_op->write)
                ret2 = loop_rw_iter(WRITE, req, &s->iter);
@@ -6778,10 +6761,8 @@ static void io_fixed_file_set(struct io_fixed_file *file_slot, struct file *file
 {
        unsigned long file_ptr = (unsigned long) file;
 
-       if (__io_file_supports_nowait(file, READ))
-               file_ptr |= FFS_ASYNC_READ;
-       if (__io_file_supports_nowait(file, WRITE))
-               file_ptr |= FFS_ASYNC_WRITE;
+       if (__io_file_supports_nowait(file))
+               file_ptr |= FFS_NOWAIT;
        if (S_ISREG(file_inode(file)->i_mode))
                file_ptr |= FFS_ISREG;
        file_slot->file_ptr = file_ptr;
@@ -6800,7 +6781,7 @@ static inline struct file *io_file_get_fixed(struct io_ring_ctx *ctx,
        file = (struct file *) (file_ptr & FFS_MASK);
        file_ptr &= ~FFS_MASK;
        /* mask in overlapping REQ_F and FFS bits */
-       req->flags |= (file_ptr << REQ_F_NOWAIT_READ_BIT);
+       req->flags |= (file_ptr << REQ_F_SUPPORT_NOWAIT_BIT);
        io_req_set_rsrc_node(req, ctx);
        return file;
 }