splice: Make splice from a DAX file use copy_splice_read()

[linux-block.git] / fs / splice.c

diff --git a/fs/splice.c b/fs/splice.c
index 3e06611d19ae5546acb6dffa24c9afaaa786cc56..8268248df3a908d7e617c945904f0e15a9a05acc 100644 (file)
--- a/fs/splice.c
+++ b/fs/splice.c
@@ -300,19 +300,18 @@ void splice_shrink_spd(struct splice_pipe_desc *spd)
 }
 
 /*
- * Splice data from an O_DIRECT file into pages and then add them to the output
- * pipe.
+ * Copy data from a file into pages and then splice those into the output pipe.
  */
-ssize_t direct_splice_read(struct file *in, loff_t *ppos,
-                          struct pipe_inode_info *pipe,
-                          size_t len, unsigned int flags)
+ssize_t copy_splice_read(struct file *in, loff_t *ppos,
+                        struct pipe_inode_info *pipe,
+                        size_t len, unsigned int flags)
 {
        struct iov_iter to;
        struct bio_vec *bv;
        struct kiocb kiocb;
        struct page **pages;
        ssize_t ret;
-       size_t used, npages, chunk, remain, reclaim;
+       size_t used, npages, chunk, remain, keep = 0;
        int i;
 
        /* Work out how much data we can actually add into the pipe */
@@ -326,7 +325,7 @@ ssize_t direct_splice_read(struct file *in, loff_t *ppos,
        if (!bv)
                return -ENOMEM;
 
-       pages = (void *)(bv + npages);
+       pages = (struct page **)(bv + npages);
        npages = alloc_pages_bulk_array(GFP_USER, npages, pages);
        if (!npages) {
                kfree(bv);
@@ -349,11 +348,8 @@ ssize_t direct_splice_read(struct file *in, loff_t *ppos,
        kiocb.ki_pos = *ppos;
        ret = call_read_iter(in, &kiocb, &to);
 
-       reclaim = npages * PAGE_SIZE;
-       remain = 0;
        if (ret > 0) {
-               reclaim -= ret;
-               remain = ret;
+               keep = DIV_ROUND_UP(ret, PAGE_SIZE);
                *ppos = kiocb.ki_pos;
                file_accessed(in);
        } else if (ret < 0) {
@@ -366,14 +362,12 @@ ssize_t direct_splice_read(struct file *in, loff_t *ppos,
        }
 
        /* Free any pages that didn't get touched at all. */
-       reclaim /= PAGE_SIZE;
-       if (reclaim) {
-               npages -= reclaim;
-               release_pages(pages + npages, reclaim);
-       }
+       if (keep < npages)
+               release_pages(pages + keep, npages - keep);
 
        /* Push the remaining pages into the pipe. */
-       for (i = 0; i < npages; i++) {
+       remain = ret;
+       for (i = 0; i < keep; i++) {
                struct pipe_buffer *buf = pipe_head_buf(pipe);
 
                chunk = min_t(size_t, remain, PAGE_SIZE);
@@ -390,7 +384,7 @@ ssize_t direct_splice_read(struct file *in, loff_t *ppos,
        kfree(bv);
        return ret;
 }
-EXPORT_SYMBOL(direct_splice_read);
+EXPORT_SYMBOL(copy_splice_read);
 
 /**
  * generic_file_splice_read - splice data from file to a pipe
@@ -873,18 +867,32 @@ static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
        return out->f_op->splice_write(pipe, out, ppos, len, flags);
 }
 
-/*
- * Attempt to initiate a splice from a file to a pipe.
+/**
+ * vfs_splice_read - Read data from a file and splice it into a pipe
+ * @in:                File to splice from
+ * @ppos:      Input file offset
+ * @pipe:      Pipe to splice to
+ * @len:       Number of bytes to splice
+ * @flags:     Splice modifier flags (SPLICE_F_*)
+ *
+ * Splice the requested amount of data from the input file to the pipe.  This
+ * is synchronous as the caller must hold the pipe lock across the entire
+ * operation.
+ *
+ * If successful, it returns the amount of data spliced, 0 if it hit the EOF or
+ * a hole and a negative error code otherwise.
  */
-static long do_splice_to(struct file *in, loff_t *ppos,
-                        struct pipe_inode_info *pipe, size_t len,
-                        unsigned int flags)
+long vfs_splice_read(struct file *in, loff_t *ppos,
+                    struct pipe_inode_info *pipe, size_t len,
+                    unsigned int flags)
 {
        unsigned int p_space;
        int ret;
 
        if (unlikely(!(in->f_mode & FMODE_READ)))
                return -EBADF;
+       if (!len)
+               return 0;
 
        /* Don't try to read more the pipe has space for. */
        p_space = pipe->max_usage - pipe_occupancy(pipe->head, pipe->tail);
@@ -899,8 +907,15 @@ static long do_splice_to(struct file *in, loff_t *ppos,
 
        if (unlikely(!in->f_op->splice_read))
                return warn_unsupported(in, "read");
+       /*
+        * O_DIRECT and DAX don't deal with the pagecache, so we allocate a
+        * buffer, copy into it and splice that into the pipe.
+        */
+       if ((in->f_flags & O_DIRECT) || IS_DAX(in->f_mapping->host))
+               return copy_splice_read(in, ppos, pipe, len, flags);
        return in->f_op->splice_read(in, ppos, pipe, len, flags);
 }
+EXPORT_SYMBOL_GPL(vfs_splice_read);
 
 /**
  * splice_direct_to_actor - splices data directly between two non-pipes
@@ -970,7 +985,7 @@ ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
                size_t read_len;
                loff_t pos = sd->pos, prev_pos = pos;
 
-               ret = do_splice_to(in, &pos, pipe, len, flags);
+               ret = vfs_splice_read(in, &pos, pipe, len, flags);
                if (unlikely(ret <= 0))
                        goto out_release;
 
@@ -1118,7 +1133,7 @@ long splice_file_to_pipe(struct file *in,
        pipe_lock(opipe);
        ret = wait_for_space(opipe, flags);
        if (!ret)
-               ret = do_splice_to(in, offset, opipe, len, flags);
+               ret = vfs_splice_read(in, offset, opipe, len, flags);
        pipe_unlock(opipe);
        if (ret > 0)
                wakeup_pipe_readers(opipe);