1 // SPDX-License-Identifier: GPL-2.0-only
3 * "splice": joining two ropes together by interweaving their strands.
5 * This is the "extended pipe" functionality, where a pipe is used as
6 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
7 * buffer that you can use to transfer data from one end to the other.
9 * The traditional unix read/write is extended with a "splice()" operation
10 * that transfers data buffers to or from a pipe buffer.
12 * Named by Larry McVoy, original implementation from Linus, extended by
13 * Jens to support splicing to files, network, direct splicing, etc and
14 * fixing lots of bugs.
16 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
17 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
18 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
21 #include <linux/bvec.h>
23 #include <linux/file.h>
24 #include <linux/pagemap.h>
25 #include <linux/splice.h>
26 #include <linux/memcontrol.h>
27 #include <linux/mm_inline.h>
28 #include <linux/swap.h>
29 #include <linux/writeback.h>
30 #include <linux/export.h>
31 #include <linux/syscalls.h>
32 #include <linux/uio.h>
33 #include <linux/fsnotify.h>
34 #include <linux/security.h>
35 #include <linux/gfp.h>
36 #include <linux/socket.h>
37 #include <linux/sched/signal.h>
42 * Splice doesn't support FMODE_NOWAIT. Since pipes may set this flag to
43 * indicate they support non-blocking reads or writes, we must clear it
44 * here if set to avoid blocking other users of this pipe if splice is
47 static noinline void noinline pipe_clear_nowait(struct file *file)
49 fmode_t fmode = READ_ONCE(file->f_mode);
52 if (!(fmode & FMODE_NOWAIT))
54 } while (!try_cmpxchg(&file->f_mode, &fmode, fmode & ~FMODE_NOWAIT));
58 * Attempt to steal a page from a pipe buffer. This should perhaps go into
59 * a vm helper function, it's already simplified quite a bit by the
60 * addition of remove_mapping(). If success is returned, the caller may
61 * attempt to reuse this page for another destination.
63 static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe,
64 struct pipe_buffer *buf)
66 struct folio *folio = page_folio(buf->page);
67 struct address_space *mapping;
71 mapping = folio_mapping(folio);
73 WARN_ON(!folio_test_uptodate(folio));
76 * At least for ext2 with nobh option, we need to wait on
77 * writeback completing on this folio, since we'll remove it
78 * from the pagecache. Otherwise truncate wont wait on the
79 * folio, allowing the disk blocks to be reused by someone else
80 * before we actually wrote our data to them. fs corruption
83 folio_wait_writeback(folio);
85 if (folio_has_private(folio) &&
86 !filemap_release_folio(folio, GFP_KERNEL))
90 * If we succeeded in removing the mapping, set LRU flag
93 if (remove_mapping(mapping, folio)) {
94 buf->flags |= PIPE_BUF_FLAG_LRU;
100 * Raced with truncate or failed to remove folio from current
101 * address space, unlock and return failure.
108 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
109 struct pipe_buffer *buf)
112 buf->flags &= ~PIPE_BUF_FLAG_LRU;
116 * Check whether the contents of buf is OK to access. Since the content
117 * is a page cache page, IO may be in flight.
119 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
120 struct pipe_buffer *buf)
122 struct page *page = buf->page;
125 if (!PageUptodate(page)) {
129 * Page got truncated/unhashed. This will cause a 0-byte
130 * splice, if this is the first page.
132 if (!page->mapping) {
138 * Uh oh, read-error from disk.
140 if (!PageUptodate(page)) {
146 * Page is ok afterall, we are done.
157 const struct pipe_buf_operations page_cache_pipe_buf_ops = {
158 .confirm = page_cache_pipe_buf_confirm,
159 .release = page_cache_pipe_buf_release,
160 .try_steal = page_cache_pipe_buf_try_steal,
161 .get = generic_pipe_buf_get,
164 static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe,
165 struct pipe_buffer *buf)
167 if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
170 buf->flags |= PIPE_BUF_FLAG_LRU;
171 return generic_pipe_buf_try_steal(pipe, buf);
174 static const struct pipe_buf_operations user_page_pipe_buf_ops = {
175 .release = page_cache_pipe_buf_release,
176 .try_steal = user_page_pipe_buf_try_steal,
177 .get = generic_pipe_buf_get,
180 static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
183 if (waitqueue_active(&pipe->rd_wait))
184 wake_up_interruptible(&pipe->rd_wait);
185 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
189 * splice_to_pipe - fill passed data into a pipe
190 * @pipe: pipe to fill
194 * @spd contains a map of pages and len/offset tuples, along with
195 * the struct pipe_buf_operations associated with these pages. This
196 * function will link that data to the pipe.
199 ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
200 struct splice_pipe_desc *spd)
202 unsigned int spd_pages = spd->nr_pages;
203 unsigned int tail = pipe->tail;
204 unsigned int head = pipe->head;
205 unsigned int mask = pipe->ring_size - 1;
206 int ret = 0, page_nr = 0;
211 if (unlikely(!pipe->readers)) {
212 send_sig(SIGPIPE, current, 0);
217 while (!pipe_full(head, tail, pipe->max_usage)) {
218 struct pipe_buffer *buf = &pipe->bufs[head & mask];
220 buf->page = spd->pages[page_nr];
221 buf->offset = spd->partial[page_nr].offset;
222 buf->len = spd->partial[page_nr].len;
223 buf->private = spd->partial[page_nr].private;
232 if (!--spd->nr_pages)
240 while (page_nr < spd_pages)
241 spd->spd_release(spd, page_nr++);
245 EXPORT_SYMBOL_GPL(splice_to_pipe);
247 ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
249 unsigned int head = pipe->head;
250 unsigned int tail = pipe->tail;
251 unsigned int mask = pipe->ring_size - 1;
254 if (unlikely(!pipe->readers)) {
255 send_sig(SIGPIPE, current, 0);
257 } else if (pipe_full(head, tail, pipe->max_usage)) {
260 pipe->bufs[head & mask] = *buf;
261 pipe->head = head + 1;
264 pipe_buf_release(pipe, buf);
267 EXPORT_SYMBOL(add_to_pipe);
270 * Check if we need to grow the arrays holding pages and partial page
273 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)
275 unsigned int max_usage = READ_ONCE(pipe->max_usage);
277 spd->nr_pages_max = max_usage;
278 if (max_usage <= PIPE_DEF_BUFFERS)
281 spd->pages = kmalloc_array(max_usage, sizeof(struct page *), GFP_KERNEL);
282 spd->partial = kmalloc_array(max_usage, sizeof(struct partial_page),
285 if (spd->pages && spd->partial)
293 void splice_shrink_spd(struct splice_pipe_desc *spd)
295 if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)
303 * Copy data from a file into pages and then splice those into the output pipe.
305 ssize_t copy_splice_read(struct file *in, loff_t *ppos,
306 struct pipe_inode_info *pipe,
307 size_t len, unsigned int flags)
314 size_t used, npages, chunk, remain, keep = 0;
317 /* Work out how much data we can actually add into the pipe */
318 used = pipe_occupancy(pipe->head, pipe->tail);
319 npages = max_t(ssize_t, pipe->max_usage - used, 0);
320 len = min_t(size_t, len, npages * PAGE_SIZE);
321 npages = DIV_ROUND_UP(len, PAGE_SIZE);
323 bv = kzalloc(array_size(npages, sizeof(bv[0])) +
324 array_size(npages, sizeof(struct page *)), GFP_KERNEL);
328 pages = (struct page **)(bv + npages);
329 npages = alloc_pages_bulk_array(GFP_USER, npages, pages);
335 remain = len = min_t(size_t, len, npages * PAGE_SIZE);
337 for (i = 0; i < npages; i++) {
338 chunk = min_t(size_t, PAGE_SIZE, remain);
339 bv[i].bv_page = pages[i];
341 bv[i].bv_len = chunk;
346 iov_iter_bvec(&to, ITER_DEST, bv, npages, len);
347 init_sync_kiocb(&kiocb, in);
348 kiocb.ki_pos = *ppos;
349 ret = call_read_iter(in, &kiocb, &to);
352 keep = DIV_ROUND_UP(ret, PAGE_SIZE);
353 *ppos = kiocb.ki_pos;
355 } else if (ret < 0) {
357 * callers of ->splice_read() expect -EAGAIN on
358 * "can't put anything in there", rather than -EFAULT.
364 /* Free any pages that didn't get touched at all. */
366 release_pages(pages + keep, npages - keep);
368 /* Push the remaining pages into the pipe. */
370 for (i = 0; i < keep; i++) {
371 struct pipe_buffer *buf = pipe_head_buf(pipe);
373 chunk = min_t(size_t, remain, PAGE_SIZE);
374 *buf = (struct pipe_buffer) {
375 .ops = &default_pipe_buf_ops,
376 .page = bv[i].bv_page,
387 EXPORT_SYMBOL(copy_splice_read);
390 * generic_file_splice_read - splice data from file to a pipe
391 * @in: file to splice from
392 * @ppos: position in @in
393 * @pipe: pipe to splice to
394 * @len: number of bytes to splice
395 * @flags: splice modifier flags
398 * Will read pages from given file and fill them into a pipe. Can be
399 * used as long as it has more or less sane ->read_iter().
402 ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
403 struct pipe_inode_info *pipe, size_t len,
410 iov_iter_pipe(&to, ITER_DEST, pipe, len);
411 init_sync_kiocb(&kiocb, in);
412 kiocb.ki_pos = *ppos;
413 ret = call_read_iter(in, &kiocb, &to);
415 *ppos = kiocb.ki_pos;
417 } else if (ret < 0) {
418 /* free what was emitted */
419 pipe_discard_from(pipe, to.start_head);
421 * callers of ->splice_read() expect -EAGAIN on
422 * "can't put anything in there", rather than -EFAULT.
430 EXPORT_SYMBOL(generic_file_splice_read);
432 const struct pipe_buf_operations default_pipe_buf_ops = {
433 .release = generic_pipe_buf_release,
434 .try_steal = generic_pipe_buf_try_steal,
435 .get = generic_pipe_buf_get,
438 /* Pipe buffer operations for a socket and similar. */
439 const struct pipe_buf_operations nosteal_pipe_buf_ops = {
440 .release = generic_pipe_buf_release,
441 .get = generic_pipe_buf_get,
443 EXPORT_SYMBOL(nosteal_pipe_buf_ops);
446 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
447 * using sendpage(). Return the number of bytes sent.
449 static int pipe_to_sendpage(struct pipe_inode_info *pipe,
450 struct pipe_buffer *buf, struct splice_desc *sd)
452 struct file *file = sd->u.file;
453 loff_t pos = sd->pos;
456 if (!likely(file->f_op->sendpage))
459 more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0;
461 if (sd->len < sd->total_len &&
462 pipe_occupancy(pipe->head, pipe->tail) > 1)
463 more |= MSG_SENDPAGE_NOTLAST;
465 return file->f_op->sendpage(file, buf->page, buf->offset,
466 sd->len, &pos, more);
469 static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
472 if (waitqueue_active(&pipe->wr_wait))
473 wake_up_interruptible(&pipe->wr_wait);
474 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
478 * splice_from_pipe_feed - feed available data from a pipe to a file
479 * @pipe: pipe to splice from
480 * @sd: information to @actor
481 * @actor: handler that splices the data
484 * This function loops over the pipe and calls @actor to do the
485 * actual moving of a single struct pipe_buffer to the desired
486 * destination. It returns when there's no more buffers left in
487 * the pipe or if the requested number of bytes (@sd->total_len)
488 * have been copied. It returns a positive number (one) if the
489 * pipe needs to be filled with more data, zero if the required
490 * number of bytes have been copied and -errno on error.
492 * This, together with splice_from_pipe_{begin,end,next}, may be
493 * used to implement the functionality of __splice_from_pipe() when
494 * locking is required around copying the pipe buffers to the
497 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
500 unsigned int head = pipe->head;
501 unsigned int tail = pipe->tail;
502 unsigned int mask = pipe->ring_size - 1;
505 while (!pipe_empty(head, tail)) {
506 struct pipe_buffer *buf = &pipe->bufs[tail & mask];
509 if (sd->len > sd->total_len)
510 sd->len = sd->total_len;
512 ret = pipe_buf_confirm(pipe, buf);
519 ret = actor(pipe, buf, sd);
526 sd->num_spliced += ret;
529 sd->total_len -= ret;
532 pipe_buf_release(pipe, buf);
536 sd->need_wakeup = true;
546 /* We know we have a pipe buffer, but maybe it's empty? */
547 static inline bool eat_empty_buffer(struct pipe_inode_info *pipe)
549 unsigned int tail = pipe->tail;
550 unsigned int mask = pipe->ring_size - 1;
551 struct pipe_buffer *buf = &pipe->bufs[tail & mask];
553 if (unlikely(!buf->len)) {
554 pipe_buf_release(pipe, buf);
563 * splice_from_pipe_next - wait for some data to splice from
564 * @pipe: pipe to splice from
565 * @sd: information about the splice operation
568 * This function will wait for some data and return a positive
569 * value (one) if pipe buffers are available. It will return zero
570 * or -errno if no more data needs to be spliced.
572 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
575 * Check for signal early to make process killable when there are
576 * always buffers available
578 if (signal_pending(current))
582 while (pipe_empty(pipe->head, pipe->tail)) {
589 if (sd->flags & SPLICE_F_NONBLOCK)
592 if (signal_pending(current))
595 if (sd->need_wakeup) {
596 wakeup_pipe_writers(pipe);
597 sd->need_wakeup = false;
600 pipe_wait_readable(pipe);
603 if (eat_empty_buffer(pipe))
610 * splice_from_pipe_begin - start splicing from pipe
611 * @sd: information about the splice operation
614 * This function should be called before a loop containing
615 * splice_from_pipe_next() and splice_from_pipe_feed() to
616 * initialize the necessary fields of @sd.
618 static void splice_from_pipe_begin(struct splice_desc *sd)
621 sd->need_wakeup = false;
625 * splice_from_pipe_end - finish splicing from pipe
626 * @pipe: pipe to splice from
627 * @sd: information about the splice operation
630 * This function will wake up pipe writers if necessary. It should
631 * be called after a loop containing splice_from_pipe_next() and
632 * splice_from_pipe_feed().
634 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
637 wakeup_pipe_writers(pipe);
641 * __splice_from_pipe - splice data from a pipe to given actor
642 * @pipe: pipe to splice from
643 * @sd: information to @actor
644 * @actor: handler that splices the data
647 * This function does little more than loop over the pipe and call
648 * @actor to do the actual moving of a single struct pipe_buffer to
649 * the desired destination. See pipe_to_file, pipe_to_sendpage, or
653 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
658 splice_from_pipe_begin(sd);
661 ret = splice_from_pipe_next(pipe, sd);
663 ret = splice_from_pipe_feed(pipe, sd, actor);
665 splice_from_pipe_end(pipe, sd);
667 return sd->num_spliced ? sd->num_spliced : ret;
669 EXPORT_SYMBOL(__splice_from_pipe);
672 * splice_from_pipe - splice data from a pipe to a file
673 * @pipe: pipe to splice from
674 * @out: file to splice to
675 * @ppos: position in @out
676 * @len: how many bytes to splice
677 * @flags: splice modifier flags
678 * @actor: handler that splices the data
681 * See __splice_from_pipe. This function locks the pipe inode,
682 * otherwise it's identical to __splice_from_pipe().
685 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
686 loff_t *ppos, size_t len, unsigned int flags,
690 struct splice_desc sd = {
698 ret = __splice_from_pipe(pipe, &sd, actor);
705 * iter_file_splice_write - splice data from a pipe to a file
707 * @out: file to write to
708 * @ppos: position in @out
709 * @len: number of bytes to splice
710 * @flags: splice modifier flags
713 * Will either move or copy pages (determined by @flags options) from
714 * the given pipe inode to the given file.
715 * This one is ->write_iter-based.
719 iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
720 loff_t *ppos, size_t len, unsigned int flags)
722 struct splice_desc sd = {
728 int nbufs = pipe->max_usage;
729 struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec),
733 if (unlikely(!array))
738 splice_from_pipe_begin(&sd);
739 while (sd.total_len) {
740 struct iov_iter from;
741 unsigned int head, tail, mask;
745 ret = splice_from_pipe_next(pipe, &sd);
749 if (unlikely(nbufs < pipe->max_usage)) {
751 nbufs = pipe->max_usage;
752 array = kcalloc(nbufs, sizeof(struct bio_vec),
762 mask = pipe->ring_size - 1;
764 /* build the vector */
766 for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++) {
767 struct pipe_buffer *buf = &pipe->bufs[tail & mask];
768 size_t this_len = buf->len;
770 /* zero-length bvecs are not supported, skip them */
773 this_len = min(this_len, left);
775 ret = pipe_buf_confirm(pipe, buf);
782 bvec_set_page(&array[n], buf->page, this_len,
788 iov_iter_bvec(&from, ITER_SOURCE, array, n, sd.total_len - left);
789 ret = vfs_iter_write(out, &from, &sd.pos, 0);
793 sd.num_spliced += ret;
797 /* dismiss the fully eaten buffers, adjust the partial one */
800 struct pipe_buffer *buf = &pipe->bufs[tail & mask];
801 if (ret >= buf->len) {
804 pipe_buf_release(pipe, buf);
808 sd.need_wakeup = true;
818 splice_from_pipe_end(pipe, &sd);
823 ret = sd.num_spliced;
828 EXPORT_SYMBOL(iter_file_splice_write);
831 * generic_splice_sendpage - splice data from a pipe to a socket
832 * @pipe: pipe to splice from
833 * @out: socket to write to
834 * @ppos: position in @out
835 * @len: number of bytes to splice
836 * @flags: splice modifier flags
839 * Will send @len bytes from the pipe to a network socket. No data copying
843 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
844 loff_t *ppos, size_t len, unsigned int flags)
846 return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
849 EXPORT_SYMBOL(generic_splice_sendpage);
851 static int warn_unsupported(struct file *file, const char *op)
853 pr_debug_ratelimited(
854 "splice %s not supported for file %pD4 (pid: %d comm: %.20s)\n",
855 op, file, current->pid, current->comm);
860 * Attempt to initiate a splice from pipe to file.
862 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
863 loff_t *ppos, size_t len, unsigned int flags)
865 if (unlikely(!out->f_op->splice_write))
866 return warn_unsupported(out, "write");
867 return out->f_op->splice_write(pipe, out, ppos, len, flags);
871 * vfs_splice_read - Read data from a file and splice it into a pipe
872 * @in: File to splice from
873 * @ppos: Input file offset
874 * @pipe: Pipe to splice to
875 * @len: Number of bytes to splice
876 * @flags: Splice modifier flags (SPLICE_F_*)
878 * Splice the requested amount of data from the input file to the pipe. This
879 * is synchronous as the caller must hold the pipe lock across the entire
882 * If successful, it returns the amount of data spliced, 0 if it hit the EOF or
883 * a hole and a negative error code otherwise.
885 long vfs_splice_read(struct file *in, loff_t *ppos,
886 struct pipe_inode_info *pipe, size_t len,
889 unsigned int p_space;
892 if (unlikely(!(in->f_mode & FMODE_READ)))
895 /* Don't try to read more the pipe has space for. */
896 p_space = pipe->max_usage - pipe_occupancy(pipe->head, pipe->tail);
897 len = min_t(size_t, len, p_space << PAGE_SHIFT);
899 ret = rw_verify_area(READ, in, ppos, len);
900 if (unlikely(ret < 0))
903 if (unlikely(len > MAX_RW_COUNT))
906 if (unlikely(!in->f_op->splice_read))
907 return warn_unsupported(in, "read");
908 return in->f_op->splice_read(in, ppos, pipe, len, flags);
910 EXPORT_SYMBOL_GPL(vfs_splice_read);
913 * splice_direct_to_actor - splices data directly between two non-pipes
914 * @in: file to splice from
915 * @sd: actor information on where to splice to
916 * @actor: handles the data splicing
919 * This is a special case helper to splice directly between two
920 * points, without requiring an explicit pipe. Internally an allocated
921 * pipe is cached in the process, and reused during the lifetime of
925 ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
926 splice_direct_actor *actor)
928 struct pipe_inode_info *pipe;
934 * We require the input to be seekable, as we don't want to randomly
935 * drop data for eg socket -> socket splicing. Use the piped splicing
938 if (unlikely(!(in->f_mode & FMODE_LSEEK)))
942 * neither in nor out is a pipe, setup an internal pipe attached to
943 * 'out' and transfer the wanted data from 'in' to 'out' through that
945 pipe = current->splice_pipe;
946 if (unlikely(!pipe)) {
947 pipe = alloc_pipe_info();
952 * We don't have an immediate reader, but we'll read the stuff
953 * out of the pipe right after the splice_to_pipe(). So set
954 * PIPE_READERS appropriately.
958 current->splice_pipe = pipe;
969 * Don't block on output, we have to drain the direct pipe.
971 sd->flags &= ~SPLICE_F_NONBLOCK;
972 more = sd->flags & SPLICE_F_MORE;
974 WARN_ON_ONCE(!pipe_empty(pipe->head, pipe->tail));
978 loff_t pos = sd->pos, prev_pos = pos;
980 ret = vfs_splice_read(in, &pos, pipe, len, flags);
981 if (unlikely(ret <= 0))
985 sd->total_len = read_len;
988 * If more data is pending, set SPLICE_F_MORE
989 * If this is the last data and SPLICE_F_MORE was not set
990 * initially, clears it.
993 sd->flags |= SPLICE_F_MORE;
995 sd->flags &= ~SPLICE_F_MORE;
997 * NOTE: nonblocking mode only applies to the input. We
998 * must not do the output in nonblocking mode as then we
999 * could get stuck data in the internal pipe:
1001 ret = actor(pipe, sd);
1002 if (unlikely(ret <= 0)) {
1011 if (ret < read_len) {
1012 sd->pos = prev_pos + ret;
1018 pipe->tail = pipe->head = 0;
1024 * If we did an incomplete transfer we must release
1025 * the pipe buffers in question:
1027 for (i = 0; i < pipe->ring_size; i++) {
1028 struct pipe_buffer *buf = &pipe->bufs[i];
1031 pipe_buf_release(pipe, buf);
1039 EXPORT_SYMBOL(splice_direct_to_actor);
1041 static int direct_splice_actor(struct pipe_inode_info *pipe,
1042 struct splice_desc *sd)
1044 struct file *file = sd->u.file;
1046 return do_splice_from(pipe, file, sd->opos, sd->total_len,
1051 * do_splice_direct - splices data directly between two files
1052 * @in: file to splice from
1053 * @ppos: input file offset
1054 * @out: file to splice to
1055 * @opos: output file offset
1056 * @len: number of bytes to splice
1057 * @flags: splice modifier flags
1060 * For use by do_sendfile(). splice can easily emulate sendfile, but
1061 * doing it in the application would incur an extra system call
1062 * (splice in + splice out, as compared to just sendfile()). So this helper
1063 * can splice directly through a process-private pipe.
1066 long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
1067 loff_t *opos, size_t len, unsigned int flags)
1069 struct splice_desc sd = {
1079 if (unlikely(!(out->f_mode & FMODE_WRITE)))
1082 if (unlikely(out->f_flags & O_APPEND))
1085 ret = rw_verify_area(WRITE, out, opos, len);
1086 if (unlikely(ret < 0))
1089 ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
1095 EXPORT_SYMBOL(do_splice_direct);
1097 static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags)
1100 if (unlikely(!pipe->readers)) {
1101 send_sig(SIGPIPE, current, 0);
1104 if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))
1106 if (flags & SPLICE_F_NONBLOCK)
1108 if (signal_pending(current))
1109 return -ERESTARTSYS;
1110 pipe_wait_writable(pipe);
1114 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1115 struct pipe_inode_info *opipe,
1116 size_t len, unsigned int flags);
1118 long splice_file_to_pipe(struct file *in,
1119 struct pipe_inode_info *opipe,
1121 size_t len, unsigned int flags)
1126 ret = wait_for_space(opipe, flags);
1128 ret = vfs_splice_read(in, offset, opipe, len, flags);
1131 wakeup_pipe_readers(opipe);
1136 * Determine where to splice to/from.
1138 long do_splice(struct file *in, loff_t *off_in, struct file *out,
1139 loff_t *off_out, size_t len, unsigned int flags)
1141 struct pipe_inode_info *ipipe;
1142 struct pipe_inode_info *opipe;
1146 if (unlikely(!(in->f_mode & FMODE_READ) ||
1147 !(out->f_mode & FMODE_WRITE)))
1150 ipipe = get_pipe_info(in, true);
1151 opipe = get_pipe_info(out, true);
1153 if (ipipe && opipe) {
1154 if (off_in || off_out)
1157 /* Splicing to self would be fun, but... */
1161 if ((in->f_flags | out->f_flags) & O_NONBLOCK)
1162 flags |= SPLICE_F_NONBLOCK;
1164 return splice_pipe_to_pipe(ipipe, opipe, len, flags);
1171 if (!(out->f_mode & FMODE_PWRITE))
1175 offset = out->f_pos;
1178 if (unlikely(out->f_flags & O_APPEND))
1181 ret = rw_verify_area(WRITE, out, &offset, len);
1182 if (unlikely(ret < 0))
1185 if (in->f_flags & O_NONBLOCK)
1186 flags |= SPLICE_F_NONBLOCK;
1188 file_start_write(out);
1189 ret = do_splice_from(ipipe, out, &offset, len, flags);
1190 file_end_write(out);
1193 fsnotify_modify(out);
1196 out->f_pos = offset;
1207 if (!(in->f_mode & FMODE_PREAD))
1214 if (out->f_flags & O_NONBLOCK)
1215 flags |= SPLICE_F_NONBLOCK;
1217 ret = splice_file_to_pipe(in, opipe, &offset, len, flags);
1220 fsnotify_access(in);
1233 static long __do_splice(struct file *in, loff_t __user *off_in,
1234 struct file *out, loff_t __user *off_out,
1235 size_t len, unsigned int flags)
1237 struct pipe_inode_info *ipipe;
1238 struct pipe_inode_info *opipe;
1239 loff_t offset, *__off_in = NULL, *__off_out = NULL;
1242 ipipe = get_pipe_info(in, true);
1243 opipe = get_pipe_info(out, true);
1248 pipe_clear_nowait(in);
1253 pipe_clear_nowait(out);
1257 if (copy_from_user(&offset, off_out, sizeof(loff_t)))
1259 __off_out = &offset;
1262 if (copy_from_user(&offset, off_in, sizeof(loff_t)))
1267 ret = do_splice(in, __off_in, out, __off_out, len, flags);
1271 if (__off_out && copy_to_user(off_out, __off_out, sizeof(loff_t)))
1273 if (__off_in && copy_to_user(off_in, __off_in, sizeof(loff_t)))
1279 static int iter_to_pipe(struct iov_iter *from,
1280 struct pipe_inode_info *pipe,
1283 struct pipe_buffer buf = {
1284 .ops = &user_page_pipe_buf_ops,
1290 while (iov_iter_count(from)) {
1291 struct page *pages[16];
1296 left = iov_iter_get_pages2(from, pages, ~0UL, 16, &start);
1302 n = DIV_ROUND_UP(left + start, PAGE_SIZE);
1303 for (i = 0; i < n; i++) {
1304 int size = min_t(int, left, PAGE_SIZE - start);
1306 buf.page = pages[i];
1309 ret = add_to_pipe(pipe, &buf);
1310 if (unlikely(ret < 0)) {
1311 iov_iter_revert(from, left);
1312 // this one got dropped by add_to_pipe()
1323 return total ? total : ret;
1326 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
1327 struct splice_desc *sd)
1329 int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data);
1330 return n == sd->len ? n : -EFAULT;
1334 * For lack of a better implementation, implement vmsplice() to userspace
1335 * as a simple copy of the pipes pages to the user iov.
1337 static long vmsplice_to_user(struct file *file, struct iov_iter *iter,
1340 struct pipe_inode_info *pipe = get_pipe_info(file, true);
1341 struct splice_desc sd = {
1342 .total_len = iov_iter_count(iter),
1351 pipe_clear_nowait(file);
1355 ret = __splice_from_pipe(pipe, &sd, pipe_to_user);
1363 * vmsplice splices a user address range into a pipe. It can be thought of
1364 * as splice-from-memory, where the regular splice is splice-from-file (or
1365 * to file). In both cases the output is a pipe, naturally.
1367 static long vmsplice_to_pipe(struct file *file, struct iov_iter *iter,
1370 struct pipe_inode_info *pipe;
1372 unsigned buf_flag = 0;
1374 if (flags & SPLICE_F_GIFT)
1375 buf_flag = PIPE_BUF_FLAG_GIFT;
1377 pipe = get_pipe_info(file, true);
1381 pipe_clear_nowait(file);
1384 ret = wait_for_space(pipe, flags);
1386 ret = iter_to_pipe(iter, pipe, buf_flag);
1389 wakeup_pipe_readers(pipe);
1393 static int vmsplice_type(struct fd f, int *type)
1397 if (f.file->f_mode & FMODE_WRITE) {
1398 *type = ITER_SOURCE;
1399 } else if (f.file->f_mode & FMODE_READ) {
1409 * Note that vmsplice only really supports true splicing _from_ user memory
1410 * to a pipe, not the other way around. Splicing from user memory is a simple
1411 * operation that can be supported without any funky alignment restrictions
1412 * or nasty vm tricks. We simply map in the user memory and fill them into
1413 * a pipe. The reverse isn't quite as easy, though. There are two possible
1414 * solutions for that:
1416 * - memcpy() the data internally, at which point we might as well just
1417 * do a regular read() on the buffer anyway.
1418 * - Lots of nasty vm tricks, that are neither fast nor flexible (it
1419 * has restriction limitations on both ends of the pipe).
1421 * Currently we punt and implement it as a normal copy, see pipe_to_user().
1424 SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, uiov,
1425 unsigned long, nr_segs, unsigned int, flags)
1427 struct iovec iovstack[UIO_FASTIOV];
1428 struct iovec *iov = iovstack;
1429 struct iov_iter iter;
1434 if (unlikely(flags & ~SPLICE_F_ALL))
1438 error = vmsplice_type(f, &type);
1442 error = import_iovec(type, uiov, nr_segs,
1443 ARRAY_SIZE(iovstack), &iov, &iter);
1447 if (!iov_iter_count(&iter))
1449 else if (type == ITER_SOURCE)
1450 error = vmsplice_to_pipe(f.file, &iter, flags);
1452 error = vmsplice_to_user(f.file, &iter, flags);
1460 SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
1461 int, fd_out, loff_t __user *, off_out,
1462 size_t, len, unsigned int, flags)
1470 if (unlikely(flags & ~SPLICE_F_ALL))
1476 out = fdget(fd_out);
1478 error = __do_splice(in.file, off_in, out.file, off_out,
1488 * Make sure there's data to read. Wait for input if we can, otherwise
1489 * return an appropriate error.
1491 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1496 * Check the pipe occupancy without the inode lock first. This function
1497 * is speculative anyways, so missing one is ok.
1499 if (!pipe_empty(pipe->head, pipe->tail))
1505 while (pipe_empty(pipe->head, pipe->tail)) {
1506 if (signal_pending(current)) {
1512 if (flags & SPLICE_F_NONBLOCK) {
1516 pipe_wait_readable(pipe);
1524 * Make sure there's writeable room. Wait for room if we can, otherwise
1525 * return an appropriate error.
1527 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1532 * Check pipe occupancy without the inode lock first. This function
1533 * is speculative anyways, so missing one is ok.
1535 if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))
1541 while (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) {
1542 if (!pipe->readers) {
1543 send_sig(SIGPIPE, current, 0);
1547 if (flags & SPLICE_F_NONBLOCK) {
1551 if (signal_pending(current)) {
1555 pipe_wait_writable(pipe);
1563 * Splice contents of ipipe to opipe.
1565 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1566 struct pipe_inode_info *opipe,
1567 size_t len, unsigned int flags)
1569 struct pipe_buffer *ibuf, *obuf;
1570 unsigned int i_head, o_head;
1571 unsigned int i_tail, o_tail;
1572 unsigned int i_mask, o_mask;
1574 bool input_wakeup = false;
1578 ret = ipipe_prep(ipipe, flags);
1582 ret = opipe_prep(opipe, flags);
1587 * Potential ABBA deadlock, work around it by ordering lock
1588 * grabbing by pipe info address. Otherwise two different processes
1589 * could deadlock (one doing tee from A -> B, the other from B -> A).
1591 pipe_double_lock(ipipe, opipe);
1593 i_tail = ipipe->tail;
1594 i_mask = ipipe->ring_size - 1;
1595 o_head = opipe->head;
1596 o_mask = opipe->ring_size - 1;
1601 if (!opipe->readers) {
1602 send_sig(SIGPIPE, current, 0);
1608 i_head = ipipe->head;
1609 o_tail = opipe->tail;
1611 if (pipe_empty(i_head, i_tail) && !ipipe->writers)
1615 * Cannot make any progress, because either the input
1616 * pipe is empty or the output pipe is full.
1618 if (pipe_empty(i_head, i_tail) ||
1619 pipe_full(o_head, o_tail, opipe->max_usage)) {
1620 /* Already processed some buffers, break */
1624 if (flags & SPLICE_F_NONBLOCK) {
1630 * We raced with another reader/writer and haven't
1631 * managed to process any buffers. A zero return
1632 * value means EOF, so retry instead.
1639 ibuf = &ipipe->bufs[i_tail & i_mask];
1640 obuf = &opipe->bufs[o_head & o_mask];
1642 if (len >= ibuf->len) {
1644 * Simply move the whole buffer from ipipe to opipe
1649 ipipe->tail = i_tail;
1650 input_wakeup = true;
1653 opipe->head = o_head;
1656 * Get a reference to this pipe buffer,
1657 * so we can copy the contents over.
1659 if (!pipe_buf_get(ipipe, ibuf)) {
1667 * Don't inherit the gift and merge flags, we need to
1668 * prevent multiple steals of this page.
1670 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1671 obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
1674 ibuf->offset += len;
1678 opipe->head = o_head;
1688 * If we put data in the output pipe, wakeup any potential readers.
1691 wakeup_pipe_readers(opipe);
1694 wakeup_pipe_writers(ipipe);
1700 * Link contents of ipipe to opipe.
1702 static int link_pipe(struct pipe_inode_info *ipipe,
1703 struct pipe_inode_info *opipe,
1704 size_t len, unsigned int flags)
1706 struct pipe_buffer *ibuf, *obuf;
1707 unsigned int i_head, o_head;
1708 unsigned int i_tail, o_tail;
1709 unsigned int i_mask, o_mask;
1713 * Potential ABBA deadlock, work around it by ordering lock
1714 * grabbing by pipe info address. Otherwise two different processes
1715 * could deadlock (one doing tee from A -> B, the other from B -> A).
1717 pipe_double_lock(ipipe, opipe);
1719 i_tail = ipipe->tail;
1720 i_mask = ipipe->ring_size - 1;
1721 o_head = opipe->head;
1722 o_mask = opipe->ring_size - 1;
1725 if (!opipe->readers) {
1726 send_sig(SIGPIPE, current, 0);
1732 i_head = ipipe->head;
1733 o_tail = opipe->tail;
1736 * If we have iterated all input buffers or run out of
1737 * output room, break.
1739 if (pipe_empty(i_head, i_tail) ||
1740 pipe_full(o_head, o_tail, opipe->max_usage))
1743 ibuf = &ipipe->bufs[i_tail & i_mask];
1744 obuf = &opipe->bufs[o_head & o_mask];
1747 * Get a reference to this pipe buffer,
1748 * so we can copy the contents over.
1750 if (!pipe_buf_get(ipipe, ibuf)) {
1759 * Don't inherit the gift and merge flag, we need to prevent
1760 * multiple steals of this page.
1762 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1763 obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
1765 if (obuf->len > len)
1771 opipe->head = o_head;
1779 * If we put data in the output pipe, wakeup any potential readers.
1782 wakeup_pipe_readers(opipe);
1788 * This is a tee(1) implementation that works on pipes. It doesn't copy
1789 * any data, it simply references the 'in' pages on the 'out' pipe.
1790 * The 'flags' used are the SPLICE_F_* variants, currently the only
1791 * applicable one is SPLICE_F_NONBLOCK.
1793 long do_tee(struct file *in, struct file *out, size_t len, unsigned int flags)
1795 struct pipe_inode_info *ipipe = get_pipe_info(in, true);
1796 struct pipe_inode_info *opipe = get_pipe_info(out, true);
1799 if (unlikely(!(in->f_mode & FMODE_READ) ||
1800 !(out->f_mode & FMODE_WRITE)))
1804 * Duplicate the contents of ipipe to opipe without actually
1807 if (ipipe && opipe && ipipe != opipe) {
1808 if ((in->f_flags | out->f_flags) & O_NONBLOCK)
1809 flags |= SPLICE_F_NONBLOCK;
1812 * Keep going, unless we encounter an error. The ipipe/opipe
1813 * ordering doesn't really matter.
1815 ret = ipipe_prep(ipipe, flags);
1817 ret = opipe_prep(opipe, flags);
1819 ret = link_pipe(ipipe, opipe, len, flags);
1826 SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
1831 if (unlikely(flags & ~SPLICE_F_ALL))
1842 error = do_tee(in.file, out.file, len, flags);