Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
5274f052 JA |
2 | /* |
3 | * "splice": joining two ropes together by interweaving their strands. | |
4 | * | |
5 | * This is the "extended pipe" functionality, where a pipe is used as | |
6 | * an arbitrary in-memory buffer. Think of a pipe as a small kernel | |
7 | * buffer that you can use to transfer data from one end to the other. | |
8 | * | |
9 | * The traditional unix read/write is extended with a "splice()" operation | |
10 | * that transfers data buffers to or from a pipe buffer. | |
11 | * | |
12 | * Named by Larry McVoy, original implementation from Linus, extended by | |
c2058e06 JA |
13 | * Jens to support splicing to files, network, direct splicing, etc and |
14 | * fixing lots of bugs. | |
5274f052 | 15 | * |
0fe23479 | 16 | * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk> |
c2058e06 JA |
17 | * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org> |
18 | * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu> | |
5274f052 JA |
19 | * |
20 | */ | |
be297968 | 21 | #include <linux/bvec.h> |
5274f052 JA |
22 | #include <linux/fs.h> |
23 | #include <linux/file.h> | |
24 | #include <linux/pagemap.h> | |
d6b29d7c | 25 | #include <linux/splice.h> |
08e552c6 | 26 | #include <linux/memcontrol.h> |
5274f052 | 27 | #include <linux/mm_inline.h> |
5abc97aa | 28 | #include <linux/swap.h> |
4f6f0bd2 | 29 | #include <linux/writeback.h> |
630d9c47 | 30 | #include <linux/export.h> |
4f6f0bd2 | 31 | #include <linux/syscalls.h> |
912d35f8 | 32 | #include <linux/uio.h> |
983652c6 | 33 | #include <linux/fsnotify.h> |
29ce2058 | 34 | #include <linux/security.h> |
5a0e3ad6 | 35 | #include <linux/gfp.h> |
35f9c09f | 36 | #include <linux/socket.h> |
174cd4b1 IM |
37 | #include <linux/sched/signal.h> |
38 | ||
06ae43f3 | 39 | #include "internal.h" |
5274f052 | 40 | |
0f99fc51 JA |
41 | /* |
42 | * Splice doesn't support FMODE_NOWAIT. Since pipes may set this flag to | |
43 | * indicate they support non-blocking reads or writes, we must clear it | |
44 | * here if set to avoid blocking other users of this pipe if splice is | |
45 | * being done on it. | |
46 | */ | |
47 | static noinline void noinline pipe_clear_nowait(struct file *file) | |
48 | { | |
49 | fmode_t fmode = READ_ONCE(file->f_mode); | |
50 | ||
51 | do { | |
52 | if (!(fmode & FMODE_NOWAIT)) | |
53 | break; | |
54 | } while (!try_cmpxchg(&file->f_mode, &fmode, fmode & ~FMODE_NOWAIT)); | |
55 | } | |
56 | ||
83f9135b JA |
57 | /* |
58 | * Attempt to steal a page from a pipe buffer. This should perhaps go into | |
59 | * a vm helper function, it's already simplified quite a bit by the | |
60 | * addition of remove_mapping(). If success is returned, the caller may | |
61 | * attempt to reuse this page for another destination. | |
62 | */ | |
c928f642 CH |
63 | static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe, |
64 | struct pipe_buffer *buf) | |
5abc97aa | 65 | { |
5100da38 | 66 | struct folio *folio = page_folio(buf->page); |
9e94cd4f | 67 | struct address_space *mapping; |
5abc97aa | 68 | |
b9ccad2e | 69 | folio_lock(folio); |
9e0267c2 | 70 | |
b9ccad2e | 71 | mapping = folio_mapping(folio); |
9e94cd4f | 72 | if (mapping) { |
b9ccad2e | 73 | WARN_ON(!folio_test_uptodate(folio)); |
5abc97aa | 74 | |
9e94cd4f JA |
75 | /* |
76 | * At least for ext2 with nobh option, we need to wait on | |
b9ccad2e | 77 | * writeback completing on this folio, since we'll remove it |
9e94cd4f | 78 | * from the pagecache. Otherwise truncate wont wait on the |
b9ccad2e | 79 | * folio, allowing the disk blocks to be reused by someone else |
9e94cd4f JA |
80 | * before we actually wrote our data to them. fs corruption |
81 | * ensues. | |
82 | */ | |
b9ccad2e | 83 | folio_wait_writeback(folio); |
ad8d6f0a | 84 | |
b9ccad2e MWO |
85 | if (folio_has_private(folio) && |
86 | !filemap_release_folio(folio, GFP_KERNEL)) | |
ca39d651 | 87 | goto out_unlock; |
4f6f0bd2 | 88 | |
9e94cd4f JA |
89 | /* |
90 | * If we succeeded in removing the mapping, set LRU flag | |
91 | * and return good. | |
92 | */ | |
5100da38 | 93 | if (remove_mapping(mapping, folio)) { |
9e94cd4f | 94 | buf->flags |= PIPE_BUF_FLAG_LRU; |
c928f642 | 95 | return true; |
9e94cd4f | 96 | } |
9e0267c2 | 97 | } |
5abc97aa | 98 | |
9e94cd4f | 99 | /* |
b9ccad2e | 100 | * Raced with truncate or failed to remove folio from current |
9e94cd4f JA |
101 | * address space, unlock and return failure. |
102 | */ | |
ca39d651 | 103 | out_unlock: |
b9ccad2e | 104 | folio_unlock(folio); |
c928f642 | 105 | return false; |
5abc97aa JA |
106 | } |
107 | ||
76ad4d11 | 108 | static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe, |
5274f052 JA |
109 | struct pipe_buffer *buf) |
110 | { | |
09cbfeaf | 111 | put_page(buf->page); |
1432873a | 112 | buf->flags &= ~PIPE_BUF_FLAG_LRU; |
5274f052 JA |
113 | } |
114 | ||
0845718d JA |
115 | /* |
116 | * Check whether the contents of buf is OK to access. Since the content | |
117 | * is a page cache page, IO may be in flight. | |
118 | */ | |
cac36bb0 JA |
119 | static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe, |
120 | struct pipe_buffer *buf) | |
5274f052 JA |
121 | { |
122 | struct page *page = buf->page; | |
49d0b21b | 123 | int err; |
5274f052 JA |
124 | |
125 | if (!PageUptodate(page)) { | |
49d0b21b JA |
126 | lock_page(page); |
127 | ||
128 | /* | |
129 | * Page got truncated/unhashed. This will cause a 0-byte | |
73d62d83 | 130 | * splice, if this is the first page. |
49d0b21b JA |
131 | */ |
132 | if (!page->mapping) { | |
133 | err = -ENODATA; | |
134 | goto error; | |
135 | } | |
5274f052 | 136 | |
49d0b21b | 137 | /* |
73d62d83 | 138 | * Uh oh, read-error from disk. |
49d0b21b JA |
139 | */ |
140 | if (!PageUptodate(page)) { | |
141 | err = -EIO; | |
142 | goto error; | |
143 | } | |
144 | ||
145 | /* | |
f84d7519 | 146 | * Page is ok afterall, we are done. |
49d0b21b | 147 | */ |
5274f052 | 148 | unlock_page(page); |
5274f052 JA |
149 | } |
150 | ||
f84d7519 | 151 | return 0; |
49d0b21b JA |
152 | error: |
153 | unlock_page(page); | |
f84d7519 | 154 | return err; |
70524490 JA |
155 | } |
156 | ||
708e3508 | 157 | const struct pipe_buf_operations page_cache_pipe_buf_ops = { |
c928f642 CH |
158 | .confirm = page_cache_pipe_buf_confirm, |
159 | .release = page_cache_pipe_buf_release, | |
160 | .try_steal = page_cache_pipe_buf_try_steal, | |
161 | .get = generic_pipe_buf_get, | |
5274f052 JA |
162 | }; |
163 | ||
c928f642 CH |
164 | static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe, |
165 | struct pipe_buffer *buf) | |
912d35f8 | 166 | { |
7afa6fd0 | 167 | if (!(buf->flags & PIPE_BUF_FLAG_GIFT)) |
c928f642 | 168 | return false; |
7afa6fd0 | 169 | |
1432873a | 170 | buf->flags |= PIPE_BUF_FLAG_LRU; |
c928f642 | 171 | return generic_pipe_buf_try_steal(pipe, buf); |
912d35f8 JA |
172 | } |
173 | ||
d4c3cca9 | 174 | static const struct pipe_buf_operations user_page_pipe_buf_ops = { |
c928f642 CH |
175 | .release = page_cache_pipe_buf_release, |
176 | .try_steal = user_page_pipe_buf_try_steal, | |
177 | .get = generic_pipe_buf_get, | |
912d35f8 JA |
178 | }; |
179 | ||
825cdcb1 NK |
180 | static void wakeup_pipe_readers(struct pipe_inode_info *pipe) |
181 | { | |
182 | smp_mb(); | |
0ddad21d LT |
183 | if (waitqueue_active(&pipe->rd_wait)) |
184 | wake_up_interruptible(&pipe->rd_wait); | |
825cdcb1 NK |
185 | kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); |
186 | } | |
187 | ||
932cc6d4 JA |
188 | /** |
189 | * splice_to_pipe - fill passed data into a pipe | |
190 | * @pipe: pipe to fill | |
191 | * @spd: data to fill | |
192 | * | |
193 | * Description: | |
79685b8d | 194 | * @spd contains a map of pages and len/offset tuples, along with |
932cc6d4 JA |
195 | * the struct pipe_buf_operations associated with these pages. This |
196 | * function will link that data to the pipe. | |
197 | * | |
83f9135b | 198 | */ |
d6b29d7c JA |
199 | ssize_t splice_to_pipe(struct pipe_inode_info *pipe, |
200 | struct splice_pipe_desc *spd) | |
5274f052 | 201 | { |
00de00bd | 202 | unsigned int spd_pages = spd->nr_pages; |
8cefc107 DH |
203 | unsigned int tail = pipe->tail; |
204 | unsigned int head = pipe->head; | |
205 | unsigned int mask = pipe->ring_size - 1; | |
8924feff | 206 | int ret = 0, page_nr = 0; |
5274f052 | 207 | |
d6785d91 RV |
208 | if (!spd_pages) |
209 | return 0; | |
210 | ||
8924feff AV |
211 | if (unlikely(!pipe->readers)) { |
212 | send_sig(SIGPIPE, current, 0); | |
213 | ret = -EPIPE; | |
214 | goto out; | |
215 | } | |
5274f052 | 216 | |
6718b6f8 | 217 | while (!pipe_full(head, tail, pipe->max_usage)) { |
8cefc107 | 218 | struct pipe_buffer *buf = &pipe->bufs[head & mask]; |
5274f052 | 219 | |
8924feff AV |
220 | buf->page = spd->pages[page_nr]; |
221 | buf->offset = spd->partial[page_nr].offset; | |
222 | buf->len = spd->partial[page_nr].len; | |
223 | buf->private = spd->partial[page_nr].private; | |
224 | buf->ops = spd->ops; | |
5a81e6a1 | 225 | buf->flags = 0; |
5274f052 | 226 | |
8cefc107 DH |
227 | head++; |
228 | pipe->head = head; | |
8924feff AV |
229 | page_nr++; |
230 | ret += buf->len; | |
29e35094 | 231 | |
8924feff | 232 | if (!--spd->nr_pages) |
5274f052 | 233 | break; |
5274f052 JA |
234 | } |
235 | ||
8924feff AV |
236 | if (!ret) |
237 | ret = -EAGAIN; | |
5274f052 | 238 | |
8924feff | 239 | out: |
00de00bd | 240 | while (page_nr < spd_pages) |
bbdfc2f7 | 241 | spd->spd_release(spd, page_nr++); |
5274f052 JA |
242 | |
243 | return ret; | |
244 | } | |
2b514574 | 245 | EXPORT_SYMBOL_GPL(splice_to_pipe); |
5274f052 | 246 | |
79fddc4e AV |
247 | ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf) |
248 | { | |
8cefc107 DH |
249 | unsigned int head = pipe->head; |
250 | unsigned int tail = pipe->tail; | |
251 | unsigned int mask = pipe->ring_size - 1; | |
79fddc4e AV |
252 | int ret; |
253 | ||
254 | if (unlikely(!pipe->readers)) { | |
255 | send_sig(SIGPIPE, current, 0); | |
256 | ret = -EPIPE; | |
6718b6f8 | 257 | } else if (pipe_full(head, tail, pipe->max_usage)) { |
79fddc4e AV |
258 | ret = -EAGAIN; |
259 | } else { | |
8cefc107 DH |
260 | pipe->bufs[head & mask] = *buf; |
261 | pipe->head = head + 1; | |
79fddc4e AV |
262 | return buf->len; |
263 | } | |
a779638c | 264 | pipe_buf_release(pipe, buf); |
79fddc4e AV |
265 | return ret; |
266 | } | |
267 | EXPORT_SYMBOL(add_to_pipe); | |
268 | ||
35f3d14d JA |
269 | /* |
270 | * Check if we need to grow the arrays holding pages and partial page | |
271 | * descriptions. | |
272 | */ | |
047fe360 | 273 | int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd) |
35f3d14d | 274 | { |
6718b6f8 | 275 | unsigned int max_usage = READ_ONCE(pipe->max_usage); |
047fe360 | 276 | |
8cefc107 DH |
277 | spd->nr_pages_max = max_usage; |
278 | if (max_usage <= PIPE_DEF_BUFFERS) | |
35f3d14d JA |
279 | return 0; |
280 | ||
8cefc107 DH |
281 | spd->pages = kmalloc_array(max_usage, sizeof(struct page *), GFP_KERNEL); |
282 | spd->partial = kmalloc_array(max_usage, sizeof(struct partial_page), | |
6da2ec56 | 283 | GFP_KERNEL); |
35f3d14d JA |
284 | |
285 | if (spd->pages && spd->partial) | |
286 | return 0; | |
287 | ||
288 | kfree(spd->pages); | |
289 | kfree(spd->partial); | |
290 | return -ENOMEM; | |
291 | } | |
292 | ||
047fe360 | 293 | void splice_shrink_spd(struct splice_pipe_desc *spd) |
35f3d14d | 294 | { |
047fe360 | 295 | if (spd->nr_pages_max <= PIPE_DEF_BUFFERS) |
35f3d14d JA |
296 | return; |
297 | ||
298 | kfree(spd->pages); | |
299 | kfree(spd->partial); | |
300 | } | |
301 | ||
33b3b041 | 302 | /* |
69df79a4 | 303 | * Copy data from a file into pages and then splice those into the output pipe. |
33b3b041 | 304 | */ |
69df79a4 DH |
305 | ssize_t copy_splice_read(struct file *in, loff_t *ppos, |
306 | struct pipe_inode_info *pipe, | |
307 | size_t len, unsigned int flags) | |
33b3b041 DH |
308 | { |
309 | struct iov_iter to; | |
310 | struct bio_vec *bv; | |
311 | struct kiocb kiocb; | |
312 | struct page **pages; | |
313 | ssize_t ret; | |
e69f37bc | 314 | size_t used, npages, chunk, remain, keep = 0; |
33b3b041 DH |
315 | int i; |
316 | ||
317 | /* Work out how much data we can actually add into the pipe */ | |
318 | used = pipe_occupancy(pipe->head, pipe->tail); | |
319 | npages = max_t(ssize_t, pipe->max_usage - used, 0); | |
320 | len = min_t(size_t, len, npages * PAGE_SIZE); | |
321 | npages = DIV_ROUND_UP(len, PAGE_SIZE); | |
322 | ||
323 | bv = kzalloc(array_size(npages, sizeof(bv[0])) + | |
324 | array_size(npages, sizeof(struct page *)), GFP_KERNEL); | |
325 | if (!bv) | |
326 | return -ENOMEM; | |
327 | ||
e69f37bc | 328 | pages = (struct page **)(bv + npages); |
33b3b041 DH |
329 | npages = alloc_pages_bulk_array(GFP_USER, npages, pages); |
330 | if (!npages) { | |
331 | kfree(bv); | |
332 | return -ENOMEM; | |
333 | } | |
334 | ||
335 | remain = len = min_t(size_t, len, npages * PAGE_SIZE); | |
336 | ||
337 | for (i = 0; i < npages; i++) { | |
338 | chunk = min_t(size_t, PAGE_SIZE, remain); | |
339 | bv[i].bv_page = pages[i]; | |
340 | bv[i].bv_offset = 0; | |
341 | bv[i].bv_len = chunk; | |
342 | remain -= chunk; | |
343 | } | |
344 | ||
345 | /* Do the I/O */ | |
346 | iov_iter_bvec(&to, ITER_DEST, bv, npages, len); | |
347 | init_sync_kiocb(&kiocb, in); | |
348 | kiocb.ki_pos = *ppos; | |
349 | ret = call_read_iter(in, &kiocb, &to); | |
350 | ||
33b3b041 | 351 | if (ret > 0) { |
e69f37bc | 352 | keep = DIV_ROUND_UP(ret, PAGE_SIZE); |
33b3b041 DH |
353 | *ppos = kiocb.ki_pos; |
354 | file_accessed(in); | |
355 | } else if (ret < 0) { | |
356 | /* | |
357 | * callers of ->splice_read() expect -EAGAIN on | |
358 | * "can't put anything in there", rather than -EFAULT. | |
359 | */ | |
360 | if (ret == -EFAULT) | |
361 | ret = -EAGAIN; | |
362 | } | |
363 | ||
364 | /* Free any pages that didn't get touched at all. */ | |
e69f37bc DH |
365 | if (keep < npages) |
366 | release_pages(pages + keep, npages - keep); | |
33b3b041 DH |
367 | |
368 | /* Push the remaining pages into the pipe. */ | |
e69f37bc DH |
369 | remain = ret; |
370 | for (i = 0; i < keep; i++) { | |
33b3b041 DH |
371 | struct pipe_buffer *buf = pipe_head_buf(pipe); |
372 | ||
373 | chunk = min_t(size_t, remain, PAGE_SIZE); | |
374 | *buf = (struct pipe_buffer) { | |
375 | .ops = &default_pipe_buf_ops, | |
376 | .page = bv[i].bv_page, | |
377 | .offset = 0, | |
378 | .len = chunk, | |
379 | }; | |
380 | pipe->head++; | |
381 | remain -= chunk; | |
382 | } | |
383 | ||
384 | kfree(bv); | |
385 | return ret; | |
386 | } | |
69df79a4 | 387 | EXPORT_SYMBOL(copy_splice_read); |
33b3b041 | 388 | |
83f9135b JA |
389 | /** |
390 | * generic_file_splice_read - splice data from file to a pipe | |
391 | * @in: file to splice from | |
932cc6d4 | 392 | * @ppos: position in @in |
83f9135b JA |
393 | * @pipe: pipe to splice to |
394 | * @len: number of bytes to splice | |
395 | * @flags: splice modifier flags | |
396 | * | |
932cc6d4 JA |
397 | * Description: |
398 | * Will read pages from given file and fill them into a pipe. Can be | |
82c156f8 | 399 | * used as long as it has more or less sane ->read_iter(). |
932cc6d4 | 400 | * |
83f9135b | 401 | */ |
cbb7e577 JA |
402 | ssize_t generic_file_splice_read(struct file *in, loff_t *ppos, |
403 | struct pipe_inode_info *pipe, size_t len, | |
404 | unsigned int flags) | |
5274f052 | 405 | { |
82c156f8 AV |
406 | struct iov_iter to; |
407 | struct kiocb kiocb; | |
8cefc107 | 408 | int ret; |
be64f884 | 409 | |
de4eda9d | 410 | iov_iter_pipe(&to, ITER_DEST, pipe, len); |
82c156f8 AV |
411 | init_sync_kiocb(&kiocb, in); |
412 | kiocb.ki_pos = *ppos; | |
bb7462b6 | 413 | ret = call_read_iter(in, &kiocb, &to); |
723590ed | 414 | if (ret > 0) { |
82c156f8 | 415 | *ppos = kiocb.ki_pos; |
723590ed | 416 | file_accessed(in); |
82c156f8 | 417 | } else if (ret < 0) { |
0d964934 AV |
418 | /* free what was emitted */ |
419 | pipe_discard_from(pipe, to.start_head); | |
82c156f8 AV |
420 | /* |
421 | * callers of ->splice_read() expect -EAGAIN on | |
422 | * "can't put anything in there", rather than -EFAULT. | |
423 | */ | |
424 | if (ret == -EFAULT) | |
425 | ret = -EAGAIN; | |
723590ed | 426 | } |
5274f052 JA |
427 | |
428 | return ret; | |
429 | } | |
059a8f37 JA |
430 | EXPORT_SYMBOL(generic_file_splice_read); |
431 | ||
241699cd | 432 | const struct pipe_buf_operations default_pipe_buf_ops = { |
c928f642 CH |
433 | .release = generic_pipe_buf_release, |
434 | .try_steal = generic_pipe_buf_try_steal, | |
435 | .get = generic_pipe_buf_get, | |
6818173b MS |
436 | }; |
437 | ||
28a625cb MS |
438 | /* Pipe buffer operations for a socket and similar. */ |
439 | const struct pipe_buf_operations nosteal_pipe_buf_ops = { | |
c928f642 CH |
440 | .release = generic_pipe_buf_release, |
441 | .get = generic_pipe_buf_get, | |
28a625cb MS |
442 | }; |
443 | EXPORT_SYMBOL(nosteal_pipe_buf_ops); | |
444 | ||
5274f052 | 445 | /* |
4f6f0bd2 | 446 | * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos' |
016b661e | 447 | * using sendpage(). Return the number of bytes sent. |
5274f052 | 448 | */ |
76ad4d11 | 449 | static int pipe_to_sendpage(struct pipe_inode_info *pipe, |
5274f052 JA |
450 | struct pipe_buffer *buf, struct splice_desc *sd) |
451 | { | |
6a14b90b | 452 | struct file *file = sd->u.file; |
5274f052 | 453 | loff_t pos = sd->pos; |
a8adbe37 | 454 | int more; |
5274f052 | 455 | |
72c2d531 | 456 | if (!likely(file->f_op->sendpage)) |
a8adbe37 MM |
457 | return -EINVAL; |
458 | ||
35f9c09f | 459 | more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0; |
ae62ca7b | 460 | |
8cefc107 DH |
461 | if (sd->len < sd->total_len && |
462 | pipe_occupancy(pipe->head, pipe->tail) > 1) | |
35f9c09f | 463 | more |= MSG_SENDPAGE_NOTLAST; |
ae62ca7b | 464 | |
a8adbe37 MM |
465 | return file->f_op->sendpage(file, buf->page, buf->offset, |
466 | sd->len, &pos, more); | |
5274f052 JA |
467 | } |
468 | ||
b3c2d2dd MS |
469 | static void wakeup_pipe_writers(struct pipe_inode_info *pipe) |
470 | { | |
471 | smp_mb(); | |
0ddad21d LT |
472 | if (waitqueue_active(&pipe->wr_wait)) |
473 | wake_up_interruptible(&pipe->wr_wait); | |
b3c2d2dd MS |
474 | kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); |
475 | } | |
476 | ||
932cc6d4 | 477 | /** |
b3c2d2dd | 478 | * splice_from_pipe_feed - feed available data from a pipe to a file |
932cc6d4 JA |
479 | * @pipe: pipe to splice from |
480 | * @sd: information to @actor | |
481 | * @actor: handler that splices the data | |
482 | * | |
483 | * Description: | |
b3c2d2dd MS |
484 | * This function loops over the pipe and calls @actor to do the |
485 | * actual moving of a single struct pipe_buffer to the desired | |
486 | * destination. It returns when there's no more buffers left in | |
487 | * the pipe or if the requested number of bytes (@sd->total_len) | |
488 | * have been copied. It returns a positive number (one) if the | |
489 | * pipe needs to be filled with more data, zero if the required | |
490 | * number of bytes have been copied and -errno on error. | |
932cc6d4 | 491 | * |
b3c2d2dd MS |
492 | * This, together with splice_from_pipe_{begin,end,next}, may be |
493 | * used to implement the functionality of __splice_from_pipe() when | |
494 | * locking is required around copying the pipe buffers to the | |
495 | * destination. | |
83f9135b | 496 | */ |
96f9bc8f | 497 | static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd, |
b3c2d2dd | 498 | splice_actor *actor) |
5274f052 | 499 | { |
8cefc107 DH |
500 | unsigned int head = pipe->head; |
501 | unsigned int tail = pipe->tail; | |
502 | unsigned int mask = pipe->ring_size - 1; | |
b3c2d2dd | 503 | int ret; |
5274f052 | 504 | |
ec057595 | 505 | while (!pipe_empty(head, tail)) { |
8cefc107 | 506 | struct pipe_buffer *buf = &pipe->bufs[tail & mask]; |
5274f052 | 507 | |
b3c2d2dd MS |
508 | sd->len = buf->len; |
509 | if (sd->len > sd->total_len) | |
510 | sd->len = sd->total_len; | |
5274f052 | 511 | |
fba597db | 512 | ret = pipe_buf_confirm(pipe, buf); |
a8adbe37 | 513 | if (unlikely(ret)) { |
b3c2d2dd MS |
514 | if (ret == -ENODATA) |
515 | ret = 0; | |
516 | return ret; | |
517 | } | |
a8adbe37 MM |
518 | |
519 | ret = actor(pipe, buf, sd); | |
520 | if (ret <= 0) | |
521 | return ret; | |
522 | ||
b3c2d2dd MS |
523 | buf->offset += ret; |
524 | buf->len -= ret; | |
525 | ||
526 | sd->num_spliced += ret; | |
527 | sd->len -= ret; | |
528 | sd->pos += ret; | |
529 | sd->total_len -= ret; | |
530 | ||
531 | if (!buf->len) { | |
a779638c | 532 | pipe_buf_release(pipe, buf); |
8cefc107 DH |
533 | tail++; |
534 | pipe->tail = tail; | |
6447a3cf | 535 | if (pipe->files) |
b3c2d2dd MS |
536 | sd->need_wakeup = true; |
537 | } | |
5274f052 | 538 | |
b3c2d2dd MS |
539 | if (!sd->total_len) |
540 | return 0; | |
541 | } | |
5274f052 | 542 | |
b3c2d2dd MS |
543 | return 1; |
544 | } | |
5274f052 | 545 | |
d1a819a2 LT |
546 | /* We know we have a pipe buffer, but maybe it's empty? */ |
547 | static inline bool eat_empty_buffer(struct pipe_inode_info *pipe) | |
548 | { | |
549 | unsigned int tail = pipe->tail; | |
550 | unsigned int mask = pipe->ring_size - 1; | |
551 | struct pipe_buffer *buf = &pipe->bufs[tail & mask]; | |
552 | ||
553 | if (unlikely(!buf->len)) { | |
554 | pipe_buf_release(pipe, buf); | |
555 | pipe->tail = tail+1; | |
556 | return true; | |
557 | } | |
558 | ||
559 | return false; | |
560 | } | |
561 | ||
b3c2d2dd MS |
562 | /** |
563 | * splice_from_pipe_next - wait for some data to splice from | |
564 | * @pipe: pipe to splice from | |
565 | * @sd: information about the splice operation | |
566 | * | |
567 | * Description: | |
568 | * This function will wait for some data and return a positive | |
569 | * value (one) if pipe buffers are available. It will return zero | |
570 | * or -errno if no more data needs to be spliced. | |
571 | */ | |
96f9bc8f | 572 | static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd) |
b3c2d2dd | 573 | { |
c725bfce JK |
574 | /* |
575 | * Check for signal early to make process killable when there are | |
576 | * always buffers available | |
577 | */ | |
578 | if (signal_pending(current)) | |
579 | return -ERESTARTSYS; | |
580 | ||
d1a819a2 | 581 | repeat: |
8cefc107 | 582 | while (pipe_empty(pipe->head, pipe->tail)) { |
b3c2d2dd MS |
583 | if (!pipe->writers) |
584 | return 0; | |
016b661e | 585 | |
a28c8b9d | 586 | if (sd->num_spliced) |
b3c2d2dd | 587 | return 0; |
73d62d83 | 588 | |
b3c2d2dd MS |
589 | if (sd->flags & SPLICE_F_NONBLOCK) |
590 | return -EAGAIN; | |
5274f052 | 591 | |
b3c2d2dd MS |
592 | if (signal_pending(current)) |
593 | return -ERESTARTSYS; | |
5274f052 | 594 | |
b3c2d2dd MS |
595 | if (sd->need_wakeup) { |
596 | wakeup_pipe_writers(pipe); | |
597 | sd->need_wakeup = false; | |
5274f052 JA |
598 | } |
599 | ||
472e5b05 | 600 | pipe_wait_readable(pipe); |
b3c2d2dd | 601 | } |
29e35094 | 602 | |
d1a819a2 LT |
603 | if (eat_empty_buffer(pipe)) |
604 | goto repeat; | |
605 | ||
b3c2d2dd MS |
606 | return 1; |
607 | } | |
5274f052 | 608 | |
b3c2d2dd MS |
609 | /** |
610 | * splice_from_pipe_begin - start splicing from pipe | |
b80901bb | 611 | * @sd: information about the splice operation |
b3c2d2dd MS |
612 | * |
613 | * Description: | |
614 | * This function should be called before a loop containing | |
615 | * splice_from_pipe_next() and splice_from_pipe_feed() to | |
616 | * initialize the necessary fields of @sd. | |
617 | */ | |
96f9bc8f | 618 | static void splice_from_pipe_begin(struct splice_desc *sd) |
b3c2d2dd MS |
619 | { |
620 | sd->num_spliced = 0; | |
621 | sd->need_wakeup = false; | |
622 | } | |
5274f052 | 623 | |
b3c2d2dd MS |
624 | /** |
625 | * splice_from_pipe_end - finish splicing from pipe | |
626 | * @pipe: pipe to splice from | |
627 | * @sd: information about the splice operation | |
628 | * | |
629 | * Description: | |
630 | * This function will wake up pipe writers if necessary. It should | |
631 | * be called after a loop containing splice_from_pipe_next() and | |
632 | * splice_from_pipe_feed(). | |
633 | */ | |
96f9bc8f | 634 | static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd) |
b3c2d2dd MS |
635 | { |
636 | if (sd->need_wakeup) | |
637 | wakeup_pipe_writers(pipe); | |
638 | } | |
5274f052 | 639 | |
b3c2d2dd MS |
640 | /** |
641 | * __splice_from_pipe - splice data from a pipe to given actor | |
642 | * @pipe: pipe to splice from | |
643 | * @sd: information to @actor | |
644 | * @actor: handler that splices the data | |
645 | * | |
646 | * Description: | |
647 | * This function does little more than loop over the pipe and call | |
648 | * @actor to do the actual moving of a single struct pipe_buffer to | |
649 | * the desired destination. See pipe_to_file, pipe_to_sendpage, or | |
650 | * pipe_to_user. | |
651 | * | |
652 | */ | |
653 | ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd, | |
654 | splice_actor *actor) | |
655 | { | |
656 | int ret; | |
5274f052 | 657 | |
b3c2d2dd MS |
658 | splice_from_pipe_begin(sd); |
659 | do { | |
c2489e07 | 660 | cond_resched(); |
b3c2d2dd MS |
661 | ret = splice_from_pipe_next(pipe, sd); |
662 | if (ret > 0) | |
663 | ret = splice_from_pipe_feed(pipe, sd, actor); | |
664 | } while (ret > 0); | |
665 | splice_from_pipe_end(pipe, sd); | |
666 | ||
667 | return sd->num_spliced ? sd->num_spliced : ret; | |
5274f052 | 668 | } |
40bee44e | 669 | EXPORT_SYMBOL(__splice_from_pipe); |
5274f052 | 670 | |
932cc6d4 JA |
671 | /** |
672 | * splice_from_pipe - splice data from a pipe to a file | |
673 | * @pipe: pipe to splice from | |
674 | * @out: file to splice to | |
675 | * @ppos: position in @out | |
676 | * @len: how many bytes to splice | |
677 | * @flags: splice modifier flags | |
678 | * @actor: handler that splices the data | |
679 | * | |
680 | * Description: | |
2933970b | 681 | * See __splice_from_pipe. This function locks the pipe inode, |
932cc6d4 JA |
682 | * otherwise it's identical to __splice_from_pipe(). |
683 | * | |
684 | */ | |
6da61809 MF |
685 | ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out, |
686 | loff_t *ppos, size_t len, unsigned int flags, | |
687 | splice_actor *actor) | |
688 | { | |
689 | ssize_t ret; | |
c66ab6fa JA |
690 | struct splice_desc sd = { |
691 | .total_len = len, | |
692 | .flags = flags, | |
693 | .pos = *ppos, | |
6a14b90b | 694 | .u.file = out, |
c66ab6fa | 695 | }; |
6da61809 | 696 | |
61e0d47c | 697 | pipe_lock(pipe); |
c66ab6fa | 698 | ret = __splice_from_pipe(pipe, &sd, actor); |
61e0d47c | 699 | pipe_unlock(pipe); |
6da61809 MF |
700 | |
701 | return ret; | |
702 | } | |
703 | ||
8d020765 AV |
704 | /** |
705 | * iter_file_splice_write - splice data from a pipe to a file | |
706 | * @pipe: pipe info | |
707 | * @out: file to write to | |
708 | * @ppos: position in @out | |
709 | * @len: number of bytes to splice | |
710 | * @flags: splice modifier flags | |
711 | * | |
712 | * Description: | |
713 | * Will either move or copy pages (determined by @flags options) from | |
714 | * the given pipe inode to the given file. | |
715 | * This one is ->write_iter-based. | |
716 | * | |
717 | */ | |
718 | ssize_t | |
719 | iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, | |
720 | loff_t *ppos, size_t len, unsigned int flags) | |
721 | { | |
722 | struct splice_desc sd = { | |
723 | .total_len = len, | |
724 | .flags = flags, | |
725 | .pos = *ppos, | |
726 | .u.file = out, | |
727 | }; | |
6718b6f8 | 728 | int nbufs = pipe->max_usage; |
8d020765 AV |
729 | struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec), |
730 | GFP_KERNEL); | |
731 | ssize_t ret; | |
732 | ||
733 | if (unlikely(!array)) | |
734 | return -ENOMEM; | |
735 | ||
736 | pipe_lock(pipe); | |
737 | ||
738 | splice_from_pipe_begin(&sd); | |
739 | while (sd.total_len) { | |
740 | struct iov_iter from; | |
ec057595 | 741 | unsigned int head, tail, mask; |
8d020765 | 742 | size_t left; |
8cefc107 | 743 | int n; |
8d020765 AV |
744 | |
745 | ret = splice_from_pipe_next(pipe, &sd); | |
746 | if (ret <= 0) | |
747 | break; | |
748 | ||
6718b6f8 | 749 | if (unlikely(nbufs < pipe->max_usage)) { |
8d020765 | 750 | kfree(array); |
6718b6f8 | 751 | nbufs = pipe->max_usage; |
8d020765 AV |
752 | array = kcalloc(nbufs, sizeof(struct bio_vec), |
753 | GFP_KERNEL); | |
754 | if (!array) { | |
755 | ret = -ENOMEM; | |
756 | break; | |
757 | } | |
758 | } | |
759 | ||
ec057595 LT |
760 | head = pipe->head; |
761 | tail = pipe->tail; | |
762 | mask = pipe->ring_size - 1; | |
763 | ||
8d020765 AV |
764 | /* build the vector */ |
765 | left = sd.total_len; | |
0f1d344f | 766 | for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++) { |
8cefc107 | 767 | struct pipe_buffer *buf = &pipe->bufs[tail & mask]; |
8d020765 AV |
768 | size_t this_len = buf->len; |
769 | ||
0f1d344f PB |
770 | /* zero-length bvecs are not supported, skip them */ |
771 | if (!this_len) | |
772 | continue; | |
773 | this_len = min(this_len, left); | |
8d020765 | 774 | |
fba597db | 775 | ret = pipe_buf_confirm(pipe, buf); |
8d020765 AV |
776 | if (unlikely(ret)) { |
777 | if (ret == -ENODATA) | |
778 | ret = 0; | |
779 | goto done; | |
780 | } | |
781 | ||
664e4078 CH |
782 | bvec_set_page(&array[n], buf->page, this_len, |
783 | buf->offset); | |
8d020765 | 784 | left -= this_len; |
0f1d344f | 785 | n++; |
8d020765 AV |
786 | } |
787 | ||
de4eda9d | 788 | iov_iter_bvec(&from, ITER_SOURCE, array, n, sd.total_len - left); |
abbb6589 | 789 | ret = vfs_iter_write(out, &from, &sd.pos, 0); |
8d020765 AV |
790 | if (ret <= 0) |
791 | break; | |
792 | ||
793 | sd.num_spliced += ret; | |
794 | sd.total_len -= ret; | |
dbe4e192 | 795 | *ppos = sd.pos; |
8d020765 AV |
796 | |
797 | /* dismiss the fully eaten buffers, adjust the partial one */ | |
8cefc107 | 798 | tail = pipe->tail; |
8d020765 | 799 | while (ret) { |
8cefc107 | 800 | struct pipe_buffer *buf = &pipe->bufs[tail & mask]; |
8d020765 | 801 | if (ret >= buf->len) { |
8d020765 AV |
802 | ret -= buf->len; |
803 | buf->len = 0; | |
a779638c | 804 | pipe_buf_release(pipe, buf); |
8cefc107 DH |
805 | tail++; |
806 | pipe->tail = tail; | |
8d020765 AV |
807 | if (pipe->files) |
808 | sd.need_wakeup = true; | |
809 | } else { | |
810 | buf->offset += ret; | |
811 | buf->len -= ret; | |
812 | ret = 0; | |
813 | } | |
814 | } | |
815 | } | |
816 | done: | |
817 | kfree(array); | |
818 | splice_from_pipe_end(pipe, &sd); | |
819 | ||
820 | pipe_unlock(pipe); | |
821 | ||
822 | if (sd.num_spliced) | |
823 | ret = sd.num_spliced; | |
824 | ||
825 | return ret; | |
826 | } | |
827 | ||
828 | EXPORT_SYMBOL(iter_file_splice_write); | |
829 | ||
83f9135b JA |
830 | /** |
831 | * generic_splice_sendpage - splice data from a pipe to a socket | |
932cc6d4 | 832 | * @pipe: pipe to splice from |
83f9135b | 833 | * @out: socket to write to |
932cc6d4 | 834 | * @ppos: position in @out |
83f9135b JA |
835 | * @len: number of bytes to splice |
836 | * @flags: splice modifier flags | |
837 | * | |
932cc6d4 JA |
838 | * Description: |
839 | * Will send @len bytes from the pipe to a network socket. No data copying | |
840 | * is involved. | |
83f9135b JA |
841 | * |
842 | */ | |
3a326a2c | 843 | ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out, |
cbb7e577 | 844 | loff_t *ppos, size_t len, unsigned int flags) |
5274f052 | 845 | { |
00522fb4 | 846 | return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage); |
5274f052 JA |
847 | } |
848 | ||
059a8f37 | 849 | EXPORT_SYMBOL(generic_splice_sendpage); |
a0f06780 | 850 | |
36e2c742 CH |
851 | static int warn_unsupported(struct file *file, const char *op) |
852 | { | |
853 | pr_debug_ratelimited( | |
854 | "splice %s not supported for file %pD4 (pid: %d comm: %.20s)\n", | |
855 | op, file, current->pid, current->comm); | |
856 | return -EINVAL; | |
857 | } | |
858 | ||
83f9135b JA |
859 | /* |
860 | * Attempt to initiate a splice from pipe to file. | |
861 | */ | |
3a326a2c | 862 | static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, |
cbb7e577 | 863 | loff_t *ppos, size_t len, unsigned int flags) |
5274f052 | 864 | { |
36e2c742 CH |
865 | if (unlikely(!out->f_op->splice_write)) |
866 | return warn_unsupported(out, "write"); | |
867 | return out->f_op->splice_write(pipe, out, ppos, len, flags); | |
5274f052 JA |
868 | } |
869 | ||
6a3f30b8 DH |
870 | /** |
871 | * vfs_splice_read - Read data from a file and splice it into a pipe | |
872 | * @in: File to splice from | |
873 | * @ppos: Input file offset | |
874 | * @pipe: Pipe to splice to | |
875 | * @len: Number of bytes to splice | |
876 | * @flags: Splice modifier flags (SPLICE_F_*) | |
877 | * | |
878 | * Splice the requested amount of data from the input file to the pipe. This | |
879 | * is synchronous as the caller must hold the pipe lock across the entire | |
880 | * operation. | |
881 | * | |
882 | * If successful, it returns the amount of data spliced, 0 if it hit the EOF or | |
883 | * a hole and a negative error code otherwise. | |
83f9135b | 884 | */ |
6a3f30b8 DH |
885 | long vfs_splice_read(struct file *in, loff_t *ppos, |
886 | struct pipe_inode_info *pipe, size_t len, | |
887 | unsigned int flags) | |
5274f052 | 888 | { |
313d64a3 | 889 | unsigned int p_space; |
5274f052 JA |
890 | int ret; |
891 | ||
49570e9b | 892 | if (unlikely(!(in->f_mode & FMODE_READ))) |
5274f052 | 893 | return -EBADF; |
123856f0 DH |
894 | if (!len) |
895 | return 0; | |
5274f052 | 896 | |
313d64a3 AV |
897 | /* Don't try to read more the pipe has space for. */ |
898 | p_space = pipe->max_usage - pipe_occupancy(pipe->head, pipe->tail); | |
899 | len = min_t(size_t, len, p_space << PAGE_SHIFT); | |
900 | ||
cbb7e577 | 901 | ret = rw_verify_area(READ, in, ppos, len); |
5274f052 JA |
902 | if (unlikely(ret < 0)) |
903 | return ret; | |
904 | ||
03cc0789 AV |
905 | if (unlikely(len > MAX_RW_COUNT)) |
906 | len = MAX_RW_COUNT; | |
907 | ||
36e2c742 CH |
908 | if (unlikely(!in->f_op->splice_read)) |
909 | return warn_unsupported(in, "read"); | |
aa3dbde8 | 910 | /* |
b85930a0 DH |
911 | * O_DIRECT and DAX don't deal with the pagecache, so we allocate a |
912 | * buffer, copy into it and splice that into the pipe. | |
aa3dbde8 | 913 | */ |
b85930a0 | 914 | if ((in->f_flags & O_DIRECT) || IS_DAX(in->f_mapping->host)) |
aa3dbde8 | 915 | return copy_splice_read(in, ppos, pipe, len, flags); |
36e2c742 | 916 | return in->f_op->splice_read(in, ppos, pipe, len, flags); |
5274f052 | 917 | } |
6a3f30b8 | 918 | EXPORT_SYMBOL_GPL(vfs_splice_read); |
5274f052 | 919 | |
932cc6d4 JA |
920 | /** |
921 | * splice_direct_to_actor - splices data directly between two non-pipes | |
922 | * @in: file to splice from | |
923 | * @sd: actor information on where to splice to | |
924 | * @actor: handles the data splicing | |
925 | * | |
926 | * Description: | |
927 | * This is a special case helper to splice directly between two | |
928 | * points, without requiring an explicit pipe. Internally an allocated | |
79685b8d | 929 | * pipe is cached in the process, and reused during the lifetime of |
932cc6d4 JA |
930 | * that process. |
931 | * | |
c66ab6fa JA |
932 | */ |
933 | ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd, | |
934 | splice_direct_actor *actor) | |
b92ce558 JA |
935 | { |
936 | struct pipe_inode_info *pipe; | |
937 | long ret, bytes; | |
c66ab6fa | 938 | size_t len; |
0ff28d9f | 939 | int i, flags, more; |
b92ce558 JA |
940 | |
941 | /* | |
97ef77c5 JD |
942 | * We require the input to be seekable, as we don't want to randomly |
943 | * drop data for eg socket -> socket splicing. Use the piped splicing | |
944 | * for that! | |
b92ce558 | 945 | */ |
97ef77c5 | 946 | if (unlikely(!(in->f_mode & FMODE_LSEEK))) |
b92ce558 JA |
947 | return -EINVAL; |
948 | ||
949 | /* | |
950 | * neither in nor out is a pipe, setup an internal pipe attached to | |
951 | * 'out' and transfer the wanted data from 'in' to 'out' through that | |
952 | */ | |
953 | pipe = current->splice_pipe; | |
49570e9b | 954 | if (unlikely(!pipe)) { |
7bee130e | 955 | pipe = alloc_pipe_info(); |
b92ce558 JA |
956 | if (!pipe) |
957 | return -ENOMEM; | |
958 | ||
959 | /* | |
960 | * We don't have an immediate reader, but we'll read the stuff | |
00522fb4 | 961 | * out of the pipe right after the splice_to_pipe(). So set |
b92ce558 JA |
962 | * PIPE_READERS appropriately. |
963 | */ | |
964 | pipe->readers = 1; | |
965 | ||
966 | current->splice_pipe = pipe; | |
967 | } | |
968 | ||
969 | /* | |
73d62d83 | 970 | * Do the splice. |
b92ce558 | 971 | */ |
b92ce558 | 972 | bytes = 0; |
c66ab6fa JA |
973 | len = sd->total_len; |
974 | flags = sd->flags; | |
975 | ||
976 | /* | |
977 | * Don't block on output, we have to drain the direct pipe. | |
978 | */ | |
979 | sd->flags &= ~SPLICE_F_NONBLOCK; | |
0ff28d9f | 980 | more = sd->flags & SPLICE_F_MORE; |
b92ce558 | 981 | |
8cefc107 | 982 | WARN_ON_ONCE(!pipe_empty(pipe->head, pipe->tail)); |
17614445 | 983 | |
b92ce558 | 984 | while (len) { |
51a92c0f | 985 | size_t read_len; |
a82c53a0 | 986 | loff_t pos = sd->pos, prev_pos = pos; |
b92ce558 | 987 | |
6a3f30b8 | 988 | ret = vfs_splice_read(in, &pos, pipe, len, flags); |
51a92c0f | 989 | if (unlikely(ret <= 0)) |
b92ce558 JA |
990 | goto out_release; |
991 | ||
992 | read_len = ret; | |
c66ab6fa | 993 | sd->total_len = read_len; |
b92ce558 | 994 | |
0ff28d9f CL |
995 | /* |
996 | * If more data is pending, set SPLICE_F_MORE | |
997 | * If this is the last data and SPLICE_F_MORE was not set | |
998 | * initially, clears it. | |
999 | */ | |
1000 | if (read_len < len) | |
1001 | sd->flags |= SPLICE_F_MORE; | |
1002 | else if (!more) | |
1003 | sd->flags &= ~SPLICE_F_MORE; | |
b92ce558 JA |
1004 | /* |
1005 | * NOTE: nonblocking mode only applies to the input. We | |
1006 | * must not do the output in nonblocking mode as then we | |
1007 | * could get stuck data in the internal pipe: | |
1008 | */ | |
c66ab6fa | 1009 | ret = actor(pipe, sd); |
a82c53a0 TZ |
1010 | if (unlikely(ret <= 0)) { |
1011 | sd->pos = prev_pos; | |
b92ce558 | 1012 | goto out_release; |
a82c53a0 | 1013 | } |
b92ce558 JA |
1014 | |
1015 | bytes += ret; | |
1016 | len -= ret; | |
bcd4f3ac | 1017 | sd->pos = pos; |
b92ce558 | 1018 | |
a82c53a0 TZ |
1019 | if (ret < read_len) { |
1020 | sd->pos = prev_pos + ret; | |
51a92c0f | 1021 | goto out_release; |
a82c53a0 | 1022 | } |
b92ce558 JA |
1023 | } |
1024 | ||
9e97198d | 1025 | done: |
8cefc107 | 1026 | pipe->tail = pipe->head = 0; |
80848708 | 1027 | file_accessed(in); |
b92ce558 JA |
1028 | return bytes; |
1029 | ||
1030 | out_release: | |
1031 | /* | |
1032 | * If we did an incomplete transfer we must release | |
1033 | * the pipe buffers in question: | |
1034 | */ | |
8cefc107 DH |
1035 | for (i = 0; i < pipe->ring_size; i++) { |
1036 | struct pipe_buffer *buf = &pipe->bufs[i]; | |
b92ce558 | 1037 | |
a779638c MS |
1038 | if (buf->ops) |
1039 | pipe_buf_release(pipe, buf); | |
b92ce558 | 1040 | } |
b92ce558 | 1041 | |
9e97198d JA |
1042 | if (!bytes) |
1043 | bytes = ret; | |
c66ab6fa | 1044 | |
9e97198d | 1045 | goto done; |
c66ab6fa JA |
1046 | } |
1047 | EXPORT_SYMBOL(splice_direct_to_actor); | |
1048 | ||
1049 | static int direct_splice_actor(struct pipe_inode_info *pipe, | |
1050 | struct splice_desc *sd) | |
1051 | { | |
6a14b90b | 1052 | struct file *file = sd->u.file; |
c66ab6fa | 1053 | |
7995bd28 | 1054 | return do_splice_from(pipe, file, sd->opos, sd->total_len, |
2cb4b05e | 1055 | sd->flags); |
c66ab6fa JA |
1056 | } |
1057 | ||
932cc6d4 JA |
1058 | /** |
1059 | * do_splice_direct - splices data directly between two files | |
1060 | * @in: file to splice from | |
1061 | * @ppos: input file offset | |
1062 | * @out: file to splice to | |
acdb37c3 | 1063 | * @opos: output file offset |
932cc6d4 JA |
1064 | * @len: number of bytes to splice |
1065 | * @flags: splice modifier flags | |
1066 | * | |
1067 | * Description: | |
1068 | * For use by do_sendfile(). splice can easily emulate sendfile, but | |
1069 | * doing it in the application would incur an extra system call | |
1070 | * (splice in + splice out, as compared to just sendfile()). So this helper | |
1071 | * can splice directly through a process-private pipe. | |
1072 | * | |
1073 | */ | |
c66ab6fa | 1074 | long do_splice_direct(struct file *in, loff_t *ppos, struct file *out, |
7995bd28 | 1075 | loff_t *opos, size_t len, unsigned int flags) |
c66ab6fa JA |
1076 | { |
1077 | struct splice_desc sd = { | |
1078 | .len = len, | |
1079 | .total_len = len, | |
1080 | .flags = flags, | |
1081 | .pos = *ppos, | |
6a14b90b | 1082 | .u.file = out, |
7995bd28 | 1083 | .opos = opos, |
c66ab6fa | 1084 | }; |
51a92c0f | 1085 | long ret; |
c66ab6fa | 1086 | |
18c67cb9 AV |
1087 | if (unlikely(!(out->f_mode & FMODE_WRITE))) |
1088 | return -EBADF; | |
1089 | ||
1090 | if (unlikely(out->f_flags & O_APPEND)) | |
1091 | return -EINVAL; | |
1092 | ||
1093 | ret = rw_verify_area(WRITE, out, opos, len); | |
1094 | if (unlikely(ret < 0)) | |
1095 | return ret; | |
1096 | ||
c66ab6fa | 1097 | ret = splice_direct_to_actor(in, &sd, direct_splice_actor); |
51a92c0f | 1098 | if (ret > 0) |
a82c53a0 | 1099 | *ppos = sd.pos; |
51a92c0f | 1100 | |
c66ab6fa | 1101 | return ret; |
b92ce558 | 1102 | } |
1c118596 | 1103 | EXPORT_SYMBOL(do_splice_direct); |
b92ce558 | 1104 | |
8924feff AV |
1105 | static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags) |
1106 | { | |
52bce911 LT |
1107 | for (;;) { |
1108 | if (unlikely(!pipe->readers)) { | |
1109 | send_sig(SIGPIPE, current, 0); | |
1110 | return -EPIPE; | |
1111 | } | |
6718b6f8 | 1112 | if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage)) |
52bce911 | 1113 | return 0; |
8924feff AV |
1114 | if (flags & SPLICE_F_NONBLOCK) |
1115 | return -EAGAIN; | |
1116 | if (signal_pending(current)) | |
1117 | return -ERESTARTSYS; | |
472e5b05 | 1118 | pipe_wait_writable(pipe); |
8924feff | 1119 | } |
8924feff AV |
1120 | } |
1121 | ||
7c77f0b3 MS |
1122 | static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, |
1123 | struct pipe_inode_info *opipe, | |
1124 | size_t len, unsigned int flags); | |
ddac0d39 | 1125 | |
b964bf53 | 1126 | long splice_file_to_pipe(struct file *in, |
faa97c48 AV |
1127 | struct pipe_inode_info *opipe, |
1128 | loff_t *offset, | |
1129 | size_t len, unsigned int flags) | |
1130 | { | |
1131 | long ret; | |
1132 | ||
1133 | pipe_lock(opipe); | |
1134 | ret = wait_for_space(opipe, flags); | |
1135 | if (!ret) | |
6a3f30b8 | 1136 | ret = vfs_splice_read(in, offset, opipe, len, flags); |
faa97c48 AV |
1137 | pipe_unlock(opipe); |
1138 | if (ret > 0) | |
1139 | wakeup_pipe_readers(opipe); | |
1140 | return ret; | |
1141 | } | |
1142 | ||
83f9135b JA |
1143 | /* |
1144 | * Determine where to splice to/from. | |
1145 | */ | |
ee6e00c8 JA |
1146 | long do_splice(struct file *in, loff_t *off_in, struct file *out, |
1147 | loff_t *off_out, size_t len, unsigned int flags) | |
5274f052 | 1148 | { |
7c77f0b3 MS |
1149 | struct pipe_inode_info *ipipe; |
1150 | struct pipe_inode_info *opipe; | |
7995bd28 | 1151 | loff_t offset; |
a4514ebd | 1152 | long ret; |
5274f052 | 1153 | |
90da2e3f PB |
1154 | if (unlikely(!(in->f_mode & FMODE_READ) || |
1155 | !(out->f_mode & FMODE_WRITE))) | |
1156 | return -EBADF; | |
1157 | ||
c73be61c DH |
1158 | ipipe = get_pipe_info(in, true); |
1159 | opipe = get_pipe_info(out, true); | |
7c77f0b3 MS |
1160 | |
1161 | if (ipipe && opipe) { | |
1162 | if (off_in || off_out) | |
1163 | return -ESPIPE; | |
1164 | ||
7c77f0b3 MS |
1165 | /* Splicing to self would be fun, but... */ |
1166 | if (ipipe == opipe) | |
1167 | return -EINVAL; | |
1168 | ||
ee5e0011 SK |
1169 | if ((in->f_flags | out->f_flags) & O_NONBLOCK) |
1170 | flags |= SPLICE_F_NONBLOCK; | |
1171 | ||
7c77f0b3 MS |
1172 | return splice_pipe_to_pipe(ipipe, opipe, len, flags); |
1173 | } | |
1174 | ||
1175 | if (ipipe) { | |
529565dc IM |
1176 | if (off_in) |
1177 | return -ESPIPE; | |
b92ce558 | 1178 | if (off_out) { |
19c9a49b | 1179 | if (!(out->f_mode & FMODE_PWRITE)) |
b92ce558 | 1180 | return -EINVAL; |
ee6e00c8 | 1181 | offset = *off_out; |
7995bd28 AV |
1182 | } else { |
1183 | offset = out->f_pos; | |
1184 | } | |
529565dc | 1185 | |
18c67cb9 AV |
1186 | if (unlikely(out->f_flags & O_APPEND)) |
1187 | return -EINVAL; | |
1188 | ||
1189 | ret = rw_verify_area(WRITE, out, &offset, len); | |
1190 | if (unlikely(ret < 0)) | |
1191 | return ret; | |
1192 | ||
ee5e0011 SK |
1193 | if (in->f_flags & O_NONBLOCK) |
1194 | flags |= SPLICE_F_NONBLOCK; | |
1195 | ||
500368f7 | 1196 | file_start_write(out); |
7995bd28 | 1197 | ret = do_splice_from(ipipe, out, &offset, len, flags); |
500368f7 | 1198 | file_end_write(out); |
a4514ebd | 1199 | |
983652c6 CCC |
1200 | if (ret > 0) |
1201 | fsnotify_modify(out); | |
1202 | ||
7995bd28 AV |
1203 | if (!off_out) |
1204 | out->f_pos = offset; | |
ee6e00c8 JA |
1205 | else |
1206 | *off_out = offset; | |
a4514ebd JA |
1207 | |
1208 | return ret; | |
529565dc | 1209 | } |
5274f052 | 1210 | |
7c77f0b3 | 1211 | if (opipe) { |
529565dc IM |
1212 | if (off_out) |
1213 | return -ESPIPE; | |
b92ce558 | 1214 | if (off_in) { |
19c9a49b | 1215 | if (!(in->f_mode & FMODE_PREAD)) |
b92ce558 | 1216 | return -EINVAL; |
ee6e00c8 | 1217 | offset = *off_in; |
7995bd28 AV |
1218 | } else { |
1219 | offset = in->f_pos; | |
1220 | } | |
529565dc | 1221 | |
ee5e0011 SK |
1222 | if (out->f_flags & O_NONBLOCK) |
1223 | flags |= SPLICE_F_NONBLOCK; | |
1224 | ||
faa97c48 | 1225 | ret = splice_file_to_pipe(in, opipe, &offset, len, flags); |
983652c6 CCC |
1226 | |
1227 | if (ret > 0) | |
1228 | fsnotify_access(in); | |
1229 | ||
7995bd28 AV |
1230 | if (!off_in) |
1231 | in->f_pos = offset; | |
ee6e00c8 JA |
1232 | else |
1233 | *off_in = offset; | |
a4514ebd JA |
1234 | |
1235 | return ret; | |
529565dc | 1236 | } |
5274f052 JA |
1237 | |
1238 | return -EINVAL; | |
1239 | } | |
1240 | ||
ee6e00c8 JA |
1241 | static long __do_splice(struct file *in, loff_t __user *off_in, |
1242 | struct file *out, loff_t __user *off_out, | |
1243 | size_t len, unsigned int flags) | |
1244 | { | |
1245 | struct pipe_inode_info *ipipe; | |
1246 | struct pipe_inode_info *opipe; | |
1247 | loff_t offset, *__off_in = NULL, *__off_out = NULL; | |
1248 | long ret; | |
1249 | ||
1250 | ipipe = get_pipe_info(in, true); | |
1251 | opipe = get_pipe_info(out, true); | |
1252 | ||
0f99fc51 JA |
1253 | if (ipipe) { |
1254 | if (off_in) | |
1255 | return -ESPIPE; | |
1256 | pipe_clear_nowait(in); | |
1257 | } | |
1258 | if (opipe) { | |
1259 | if (off_out) | |
1260 | return -ESPIPE; | |
1261 | pipe_clear_nowait(out); | |
1262 | } | |
ee6e00c8 JA |
1263 | |
1264 | if (off_out) { | |
1265 | if (copy_from_user(&offset, off_out, sizeof(loff_t))) | |
1266 | return -EFAULT; | |
1267 | __off_out = &offset; | |
1268 | } | |
1269 | if (off_in) { | |
1270 | if (copy_from_user(&offset, off_in, sizeof(loff_t))) | |
1271 | return -EFAULT; | |
1272 | __off_in = &offset; | |
1273 | } | |
1274 | ||
1275 | ret = do_splice(in, __off_in, out, __off_out, len, flags); | |
1276 | if (ret < 0) | |
1277 | return ret; | |
1278 | ||
1279 | if (__off_out && copy_to_user(off_out, __off_out, sizeof(loff_t))) | |
1280 | return -EFAULT; | |
1281 | if (__off_in && copy_to_user(off_in, __off_in, sizeof(loff_t))) | |
1282 | return -EFAULT; | |
1283 | ||
1284 | return ret; | |
1285 | } | |
1286 | ||
79fddc4e AV |
1287 | static int iter_to_pipe(struct iov_iter *from, |
1288 | struct pipe_inode_info *pipe, | |
1289 | unsigned flags) | |
912d35f8 | 1290 | { |
79fddc4e AV |
1291 | struct pipe_buffer buf = { |
1292 | .ops = &user_page_pipe_buf_ops, | |
1293 | .flags = flags | |
1294 | }; | |
1295 | size_t total = 0; | |
1296 | int ret = 0; | |
79fddc4e | 1297 | |
7d690c15 | 1298 | while (iov_iter_count(from)) { |
79fddc4e | 1299 | struct page *pages[16]; |
7d690c15 | 1300 | ssize_t left; |
db85a9eb | 1301 | size_t start; |
7d690c15 | 1302 | int i, n; |
db85a9eb | 1303 | |
7d690c15 AV |
1304 | left = iov_iter_get_pages2(from, pages, ~0UL, 16, &start); |
1305 | if (left <= 0) { | |
1306 | ret = left; | |
79fddc4e AV |
1307 | break; |
1308 | } | |
db85a9eb | 1309 | |
7d690c15 AV |
1310 | n = DIV_ROUND_UP(left + start, PAGE_SIZE); |
1311 | for (i = 0; i < n; i++) { | |
1312 | int size = min_t(int, left, PAGE_SIZE - start); | |
1313 | ||
1314 | buf.page = pages[i]; | |
1315 | buf.offset = start; | |
1316 | buf.len = size; | |
1317 | ret = add_to_pipe(pipe, &buf); | |
1318 | if (unlikely(ret < 0)) { | |
1319 | iov_iter_revert(from, left); | |
1320 | // this one got dropped by add_to_pipe() | |
1321 | while (++i < n) | |
1322 | put_page(pages[i]); | |
1323 | goto out; | |
79fddc4e | 1324 | } |
7d690c15 AV |
1325 | total += ret; |
1326 | left -= size; | |
1327 | start = 0; | |
912d35f8 | 1328 | } |
912d35f8 | 1329 | } |
7d690c15 | 1330 | out: |
79fddc4e | 1331 | return total ? total : ret; |
912d35f8 JA |
1332 | } |
1333 | ||
6a14b90b JA |
1334 | static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf, |
1335 | struct splice_desc *sd) | |
1336 | { | |
6130f531 AV |
1337 | int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data); |
1338 | return n == sd->len ? n : -EFAULT; | |
6a14b90b JA |
1339 | } |
1340 | ||
1341 | /* | |
1342 | * For lack of a better implementation, implement vmsplice() to userspace | |
1343 | * as a simple copy of the pipes pages to the user iov. | |
1344 | */ | |
87a3002a AV |
1345 | static long vmsplice_to_user(struct file *file, struct iov_iter *iter, |
1346 | unsigned int flags) | |
6a14b90b | 1347 | { |
c73be61c | 1348 | struct pipe_inode_info *pipe = get_pipe_info(file, true); |
87a3002a AV |
1349 | struct splice_desc sd = { |
1350 | .total_len = iov_iter_count(iter), | |
1351 | .flags = flags, | |
1352 | .u.data = iter | |
1353 | }; | |
1354 | long ret = 0; | |
6a14b90b | 1355 | |
6a14b90b JA |
1356 | if (!pipe) |
1357 | return -EBADF; | |
1358 | ||
0f99fc51 JA |
1359 | pipe_clear_nowait(file); |
1360 | ||
345995fa AV |
1361 | if (sd.total_len) { |
1362 | pipe_lock(pipe); | |
1363 | ret = __splice_from_pipe(pipe, &sd, pipe_to_user); | |
1364 | pipe_unlock(pipe); | |
1365 | } | |
6a14b90b JA |
1366 | |
1367 | return ret; | |
1368 | } | |
1369 | ||
912d35f8 JA |
1370 | /* |
1371 | * vmsplice splices a user address range into a pipe. It can be thought of | |
1372 | * as splice-from-memory, where the regular splice is splice-from-file (or | |
1373 | * to file). In both cases the output is a pipe, naturally. | |
912d35f8 | 1374 | */ |
87a3002a AV |
1375 | static long vmsplice_to_pipe(struct file *file, struct iov_iter *iter, |
1376 | unsigned int flags) | |
912d35f8 | 1377 | { |
ddac0d39 | 1378 | struct pipe_inode_info *pipe; |
87a3002a | 1379 | long ret = 0; |
79fddc4e AV |
1380 | unsigned buf_flag = 0; |
1381 | ||
1382 | if (flags & SPLICE_F_GIFT) | |
1383 | buf_flag = PIPE_BUF_FLAG_GIFT; | |
912d35f8 | 1384 | |
c73be61c | 1385 | pipe = get_pipe_info(file, true); |
ddac0d39 | 1386 | if (!pipe) |
912d35f8 | 1387 | return -EBADF; |
912d35f8 | 1388 | |
0f99fc51 JA |
1389 | pipe_clear_nowait(file); |
1390 | ||
8924feff AV |
1391 | pipe_lock(pipe); |
1392 | ret = wait_for_space(pipe, flags); | |
79fddc4e | 1393 | if (!ret) |
87a3002a | 1394 | ret = iter_to_pipe(iter, pipe, buf_flag); |
8924feff AV |
1395 | pipe_unlock(pipe); |
1396 | if (ret > 0) | |
1397 | wakeup_pipe_readers(pipe); | |
35f3d14d | 1398 | return ret; |
912d35f8 JA |
1399 | } |
1400 | ||
87a3002a AV |
1401 | static int vmsplice_type(struct fd f, int *type) |
1402 | { | |
1403 | if (!f.file) | |
1404 | return -EBADF; | |
1405 | if (f.file->f_mode & FMODE_WRITE) { | |
de4eda9d | 1406 | *type = ITER_SOURCE; |
87a3002a | 1407 | } else if (f.file->f_mode & FMODE_READ) { |
de4eda9d | 1408 | *type = ITER_DEST; |
87a3002a AV |
1409 | } else { |
1410 | fdput(f); | |
1411 | return -EBADF; | |
1412 | } | |
1413 | return 0; | |
1414 | } | |
1415 | ||
6a14b90b JA |
1416 | /* |
1417 | * Note that vmsplice only really supports true splicing _from_ user memory | |
1418 | * to a pipe, not the other way around. Splicing from user memory is a simple | |
1419 | * operation that can be supported without any funky alignment restrictions | |
1420 | * or nasty vm tricks. We simply map in the user memory and fill them into | |
1421 | * a pipe. The reverse isn't quite as easy, though. There are two possible | |
1422 | * solutions for that: | |
1423 | * | |
1424 | * - memcpy() the data internally, at which point we might as well just | |
1425 | * do a regular read() on the buffer anyway. | |
1426 | * - Lots of nasty vm tricks, that are neither fast nor flexible (it | |
1427 | * has restriction limitations on both ends of the pipe). | |
1428 | * | |
1429 | * Currently we punt and implement it as a normal copy, see pipe_to_user(). | |
1430 | * | |
1431 | */ | |
87a3002a | 1432 | SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, uiov, |
30cfe4ef DB |
1433 | unsigned long, nr_segs, unsigned int, flags) |
1434 | { | |
87a3002a AV |
1435 | struct iovec iovstack[UIO_FASTIOV]; |
1436 | struct iovec *iov = iovstack; | |
1437 | struct iov_iter iter; | |
87e5e6da | 1438 | ssize_t error; |
87a3002a AV |
1439 | struct fd f; |
1440 | int type; | |
1441 | ||
598b3cec CH |
1442 | if (unlikely(flags & ~SPLICE_F_ALL)) |
1443 | return -EINVAL; | |
1444 | ||
87a3002a AV |
1445 | f = fdget(fd); |
1446 | error = vmsplice_type(f, &type); | |
1447 | if (error) | |
1448 | return error; | |
1449 | ||
1450 | error = import_iovec(type, uiov, nr_segs, | |
1451 | ARRAY_SIZE(iovstack), &iov, &iter); | |
598b3cec CH |
1452 | if (error < 0) |
1453 | goto out_fdput; | |
30cfe4ef | 1454 | |
598b3cec CH |
1455 | if (!iov_iter_count(&iter)) |
1456 | error = 0; | |
de4eda9d | 1457 | else if (type == ITER_SOURCE) |
598b3cec CH |
1458 | error = vmsplice_to_pipe(f.file, &iter, flags); |
1459 | else | |
1460 | error = vmsplice_to_user(f.file, &iter, flags); | |
87a3002a | 1461 | |
598b3cec CH |
1462 | kfree(iov); |
1463 | out_fdput: | |
87a3002a AV |
1464 | fdput(f); |
1465 | return error; | |
76b021d0 | 1466 | } |
76b021d0 | 1467 | |
836f92ad HC |
1468 | SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in, |
1469 | int, fd_out, loff_t __user *, off_out, | |
1470 | size_t, len, unsigned int, flags) | |
5274f052 | 1471 | { |
2903ff01 | 1472 | struct fd in, out; |
5274f052 | 1473 | long error; |
5274f052 JA |
1474 | |
1475 | if (unlikely(!len)) | |
1476 | return 0; | |
1477 | ||
3d6ea290 AV |
1478 | if (unlikely(flags & ~SPLICE_F_ALL)) |
1479 | return -EINVAL; | |
1480 | ||
5274f052 | 1481 | error = -EBADF; |
2903ff01 AV |
1482 | in = fdget(fd_in); |
1483 | if (in.file) { | |
90da2e3f PB |
1484 | out = fdget(fd_out); |
1485 | if (out.file) { | |
ee6e00c8 JA |
1486 | error = __do_splice(in.file, off_in, out.file, off_out, |
1487 | len, flags); | |
90da2e3f | 1488 | fdput(out); |
5274f052 | 1489 | } |
2903ff01 | 1490 | fdput(in); |
5274f052 | 1491 | } |
5274f052 JA |
1492 | return error; |
1493 | } | |
70524490 | 1494 | |
aadd06e5 JA |
1495 | /* |
1496 | * Make sure there's data to read. Wait for input if we can, otherwise | |
1497 | * return an appropriate error. | |
1498 | */ | |
7c77f0b3 | 1499 | static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags) |
aadd06e5 JA |
1500 | { |
1501 | int ret; | |
1502 | ||
1503 | /* | |
8cefc107 | 1504 | * Check the pipe occupancy without the inode lock first. This function |
aadd06e5 JA |
1505 | * is speculative anyways, so missing one is ok. |
1506 | */ | |
8cefc107 | 1507 | if (!pipe_empty(pipe->head, pipe->tail)) |
aadd06e5 JA |
1508 | return 0; |
1509 | ||
1510 | ret = 0; | |
61e0d47c | 1511 | pipe_lock(pipe); |
aadd06e5 | 1512 | |
8cefc107 | 1513 | while (pipe_empty(pipe->head, pipe->tail)) { |
aadd06e5 JA |
1514 | if (signal_pending(current)) { |
1515 | ret = -ERESTARTSYS; | |
1516 | break; | |
1517 | } | |
1518 | if (!pipe->writers) | |
1519 | break; | |
a28c8b9d LT |
1520 | if (flags & SPLICE_F_NONBLOCK) { |
1521 | ret = -EAGAIN; | |
1522 | break; | |
aadd06e5 | 1523 | } |
472e5b05 | 1524 | pipe_wait_readable(pipe); |
aadd06e5 JA |
1525 | } |
1526 | ||
61e0d47c | 1527 | pipe_unlock(pipe); |
aadd06e5 JA |
1528 | return ret; |
1529 | } | |
1530 | ||
1531 | /* | |
1532 | * Make sure there's writeable room. Wait for room if we can, otherwise | |
1533 | * return an appropriate error. | |
1534 | */ | |
7c77f0b3 | 1535 | static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags) |
aadd06e5 JA |
1536 | { |
1537 | int ret; | |
1538 | ||
1539 | /* | |
8cefc107 | 1540 | * Check pipe occupancy without the inode lock first. This function |
aadd06e5 JA |
1541 | * is speculative anyways, so missing one is ok. |
1542 | */ | |
566d1362 | 1543 | if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage)) |
aadd06e5 JA |
1544 | return 0; |
1545 | ||
1546 | ret = 0; | |
61e0d47c | 1547 | pipe_lock(pipe); |
aadd06e5 | 1548 | |
6718b6f8 | 1549 | while (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) { |
aadd06e5 JA |
1550 | if (!pipe->readers) { |
1551 | send_sig(SIGPIPE, current, 0); | |
1552 | ret = -EPIPE; | |
1553 | break; | |
1554 | } | |
1555 | if (flags & SPLICE_F_NONBLOCK) { | |
1556 | ret = -EAGAIN; | |
1557 | break; | |
1558 | } | |
1559 | if (signal_pending(current)) { | |
1560 | ret = -ERESTARTSYS; | |
1561 | break; | |
1562 | } | |
472e5b05 | 1563 | pipe_wait_writable(pipe); |
aadd06e5 JA |
1564 | } |
1565 | ||
61e0d47c | 1566 | pipe_unlock(pipe); |
aadd06e5 JA |
1567 | return ret; |
1568 | } | |
1569 | ||
7c77f0b3 MS |
1570 | /* |
1571 | * Splice contents of ipipe to opipe. | |
1572 | */ | |
1573 | static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, | |
1574 | struct pipe_inode_info *opipe, | |
1575 | size_t len, unsigned int flags) | |
1576 | { | |
1577 | struct pipe_buffer *ibuf, *obuf; | |
8cefc107 DH |
1578 | unsigned int i_head, o_head; |
1579 | unsigned int i_tail, o_tail; | |
1580 | unsigned int i_mask, o_mask; | |
1581 | int ret = 0; | |
7c77f0b3 MS |
1582 | bool input_wakeup = false; |
1583 | ||
1584 | ||
1585 | retry: | |
1586 | ret = ipipe_prep(ipipe, flags); | |
1587 | if (ret) | |
1588 | return ret; | |
1589 | ||
1590 | ret = opipe_prep(opipe, flags); | |
1591 | if (ret) | |
1592 | return ret; | |
1593 | ||
1594 | /* | |
1595 | * Potential ABBA deadlock, work around it by ordering lock | |
1596 | * grabbing by pipe info address. Otherwise two different processes | |
1597 | * could deadlock (one doing tee from A -> B, the other from B -> A). | |
1598 | */ | |
1599 | pipe_double_lock(ipipe, opipe); | |
1600 | ||
8cefc107 DH |
1601 | i_tail = ipipe->tail; |
1602 | i_mask = ipipe->ring_size - 1; | |
1603 | o_head = opipe->head; | |
1604 | o_mask = opipe->ring_size - 1; | |
1605 | ||
7c77f0b3 | 1606 | do { |
8cefc107 DH |
1607 | size_t o_len; |
1608 | ||
7c77f0b3 MS |
1609 | if (!opipe->readers) { |
1610 | send_sig(SIGPIPE, current, 0); | |
1611 | if (!ret) | |
1612 | ret = -EPIPE; | |
1613 | break; | |
1614 | } | |
1615 | ||
8cefc107 DH |
1616 | i_head = ipipe->head; |
1617 | o_tail = opipe->tail; | |
1618 | ||
1619 | if (pipe_empty(i_head, i_tail) && !ipipe->writers) | |
7c77f0b3 MS |
1620 | break; |
1621 | ||
1622 | /* | |
1623 | * Cannot make any progress, because either the input | |
1624 | * pipe is empty or the output pipe is full. | |
1625 | */ | |
8cefc107 | 1626 | if (pipe_empty(i_head, i_tail) || |
6718b6f8 | 1627 | pipe_full(o_head, o_tail, opipe->max_usage)) { |
7c77f0b3 MS |
1628 | /* Already processed some buffers, break */ |
1629 | if (ret) | |
1630 | break; | |
1631 | ||
1632 | if (flags & SPLICE_F_NONBLOCK) { | |
1633 | ret = -EAGAIN; | |
1634 | break; | |
1635 | } | |
1636 | ||
1637 | /* | |
1638 | * We raced with another reader/writer and haven't | |
1639 | * managed to process any buffers. A zero return | |
1640 | * value means EOF, so retry instead. | |
1641 | */ | |
1642 | pipe_unlock(ipipe); | |
1643 | pipe_unlock(opipe); | |
1644 | goto retry; | |
1645 | } | |
1646 | ||
8cefc107 DH |
1647 | ibuf = &ipipe->bufs[i_tail & i_mask]; |
1648 | obuf = &opipe->bufs[o_head & o_mask]; | |
7c77f0b3 MS |
1649 | |
1650 | if (len >= ibuf->len) { | |
1651 | /* | |
1652 | * Simply move the whole buffer from ipipe to opipe | |
1653 | */ | |
1654 | *obuf = *ibuf; | |
1655 | ibuf->ops = NULL; | |
8cefc107 DH |
1656 | i_tail++; |
1657 | ipipe->tail = i_tail; | |
7c77f0b3 | 1658 | input_wakeup = true; |
8cefc107 DH |
1659 | o_len = obuf->len; |
1660 | o_head++; | |
1661 | opipe->head = o_head; | |
7c77f0b3 MS |
1662 | } else { |
1663 | /* | |
1664 | * Get a reference to this pipe buffer, | |
1665 | * so we can copy the contents over. | |
1666 | */ | |
15fab63e MW |
1667 | if (!pipe_buf_get(ipipe, ibuf)) { |
1668 | if (ret == 0) | |
1669 | ret = -EFAULT; | |
1670 | break; | |
1671 | } | |
7c77f0b3 MS |
1672 | *obuf = *ibuf; |
1673 | ||
1674 | /* | |
f6dd9755 | 1675 | * Don't inherit the gift and merge flags, we need to |
7c77f0b3 MS |
1676 | * prevent multiple steals of this page. |
1677 | */ | |
1678 | obuf->flags &= ~PIPE_BUF_FLAG_GIFT; | |
f6dd9755 | 1679 | obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE; |
a0ce2f0a | 1680 | |
7c77f0b3 | 1681 | obuf->len = len; |
8cefc107 DH |
1682 | ibuf->offset += len; |
1683 | ibuf->len -= len; | |
1684 | o_len = len; | |
1685 | o_head++; | |
1686 | opipe->head = o_head; | |
7c77f0b3 | 1687 | } |
8cefc107 DH |
1688 | ret += o_len; |
1689 | len -= o_len; | |
7c77f0b3 MS |
1690 | } while (len); |
1691 | ||
1692 | pipe_unlock(ipipe); | |
1693 | pipe_unlock(opipe); | |
1694 | ||
1695 | /* | |
1696 | * If we put data in the output pipe, wakeup any potential readers. | |
1697 | */ | |
825cdcb1 NK |
1698 | if (ret > 0) |
1699 | wakeup_pipe_readers(opipe); | |
1700 | ||
7c77f0b3 MS |
1701 | if (input_wakeup) |
1702 | wakeup_pipe_writers(ipipe); | |
1703 | ||
1704 | return ret; | |
1705 | } | |
1706 | ||
70524490 JA |
1707 | /* |
1708 | * Link contents of ipipe to opipe. | |
1709 | */ | |
1710 | static int link_pipe(struct pipe_inode_info *ipipe, | |
1711 | struct pipe_inode_info *opipe, | |
1712 | size_t len, unsigned int flags) | |
1713 | { | |
1714 | struct pipe_buffer *ibuf, *obuf; | |
8cefc107 DH |
1715 | unsigned int i_head, o_head; |
1716 | unsigned int i_tail, o_tail; | |
1717 | unsigned int i_mask, o_mask; | |
1718 | int ret = 0; | |
70524490 JA |
1719 | |
1720 | /* | |
1721 | * Potential ABBA deadlock, work around it by ordering lock | |
61e0d47c | 1722 | * grabbing by pipe info address. Otherwise two different processes |
70524490 JA |
1723 | * could deadlock (one doing tee from A -> B, the other from B -> A). |
1724 | */ | |
61e0d47c | 1725 | pipe_double_lock(ipipe, opipe); |
70524490 | 1726 | |
8cefc107 DH |
1727 | i_tail = ipipe->tail; |
1728 | i_mask = ipipe->ring_size - 1; | |
1729 | o_head = opipe->head; | |
1730 | o_mask = opipe->ring_size - 1; | |
1731 | ||
aadd06e5 | 1732 | do { |
70524490 JA |
1733 | if (!opipe->readers) { |
1734 | send_sig(SIGPIPE, current, 0); | |
1735 | if (!ret) | |
1736 | ret = -EPIPE; | |
1737 | break; | |
1738 | } | |
70524490 | 1739 | |
8cefc107 DH |
1740 | i_head = ipipe->head; |
1741 | o_tail = opipe->tail; | |
1742 | ||
aadd06e5 | 1743 | /* |
8cefc107 | 1744 | * If we have iterated all input buffers or run out of |
aadd06e5 JA |
1745 | * output room, break. |
1746 | */ | |
8cefc107 | 1747 | if (pipe_empty(i_head, i_tail) || |
6718b6f8 | 1748 | pipe_full(o_head, o_tail, opipe->max_usage)) |
aadd06e5 | 1749 | break; |
70524490 | 1750 | |
8cefc107 DH |
1751 | ibuf = &ipipe->bufs[i_tail & i_mask]; |
1752 | obuf = &opipe->bufs[o_head & o_mask]; | |
70524490 JA |
1753 | |
1754 | /* | |
aadd06e5 JA |
1755 | * Get a reference to this pipe buffer, |
1756 | * so we can copy the contents over. | |
70524490 | 1757 | */ |
15fab63e MW |
1758 | if (!pipe_buf_get(ipipe, ibuf)) { |
1759 | if (ret == 0) | |
1760 | ret = -EFAULT; | |
1761 | break; | |
1762 | } | |
aadd06e5 | 1763 | |
aadd06e5 JA |
1764 | *obuf = *ibuf; |
1765 | ||
2a27250e | 1766 | /* |
f6dd9755 CH |
1767 | * Don't inherit the gift and merge flag, we need to prevent |
1768 | * multiple steals of this page. | |
2a27250e | 1769 | */ |
aadd06e5 | 1770 | obuf->flags &= ~PIPE_BUF_FLAG_GIFT; |
f6dd9755 | 1771 | obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE; |
a0ce2f0a | 1772 | |
aadd06e5 JA |
1773 | if (obuf->len > len) |
1774 | obuf->len = len; | |
aadd06e5 JA |
1775 | ret += obuf->len; |
1776 | len -= obuf->len; | |
8cefc107 DH |
1777 | |
1778 | o_head++; | |
1779 | opipe->head = o_head; | |
1780 | i_tail++; | |
aadd06e5 | 1781 | } while (len); |
70524490 | 1782 | |
61e0d47c MS |
1783 | pipe_unlock(ipipe); |
1784 | pipe_unlock(opipe); | |
70524490 | 1785 | |
aadd06e5 JA |
1786 | /* |
1787 | * If we put data in the output pipe, wakeup any potential readers. | |
1788 | */ | |
825cdcb1 NK |
1789 | if (ret > 0) |
1790 | wakeup_pipe_readers(opipe); | |
70524490 JA |
1791 | |
1792 | return ret; | |
1793 | } | |
1794 | ||
1795 | /* | |
1796 | * This is a tee(1) implementation that works on pipes. It doesn't copy | |
1797 | * any data, it simply references the 'in' pages on the 'out' pipe. | |
1798 | * The 'flags' used are the SPLICE_F_* variants, currently the only | |
1799 | * applicable one is SPLICE_F_NONBLOCK. | |
1800 | */ | |
9dafdfc2 | 1801 | long do_tee(struct file *in, struct file *out, size_t len, unsigned int flags) |
70524490 | 1802 | { |
c73be61c DH |
1803 | struct pipe_inode_info *ipipe = get_pipe_info(in, true); |
1804 | struct pipe_inode_info *opipe = get_pipe_info(out, true); | |
aadd06e5 | 1805 | int ret = -EINVAL; |
70524490 | 1806 | |
90da2e3f PB |
1807 | if (unlikely(!(in->f_mode & FMODE_READ) || |
1808 | !(out->f_mode & FMODE_WRITE))) | |
1809 | return -EBADF; | |
1810 | ||
70524490 | 1811 | /* |
aadd06e5 JA |
1812 | * Duplicate the contents of ipipe to opipe without actually |
1813 | * copying the data. | |
70524490 | 1814 | */ |
aadd06e5 | 1815 | if (ipipe && opipe && ipipe != opipe) { |
ee5e0011 SK |
1816 | if ((in->f_flags | out->f_flags) & O_NONBLOCK) |
1817 | flags |= SPLICE_F_NONBLOCK; | |
1818 | ||
aadd06e5 JA |
1819 | /* |
1820 | * Keep going, unless we encounter an error. The ipipe/opipe | |
1821 | * ordering doesn't really matter. | |
1822 | */ | |
7c77f0b3 | 1823 | ret = ipipe_prep(ipipe, flags); |
aadd06e5 | 1824 | if (!ret) { |
7c77f0b3 | 1825 | ret = opipe_prep(opipe, flags); |
02cf01ae | 1826 | if (!ret) |
aadd06e5 | 1827 | ret = link_pipe(ipipe, opipe, len, flags); |
aadd06e5 JA |
1828 | } |
1829 | } | |
70524490 | 1830 | |
aadd06e5 | 1831 | return ret; |
70524490 JA |
1832 | } |
1833 | ||
836f92ad | 1834 | SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags) |
70524490 | 1835 | { |
90da2e3f | 1836 | struct fd in, out; |
2903ff01 | 1837 | int error; |
70524490 | 1838 | |
3d6ea290 AV |
1839 | if (unlikely(flags & ~SPLICE_F_ALL)) |
1840 | return -EINVAL; | |
1841 | ||
70524490 JA |
1842 | if (unlikely(!len)) |
1843 | return 0; | |
1844 | ||
1845 | error = -EBADF; | |
2903ff01 AV |
1846 | in = fdget(fdin); |
1847 | if (in.file) { | |
90da2e3f PB |
1848 | out = fdget(fdout); |
1849 | if (out.file) { | |
1850 | error = do_tee(in.file, out.file, len, flags); | |
1851 | fdput(out); | |
70524490 | 1852 | } |
2903ff01 | 1853 | fdput(in); |
70524490 JA |
1854 | } |
1855 | ||
1856 | return error; | |
1857 | } |