projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
eventpoll: support non-blocking do_epoll_ctl() calls
[linux-2.6-block.git] / fs / io_uring.c
CommitLineData
2b188cc1
JA		 1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Shared application/kernel submission and completion ring pairs, for
4 * supporting fast/efficient IO.
5 *
6 * A note on the read/write ordering memory barriers that are matched between
1e84b97b
SB		 7 * the application and kernel side.
8 *
9 * After the application reads the CQ ring tail, it must use an
10 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
11 * before writing the tail (using smp_load_acquire to read the tail will
12 * do). It also needs a smp_mb() before updating CQ head (ordering the
13 * entry load(s) with the head store), pairing with an implicit barrier
14 * through a control-dependency in io_get_cqring (smp_store_release to
15 * store head will do). Failure to do so could lead to reading invalid
16 * CQ entries.
17 *
18 * Likewise, the application must use an appropriate smp_wmb() before
19 * writing the SQ tail (ordering SQ entry stores with the tail store),
20 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
21 * to store the tail will do). And it needs a barrier ordering the SQ
22 * head load before writing new SQ entries (smp_load_acquire to read
23 * head will do).
24 *
25 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
26 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
27 * updating the SQ tail; a full memory barrier smp_mb() is needed
28 * between.
2b188cc1
JA		 29 *
30 * Also see the examples in the liburing library:
31 *
32 * git://git.kernel.dk/liburing
33 *
34 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
35 * from data shared between the kernel and application. This is done both
36 * for ordering purposes, but also to ensure that once a value is loaded from
37 * data that the application could potentially modify, it remains stable.
38 *
39 * Copyright (C) 2018-2019 Jens Axboe
c992fe29 40 * Copyright (c) 2018-2019 Christoph Hellwig
2b188cc1
JA		 41 */
42#include <linux/kernel.h>
43#include <linux/init.h>
44#include <linux/errno.h>
45#include <linux/syscalls.h>
46#include <linux/compat.h>
47#include <linux/refcount.h>
48#include <linux/uio.h>
6b47ee6e 49#include <linux/bits.h>
2b188cc1
JA		 50
51#include <linux/sched/signal.h>
52#include <linux/fs.h>
53#include <linux/file.h>
54#include <linux/fdtable.h>
55#include <linux/mm.h>
56#include <linux/mman.h>
57#include <linux/mmu_context.h>
58#include <linux/percpu.h>
59#include <linux/slab.h>
6c271ce2 60#include <linux/kthread.h>
2b188cc1 61#include <linux/blkdev.h>
edafccee 62#include <linux/bvec.h>
2b188cc1
JA		 63#include <linux/net.h>
64#include <net/sock.h>
65#include <net/af_unix.h>
6b06314c 66#include <net/scm.h>
2b188cc1
JA		 67#include <linux/anon_inodes.h>
68#include <linux/sched/mm.h>
69#include <linux/uaccess.h>
70#include <linux/nospec.h>
edafccee
JA		 71#include <linux/sizes.h>
72#include <linux/hugetlb.h>
aa4c3967 73#include <linux/highmem.h>
15b71abe
JA		 74#include <linux/namei.h>
75#include <linux/fsnotify.h>
4840e418 76#include <linux/fadvise.h>
2b188cc1 77
c826bd7a
DD		 78#define CREATE_TRACE_POINTS
79#include <trace/events/io_uring.h>
80
2b188cc1
JA		 81#include <uapi/linux/io_uring.h>
82
83#include "internal.h"
561fb04a 84#include "io-wq.h"
2b188cc1 85
5277deaa 86#define IORING_MAX_ENTRIES 32768
33a107f0 87#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
65e19f54
JA		 88
89/*
90 * Shift of 9 is 512 entries, or exactly one page on 64-bit archs
91 */
92#define IORING_FILE_TABLE_SHIFT 9
93#define IORING_MAX_FILES_TABLE (1U << IORING_FILE_TABLE_SHIFT)
94#define IORING_FILE_TABLE_MASK (IORING_MAX_FILES_TABLE - 1)
95#define IORING_MAX_FIXED_FILES (64 * IORING_MAX_FILES_TABLE)
2b188cc1
JA		 96
97struct io_uring {
98 u32 head ____cacheline_aligned_in_smp;
99 u32 tail ____cacheline_aligned_in_smp;
100};
101
1e84b97b 102/*
75b28aff
HV		 103 * This data is shared with the application through the mmap at offsets
104 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
1e84b97b
SB		 105 *
106 * The offsets to the member fields are published through struct
107 * io_sqring_offsets when calling io_uring_setup.
108 */
75b28aff 109struct io_rings {
1e84b97b
SB		 110 /*
111 * Head and tail offsets into the ring; the offsets need to be
112 * masked to get valid indices.
113 *
75b28aff
HV		 114 * The kernel controls head of the sq ring and the tail of the cq ring,
115 * and the application controls tail of the sq ring and the head of the
116 * cq ring.
1e84b97b 117 */
75b28aff 118 struct io_uring sq, cq;
1e84b97b 119 /*
75b28aff 120 * Bitmasks to apply to head and tail offsets (constant, equals
1e84b97b
SB		 121 * ring_entries - 1)
122 */
75b28aff
HV		 123 u32 sq_ring_mask, cq_ring_mask;
124 /* Ring sizes (constant, power of 2) */
125 u32 sq_ring_entries, cq_ring_entries;
1e84b97b
SB		 126 /*
127 * Number of invalid entries dropped by the kernel due to
128 * invalid index stored in array
129 *
130 * Written by the kernel, shouldn't be modified by the
131 * application (i.e. get number of "new events" by comparing to
132 * cached value).
133 *
134 * After a new SQ head value was read by the application this
135 * counter includes all submissions that were dropped reaching
136 * the new SQ head (and possibly more).
137 */
75b28aff 138 u32 sq_dropped;
1e84b97b
SB		 139 /*
140 * Runtime flags
141 *
142 * Written by the kernel, shouldn't be modified by the
143 * application.
144 *
145 * The application needs a full memory barrier before checking
146 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
147 */
75b28aff 148 u32 sq_flags;
1e84b97b
SB		 149 /*
150 * Number of completion events lost because the queue was full;
151 * this should be avoided by the application by making sure
0b4295b5 152 * there are not more requests pending than there is space in
1e84b97b
SB		 153 * the completion queue.
154 *
155 * Written by the kernel, shouldn't be modified by the
156 * application (i.e. get number of "new events" by comparing to
157 * cached value).
158 *
159 * As completion events come in out of order this counter is not
160 * ordered with any other data.
161 */
75b28aff 162 u32 cq_overflow;
1e84b97b
SB		 163 /*
164 * Ring buffer of completion events.
165 *
166 * The kernel writes completion events fresh every time they are
167 * produced, so the application is allowed to modify pending
168 * entries.
169 */
75b28aff 170 struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
2b188cc1
JA		 171};
172
edafccee
JA		 173struct io_mapped_ubuf {
174 u64 ubuf;
175 size_t len;
176 struct bio_vec *bvec;
177 unsigned int nr_bvecs;
178};
179
65e19f54
JA		 180struct fixed_file_table {
181 struct file **files;
31b51510
JA		 182};
183
05f3fb3c
JA		 184enum {
185 FFD_F_ATOMIC,
186};
187
188struct fixed_file_data {
189 struct fixed_file_table *table;
190 struct io_ring_ctx *ctx;
191
192 struct percpu_ref refs;
193 struct llist_head put_llist;
194 unsigned long state;
195 struct work_struct ref_work;
196 struct completion done;
197};
198
2b188cc1
JA		 199struct io_ring_ctx {
200 struct {
201 struct percpu_ref refs;
202 } ____cacheline_aligned_in_smp;
203
204 struct {
205 unsigned int flags;
69b3e546
JA		 206 int compat: 1;
207 int account_mem: 1;
208 int cq_overflow_flushed: 1;
209 int drain_next: 1;
f2842ab5 210 int eventfd_async: 1;
2b188cc1 211
75b28aff
HV		 212 /*
213 * Ring buffer of indices into array of io_uring_sqe, which is
214 * mmapped by the application using the IORING_OFF_SQES offset.
215 *
216 * This indirection could e.g. be used to assign fixed
217 * io_uring_sqe entries to operations and only submit them to
218 * the queue when needed.
219 *
220 * The kernel modifies neither the indices array nor the entries
221 * array.
222 */
223 u32 *sq_array;
2b188cc1
JA		 224 unsigned cached_sq_head;
225 unsigned sq_entries;
226 unsigned sq_mask;
6c271ce2 227 unsigned sq_thread_idle;
498ccd9e 228 unsigned cached_sq_dropped;
206aefde 229 atomic_t cached_cq_overflow;
ad3eb2c8 230 unsigned long sq_check_overflow;
de0617e4
JA		 231
232 struct list_head defer_list;
5262f567 233 struct list_head timeout_list;
1d7bb1d5 234 struct list_head cq_overflow_list;
fcb323cc
JA		 235
236 wait_queue_head_t inflight_wait;
ad3eb2c8 237 struct io_uring_sqe *sq_sqes;
2b188cc1
JA		 238 } ____cacheline_aligned_in_smp;
239
206aefde
JA		 240 struct io_rings *rings;
241
2b188cc1 242 /* IO offload */
561fb04a 243 struct io_wq *io_wq;
6c271ce2 244 struct task_struct *sqo_thread; /* if using sq thread polling */
2b188cc1 245 struct mm_struct *sqo_mm;
6c271ce2 246 wait_queue_head_t sqo_wait;
75b28aff 247
6b06314c
JA		 248 /*
249 * If used, fixed file set. Writers must ensure that ->refs is dead,
250 * readers must ensure that ->refs is alive as long as the file* is
251 * used. Only updated through io_uring_register(2).
252 */
05f3fb3c 253 struct fixed_file_data *file_data;
6b06314c 254 unsigned nr_user_files;
b14cca0c
PB		 255 int ring_fd;
256 struct file *ring_file;
6b06314c 257
edafccee
JA		 258 /* if used, fixed mapped user buffers */
259 unsigned nr_user_bufs;
260 struct io_mapped_ubuf *user_bufs;
261
2b188cc1
JA		 262 struct user_struct *user;
263
0b8c0ec7 264 const struct cred *creds;
181e448d 265
206aefde
JA		 266 /* 0 is for ctx quiesce/reinit/free, 1 is for sqo_thread started */
267 struct completion *completions;
268
0ddf92e8
JA		 269 /* if all else fails... */
270 struct io_kiocb *fallback_req;
271
206aefde
JA		 272#if defined(CONFIG_UNIX)
273 struct socket *ring_sock;
274#endif
275
071698e1
JA		 276 struct idr personality_idr;
277
206aefde
JA		 278 struct {
279 unsigned cached_cq_tail;
280 unsigned cq_entries;
281 unsigned cq_mask;
282 atomic_t cq_timeouts;
ad3eb2c8 283 unsigned long cq_check_overflow;
206aefde
JA		 284 struct wait_queue_head cq_wait;
285 struct fasync_struct *cq_fasync;
286 struct eventfd_ctx *cq_ev_fd;
287 } ____cacheline_aligned_in_smp;
2b188cc1
JA		 288
289 struct {
290 struct mutex uring_lock;
291 wait_queue_head_t wait;
292 } ____cacheline_aligned_in_smp;
293
294 struct {
295 spinlock_t completion_lock;
e94f141b
JA		 296 struct llist_head poll_llist;
297
def596e9
JA		 298 /*
299 * ->poll_list is protected by the ctx->uring_lock for
300 * io_uring instances that don't use IORING_SETUP_SQPOLL.
301 * For SQPOLL, only the single threaded io_sq_thread() will
302 * manipulate the list, hence no extra locking is needed there.
303 */
304 struct list_head poll_list;
78076bb6
JA		 305 struct hlist_head *cancel_hash;
306 unsigned cancel_hash_bits;
e94f141b 307 bool poll_multi_file;
31b51510 308
fcb323cc
JA		 309 spinlock_t inflight_lock;
310 struct list_head inflight_list;
2b188cc1 311 } ____cacheline_aligned_in_smp;
2b188cc1
JA		 312};
313
09bb8394
JA		 314/*
315 * First field must be the file pointer in all the
316 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
317 */
221c5eb2
JA		 318struct io_poll_iocb {
319 struct file *file;
0969e783
JA		 320 union {
321 struct wait_queue_head *head;
322 u64 addr;
323 };
221c5eb2 324 __poll_t events;
8c838788 325 bool done;
221c5eb2 326 bool canceled;
392edb45 327 struct wait_queue_entry wait;
221c5eb2
JA		 328};
329
b5dba59e
JA		 330struct io_close {
331 struct file *file;
332 struct file *put_file;
333 int fd;
334};
335
ad8a48ac
JA		 336struct io_timeout_data {
337 struct io_kiocb *req;
338 struct hrtimer timer;
339 struct timespec64 ts;
340 enum hrtimer_mode mode;
cc42e0ac 341 u32 seq_offset;
ad8a48ac
JA		 342};
343
8ed8d3c3
JA		 344struct io_accept {
345 struct file *file;
346 struct sockaddr __user *addr;
347 int __user *addr_len;
348 int flags;
349};
350
351struct io_sync {
352 struct file *file;
353 loff_t len;
354 loff_t off;
355 int flags;
d63d1b5e 356 int mode;
8ed8d3c3
JA		 357};
358
fbf23849
JA		 359struct io_cancel {
360 struct file *file;
361 u64 addr;
362};
363
b29472ee
JA		 364struct io_timeout {
365 struct file *file;
366 u64 addr;
367 int flags;
26a61679 368 unsigned count;
b29472ee
JA		 369};
370
9adbd45d
JA		 371struct io_rw {
372 /* NOTE: kiocb has the file as the first member, so don't do it here */
373 struct kiocb kiocb;
374 u64 addr;
375 u64 len;
376};
377
3fbb51c1
JA		 378struct io_connect {
379 struct file *file;
380 struct sockaddr __user *addr;
381 int addr_len;
382};
383
e47293fd
JA		 384struct io_sr_msg {
385 struct file *file;
fddaface
JA		 386 union {
387 struct user_msghdr __user *msg;
388 void __user *buf;
389 };
e47293fd 390 int msg_flags;
fddaface 391 size_t len;
e47293fd
JA		 392};
393
15b71abe
JA		 394struct io_open {
395 struct file *file;
396 int dfd;
eddc7ef5 397 union {
eddc7ef5
JA		 398 unsigned mask;
399 };
15b71abe 400 struct filename *filename;
eddc7ef5 401 struct statx __user *buffer;
c12cedf2 402 struct open_how how;
15b71abe
JA		 403};
404
05f3fb3c
JA		 405struct io_files_update {
406 struct file *file;
407 u64 arg;
408 u32 nr_args;
409 u32 offset;
410};
411
4840e418
JA		 412struct io_fadvise {
413 struct file *file;
414 u64 offset;
415 u32 len;
416 u32 advice;
417};
418
c1ca757b
JA		 419struct io_madvise {
420 struct file *file;
421 u64 addr;
422 u32 len;
423 u32 advice;
424};
425
f499a021
JA		 426struct io_async_connect {
427 struct sockaddr_storage address;
428};
429
03b1230c
JA		 430struct io_async_msghdr {
431 struct iovec fast_iov[UIO_FASTIOV];
432 struct iovec *iov;
433 struct sockaddr __user *uaddr;
434 struct msghdr msg;
435};
436
f67676d1
JA		 437struct io_async_rw {
438 struct iovec fast_iov[UIO_FASTIOV];
439 struct iovec *iov;
440 ssize_t nr_segs;
441 ssize_t size;
442};
443
15b71abe
JA		 444struct io_async_open {
445 struct filename *filename;
446};
447
1a6b74fc 448struct io_async_ctx {
f67676d1
JA		 449 union {
450 struct io_async_rw rw;
03b1230c 451 struct io_async_msghdr msg;
f499a021 452 struct io_async_connect connect;
2d28390a 453 struct io_timeout_data timeout;
15b71abe 454 struct io_async_open open;
f67676d1 455 };
1a6b74fc
JA		 456};
457
6b47ee6e
PB		 458enum {
459 REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
460 REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT,
461 REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,
462 REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,
463 REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,
464
465 REQ_F_LINK_NEXT_BIT,
466 REQ_F_FAIL_LINK_BIT,
467 REQ_F_INFLIGHT_BIT,
468 REQ_F_CUR_POS_BIT,
469 REQ_F_NOWAIT_BIT,
470 REQ_F_IOPOLL_COMPLETED_BIT,
471 REQ_F_LINK_TIMEOUT_BIT,
472 REQ_F_TIMEOUT_BIT,
473 REQ_F_ISREG_BIT,
474 REQ_F_MUST_PUNT_BIT,
475 REQ_F_TIMEOUT_NOSEQ_BIT,
476 REQ_F_COMP_LOCKED_BIT,
477};
478
479enum {
480 /* ctx owns file */
481 REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT),
482 /* drain existing IO first */
483 REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT),
484 /* linked sqes */
485 REQ_F_LINK = BIT(REQ_F_LINK_BIT),
486 /* doesn't sever on completion < 0 */
487 REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),
488 /* IOSQE_ASYNC */
489 REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),
490
491 /* already grabbed next link */
492 REQ_F_LINK_NEXT = BIT(REQ_F_LINK_NEXT_BIT),
493 /* fail rest of links */
494 REQ_F_FAIL_LINK = BIT(REQ_F_FAIL_LINK_BIT),
495 /* on inflight list */
496 REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT),
497 /* read/write uses file position */
498 REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT),
499 /* must not punt to workers */
500 REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT),
501 /* polled IO has completed */
502 REQ_F_IOPOLL_COMPLETED = BIT(REQ_F_IOPOLL_COMPLETED_BIT),
503 /* has linked timeout */
504 REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT),
505 /* timeout request */
506 REQ_F_TIMEOUT = BIT(REQ_F_TIMEOUT_BIT),
507 /* regular file */
508 REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),
509 /* must be punted even for NONBLOCK */
510 REQ_F_MUST_PUNT = BIT(REQ_F_MUST_PUNT_BIT),
511 /* no timeout sequence */
512 REQ_F_TIMEOUT_NOSEQ = BIT(REQ_F_TIMEOUT_NOSEQ_BIT),
513 /* completion under lock */
514 REQ_F_COMP_LOCKED = BIT(REQ_F_COMP_LOCKED_BIT),
515};
516
09bb8394
JA		 517/*
518 * NOTE! Each of the iocb union members has the file pointer
519 * as the first entry in their struct definition. So you can
520 * access the file pointer through any of the sub-structs,
521 * or directly as just 'ki_filp' in this struct.
522 */
2b188cc1 523struct io_kiocb {
221c5eb2 524 union {
09bb8394 525 struct file *file;
9adbd45d 526 struct io_rw rw;
221c5eb2 527 struct io_poll_iocb poll;
8ed8d3c3
JA		 528 struct io_accept accept;
529 struct io_sync sync;
fbf23849 530 struct io_cancel cancel;
b29472ee 531 struct io_timeout timeout;
3fbb51c1 532 struct io_connect connect;
e47293fd 533 struct io_sr_msg sr_msg;
15b71abe 534 struct io_open open;
b5dba59e 535 struct io_close close;
05f3fb3c 536 struct io_files_update files_update;
4840e418 537 struct io_fadvise fadvise;
c1ca757b 538 struct io_madvise madvise;
221c5eb2 539 };
2b188cc1 540
1a6b74fc 541 struct io_async_ctx *io;
b14cca0c
PB		 542 /*
543 * llist_node is only used for poll deferred completions
544 */
545 struct llist_node llist_node;
cf6fd4bd
PB		 546 bool has_user;
547 bool in_async;
548 bool needs_fixed_file;
d625c6ee 549 u8 opcode;
2b188cc1
JA		 550
551 struct io_ring_ctx *ctx;
eac406c6
JA		 552 union {
553 struct list_head list;
78076bb6 554 struct hlist_node hash_node;
eac406c6 555 };
9e645e11 556 struct list_head link_list;
2b188cc1 557 unsigned int flags;
c16361c1 558 refcount_t refs;
2b188cc1 559 u64 user_data;
9e645e11 560 u32 result;
de0617e4 561 u32 sequence;
2b188cc1 562
fcb323cc
JA		 563 struct list_head inflight_entry;
564
561fb04a 565 struct io_wq_work work;
2b188cc1
JA		 566};
567
568#define IO_PLUG_THRESHOLD 2
def596e9 569#define IO_IOPOLL_BATCH 8
2b188cc1 570
9a56a232
JA		 571struct io_submit_state {
572 struct blk_plug plug;
573
2579f913
JA		 574 /*
575 * io_kiocb alloc cache
576 */
577 void *reqs[IO_IOPOLL_BATCH];
578 unsigned int free_reqs;
579 unsigned int cur_req;
580
9a56a232
JA		 581 /*
582 * File reference cache
583 */
584 struct file *file;
585 unsigned int fd;
586 unsigned int has_refs;
587 unsigned int used_refs;
588 unsigned int ios_left;
589};
590
d3656344
JA		 591struct io_op_def {
592 /* needs req->io allocated for deferral/async */
593 unsigned async_ctx : 1;
594 /* needs current->mm setup, does mm access */
595 unsigned needs_mm : 1;
596 /* needs req->file assigned */
597 unsigned needs_file : 1;
598 /* needs req->file assigned IFF fd is >= 0 */
599 unsigned fd_non_neg : 1;
600 /* hash wq insertion if file is a regular file */
601 unsigned hash_reg_file : 1;
602 /* unbound wq insertion if file is a non-regular file */
603 unsigned unbound_nonreg_file : 1;
66f4af93
JA		 604 /* opcode is not supported by this kernel */
605 unsigned not_supported : 1;
f86cd20c
JA		 606 /* needs file table */
607 unsigned file_table : 1;
d3656344
JA		 608};
609
610static const struct io_op_def io_op_defs[] = {
0463b6c5
PB		 611 [IORING_OP_NOP] = {},
612 [IORING_OP_READV] = {
d3656344
JA		 613 .async_ctx = 1,
614 .needs_mm = 1,
615 .needs_file = 1,
616 .unbound_nonreg_file = 1,
617 },
0463b6c5 618 [IORING_OP_WRITEV] = {
d3656344
JA		 619 .async_ctx = 1,
620 .needs_mm = 1,
621 .needs_file = 1,
622 .hash_reg_file = 1,
623 .unbound_nonreg_file = 1,
624 },
0463b6c5 625 [IORING_OP_FSYNC] = {
d3656344
JA		 626 .needs_file = 1,
627 },
0463b6c5 628 [IORING_OP_READ_FIXED] = {
d3656344
JA		 629 .needs_file = 1,
630 .unbound_nonreg_file = 1,
631 },
0463b6c5 632 [IORING_OP_WRITE_FIXED] = {
d3656344
JA		 633 .needs_file = 1,
634 .hash_reg_file = 1,
635 .unbound_nonreg_file = 1,
636 },
0463b6c5 637 [IORING_OP_POLL_ADD] = {
d3656344
JA		 638 .needs_file = 1,
639 .unbound_nonreg_file = 1,
640 },
0463b6c5
PB		 641 [IORING_OP_POLL_REMOVE] = {},
642 [IORING_OP_SYNC_FILE_RANGE] = {
d3656344
JA		 643 .needs_file = 1,
644 },
0463b6c5 645 [IORING_OP_SENDMSG] = {
d3656344
JA		 646 .async_ctx = 1,
647 .needs_mm = 1,
648 .needs_file = 1,
649 .unbound_nonreg_file = 1,
650 },
0463b6c5 651 [IORING_OP_RECVMSG] = {
d3656344
JA		 652 .async_ctx = 1,
653 .needs_mm = 1,
654 .needs_file = 1,
655 .unbound_nonreg_file = 1,
656 },
0463b6c5 657 [IORING_OP_TIMEOUT] = {
d3656344
JA		 658 .async_ctx = 1,
659 .needs_mm = 1,
660 },
0463b6c5
PB		 661 [IORING_OP_TIMEOUT_REMOVE] = {},
662 [IORING_OP_ACCEPT] = {
d3656344
JA		 663 .needs_mm = 1,
664 .needs_file = 1,
665 .unbound_nonreg_file = 1,
f86cd20c 666 .file_table = 1,
d3656344 667 },
0463b6c5
PB		 668 [IORING_OP_ASYNC_CANCEL] = {},
669 [IORING_OP_LINK_TIMEOUT] = {
d3656344
JA		 670 .async_ctx = 1,
671 .needs_mm = 1,
672 },
0463b6c5 673 [IORING_OP_CONNECT] = {
d3656344
JA		 674 .async_ctx = 1,
675 .needs_mm = 1,
676 .needs_file = 1,
677 .unbound_nonreg_file = 1,
678 },
0463b6c5 679 [IORING_OP_FALLOCATE] = {
d3656344
JA		 680 .needs_file = 1,
681 },
0463b6c5 682 [IORING_OP_OPENAT] = {
d3656344
JA		 683 .needs_file = 1,
684 .fd_non_neg = 1,
f86cd20c 685 .file_table = 1,
d3656344 686 },
0463b6c5 687 [IORING_OP_CLOSE] = {
d3656344 688 .needs_file = 1,
f86cd20c 689 .file_table = 1,
d3656344 690 },
0463b6c5 691 [IORING_OP_FILES_UPDATE] = {
d3656344 692 .needs_mm = 1,
f86cd20c 693 .file_table = 1,
d3656344 694 },
0463b6c5 695 [IORING_OP_STATX] = {
d3656344
JA		 696 .needs_mm = 1,
697 .needs_file = 1,
698 .fd_non_neg = 1,
699 },
0463b6c5 700 [IORING_OP_READ] = {
3a6820f2
JA		 701 .needs_mm = 1,
702 .needs_file = 1,
703 .unbound_nonreg_file = 1,
704 },
0463b6c5 705 [IORING_OP_WRITE] = {
3a6820f2
JA		 706 .needs_mm = 1,
707 .needs_file = 1,
708 .unbound_nonreg_file = 1,
709 },
0463b6c5 710 [IORING_OP_FADVISE] = {
4840e418
JA		 711 .needs_file = 1,
712 },
0463b6c5 713 [IORING_OP_MADVISE] = {
c1ca757b
JA		 714 .needs_mm = 1,
715 },
0463b6c5 716 [IORING_OP_SEND] = {
fddaface
JA		 717 .needs_mm = 1,
718 .needs_file = 1,
719 .unbound_nonreg_file = 1,
720 },
0463b6c5 721 [IORING_OP_RECV] = {
fddaface
JA		 722 .needs_mm = 1,
723 .needs_file = 1,
724 .unbound_nonreg_file = 1,
725 },
0463b6c5 726 [IORING_OP_OPENAT2] = {
cebdb986
JA		 727 .needs_file = 1,
728 .fd_non_neg = 1,
f86cd20c 729 .file_table = 1,
cebdb986 730 },
d3656344
JA		 731};
732
561fb04a 733static void io_wq_submit_work(struct io_wq_work **workptr);
78e19bbe 734static void io_cqring_fill_event(struct io_kiocb *req, long res);
ec9c02ad 735static void io_put_req(struct io_kiocb *req);
978db57e 736static void __io_double_put_req(struct io_kiocb *req);
94ae5e77
JA		 737static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req);
738static void io_queue_linked_timeout(struct io_kiocb *req);
05f3fb3c
JA		 739static int __io_sqe_files_update(struct io_ring_ctx *ctx,
740 struct io_uring_files_update *ip,
741 unsigned nr_args);
f86cd20c 742static int io_grab_files(struct io_kiocb *req);
de0617e4 743
2b188cc1
JA		 744static struct kmem_cache *req_cachep;
745
746static const struct file_operations io_uring_fops;
747
748struct sock *io_uring_get_socket(struct file *file)
749{
750#if defined(CONFIG_UNIX)
751 if (file->f_op == &io_uring_fops) {
752 struct io_ring_ctx *ctx = file->private_data;
753
754 return ctx->ring_sock->sk;
755 }
756#endif
757 return NULL;
758}
759EXPORT_SYMBOL(io_uring_get_socket);
760
761static void io_ring_ctx_ref_free(struct percpu_ref *ref)
762{
763 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
764
206aefde 765 complete(&ctx->completions[0]);
2b188cc1
JA		 766}
767
768static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
769{
770 struct io_ring_ctx *ctx;
78076bb6 771 int hash_bits;
2b188cc1
JA		 772
773 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
774 if (!ctx)
775 return NULL;
776
0ddf92e8
JA		 777 ctx->fallback_req = kmem_cache_alloc(req_cachep, GFP_KERNEL);
778 if (!ctx->fallback_req)
779 goto err;
780
206aefde
JA		 781 ctx->completions = kmalloc(2 * sizeof(struct completion), GFP_KERNEL);
782 if (!ctx->completions)
783 goto err;
784
78076bb6
JA		 785 /*
786 * Use 5 bits less than the max cq entries, that should give us around
787 * 32 entries per hash list if totally full and uniformly spread.
788 */
789 hash_bits = ilog2(p->cq_entries);
790 hash_bits -= 5;
791 if (hash_bits <= 0)
792 hash_bits = 1;
793 ctx->cancel_hash_bits = hash_bits;
794 ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
795 GFP_KERNEL);
796 if (!ctx->cancel_hash)
797 goto err;
798 __hash_init(ctx->cancel_hash, 1U << hash_bits);
799
21482896 800 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
206aefde
JA		 801 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
802 goto err;
2b188cc1
JA		 803
804 ctx->flags = p->flags;
805 init_waitqueue_head(&ctx->cq_wait);
1d7bb1d5 806 INIT_LIST_HEAD(&ctx->cq_overflow_list);
206aefde
JA		 807 init_completion(&ctx->completions[0]);
808 init_completion(&ctx->completions[1]);
071698e1 809 idr_init(&ctx->personality_idr);
2b188cc1
JA		 810 mutex_init(&ctx->uring_lock);
811 init_waitqueue_head(&ctx->wait);
812 spin_lock_init(&ctx->completion_lock);
e94f141b 813 init_llist_head(&ctx->poll_llist);
def596e9 814 INIT_LIST_HEAD(&ctx->poll_list);
de0617e4 815 INIT_LIST_HEAD(&ctx->defer_list);
5262f567 816 INIT_LIST_HEAD(&ctx->timeout_list);
fcb323cc
JA		 817 init_waitqueue_head(&ctx->inflight_wait);
818 spin_lock_init(&ctx->inflight_lock);
819 INIT_LIST_HEAD(&ctx->inflight_list);
2b188cc1 820 return ctx;
206aefde 821err:
0ddf92e8
JA		 822 if (ctx->fallback_req)
823 kmem_cache_free(req_cachep, ctx->fallback_req);
206aefde 824 kfree(ctx->completions);
78076bb6 825 kfree(ctx->cancel_hash);
206aefde
JA		 826 kfree(ctx);
827 return NULL;
2b188cc1
JA		 828}
829
9d858b21 830static inline bool __req_need_defer(struct io_kiocb *req)
7adf4eaf 831{
a197f664
JL		 832 struct io_ring_ctx *ctx = req->ctx;
833
498ccd9e
JA		 834 return req->sequence != ctx->cached_cq_tail + ctx->cached_sq_dropped
835 + atomic_read(&ctx->cached_cq_overflow);
7adf4eaf
JA		 836}
837
9d858b21 838static inline bool req_need_defer(struct io_kiocb *req)
de0617e4 839{
87987898 840 if (unlikely(req->flags & REQ_F_IO_DRAIN))
9d858b21 841 return __req_need_defer(req);
de0617e4 842
9d858b21 843 return false;
de0617e4
JA		 844}
845
7adf4eaf 846static struct io_kiocb *io_get_deferred_req(struct io_ring_ctx *ctx)
de0617e4
JA		 847{
848 struct io_kiocb *req;
849
7adf4eaf 850 req = list_first_entry_or_null(&ctx->defer_list, struct io_kiocb, list);
9d858b21 851 if (req && !req_need_defer(req)) {
de0617e4
JA		 852 list_del_init(&req->list);
853 return req;
854 }
855
856 return NULL;
857}
858
5262f567
JA		 859static struct io_kiocb *io_get_timeout_req(struct io_ring_ctx *ctx)
860{
7adf4eaf
JA		 861 struct io_kiocb *req;
862
863 req = list_first_entry_or_null(&ctx->timeout_list, struct io_kiocb, list);
93bd25bb
JA		 864 if (req) {
865 if (req->flags & REQ_F_TIMEOUT_NOSEQ)
866 return NULL;
fb4b3d3f 867 if (!__req_need_defer(req)) {
93bd25bb
JA		 868 list_del_init(&req->list);
869 return req;
870 }
7adf4eaf
JA		 871 }
872
873 return NULL;
5262f567
JA		 874}
875
de0617e4 876static void __io_commit_cqring(struct io_ring_ctx *ctx)
2b188cc1 877{
75b28aff 878 struct io_rings *rings = ctx->rings;
2b188cc1 879
07910158
PB		 880 /* order cqe stores with ring update */
881 smp_store_release(&rings->cq.tail, ctx->cached_cq_tail);
2b188cc1 882
07910158
PB		 883 if (wq_has_sleeper(&ctx->cq_wait)) {
884 wake_up_interruptible(&ctx->cq_wait);
885 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
2b188cc1
JA		 886 }
887}
888
cccf0ee8
JA		 889static inline void io_req_work_grab_env(struct io_kiocb *req,
890 const struct io_op_def *def)
891{
892 if (!req->work.mm && def->needs_mm) {
893 mmgrab(current->mm);
894 req->work.mm = current->mm;
895 }
896 if (!req->work.creds)
897 req->work.creds = get_current_cred();
898}
899
900static inline void io_req_work_drop_env(struct io_kiocb *req)
901{
902 if (req->work.mm) {
903 mmdrop(req->work.mm);
904 req->work.mm = NULL;
905 }
906 if (req->work.creds) {
907 put_cred(req->work.creds);
908 req->work.creds = NULL;
909 }
910}
911
94ae5e77
JA		 912static inline bool io_prep_async_work(struct io_kiocb *req,
913 struct io_kiocb **link)
18d9be1a 914{
d3656344 915 const struct io_op_def *def = &io_op_defs[req->opcode];
561fb04a 916 bool do_hashed = false;
54a91f3b 917
d3656344
JA		 918 if (req->flags & REQ_F_ISREG) {
919 if (def->hash_reg_file)
3529d8c2 920 do_hashed = true;
d3656344
JA		 921 } else {
922 if (def->unbound_nonreg_file)
3529d8c2 923 req->work.flags |= IO_WQ_WORK_UNBOUND;
54a91f3b 924 }
cccf0ee8
JA		 925
926 io_req_work_grab_env(req, def);
54a91f3b 927
94ae5e77 928 *link = io_prep_linked_timeout(req);
561fb04a
JA		 929 return do_hashed;
930}
931
a197f664 932static inline void io_queue_async_work(struct io_kiocb *req)
561fb04a 933{
a197f664 934 struct io_ring_ctx *ctx = req->ctx;
94ae5e77
JA		 935 struct io_kiocb *link;
936 bool do_hashed;
937
938 do_hashed = io_prep_async_work(req, &link);
561fb04a
JA		 939
940 trace_io_uring_queue_async_work(ctx, do_hashed, req, &req->work,
941 req->flags);
942 if (!do_hashed) {
943 io_wq_enqueue(ctx->io_wq, &req->work);
944 } else {
945 io_wq_enqueue_hashed(ctx->io_wq, &req->work,
946 file_inode(req->file));
947 }
94ae5e77
JA		 948
949 if (link)
950 io_queue_linked_timeout(link);
18d9be1a
JA		 951}
952
5262f567
JA		 953static void io_kill_timeout(struct io_kiocb *req)
954{
955 int ret;
956
2d28390a 957 ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
5262f567
JA		 958 if (ret != -1) {
959 atomic_inc(&req->ctx->cq_timeouts);
842f9612 960 list_del_init(&req->list);
78e19bbe 961 io_cqring_fill_event(req, 0);
ec9c02ad 962 io_put_req(req);
5262f567
JA		 963 }
964}
965
966static void io_kill_timeouts(struct io_ring_ctx *ctx)
967{
968 struct io_kiocb *req, *tmp;
969
970 spin_lock_irq(&ctx->completion_lock);
971 list_for_each_entry_safe(req, tmp, &ctx->timeout_list, list)
972 io_kill_timeout(req);
973 spin_unlock_irq(&ctx->completion_lock);
974}
975
de0617e4
JA		 976static void io_commit_cqring(struct io_ring_ctx *ctx)
977{
978 struct io_kiocb *req;
979
5262f567
JA		 980 while ((req = io_get_timeout_req(ctx)) != NULL)
981 io_kill_timeout(req);
982
de0617e4
JA		 983 __io_commit_cqring(ctx);
984
87987898 985 while ((req = io_get_deferred_req(ctx)) != NULL)
a197f664 986 io_queue_async_work(req);
de0617e4
JA		 987}
988
2b188cc1
JA		 989static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
990{
75b28aff 991 struct io_rings *rings = ctx->rings;
2b188cc1
JA		 992 unsigned tail;
993
994 tail = ctx->cached_cq_tail;
115e12e5
SB		 995 /*
996 * writes to the cq entry need to come after reading head; the
997 * control dependency is enough as we're using WRITE_ONCE to
998 * fill the cq entry
999 */
75b28aff 1000 if (tail - READ_ONCE(rings->cq.head) == rings->cq_ring_entries)
2b188cc1
JA		 1001 return NULL;
1002
1003 ctx->cached_cq_tail++;
75b28aff 1004 return &rings->cqes[tail & ctx->cq_mask];
2b188cc1
JA		 1005}
1006
f2842ab5
JA		 1007static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
1008{
1009 if (!ctx->eventfd_async)
1010 return true;
1011 return io_wq_current_is_worker() || in_interrupt();
1012}
1013
1d7bb1d5
JA		 1014static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1015{
1016 if (waitqueue_active(&ctx->wait))
1017 wake_up(&ctx->wait);
1018 if (waitqueue_active(&ctx->sqo_wait))
1019 wake_up(&ctx->sqo_wait);
f2842ab5 1020 if (ctx->cq_ev_fd && io_should_trigger_evfd(ctx))
1d7bb1d5
JA		 1021 eventfd_signal(ctx->cq_ev_fd, 1);
1022}
1023
c4a2ed72
JA		 1024/* Returns true if there are no backlogged entries after the flush */
1025static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
1d7bb1d5
JA		 1026{
1027 struct io_rings *rings = ctx->rings;
1028 struct io_uring_cqe *cqe;
1029 struct io_kiocb *req;
1030 unsigned long flags;
1031 LIST_HEAD(list);
1032
1033 if (!force) {
1034 if (list_empty_careful(&ctx->cq_overflow_list))
c4a2ed72 1035 return true;
1d7bb1d5
JA		 1036 if ((ctx->cached_cq_tail - READ_ONCE(rings->cq.head) ==
1037 rings->cq_ring_entries))
c4a2ed72 1038 return false;
1d7bb1d5
JA		 1039 }
1040
1041 spin_lock_irqsave(&ctx->completion_lock, flags);
1042
1043 /* if force is set, the ring is going away. always drop after that */
1044 if (force)
69b3e546 1045 ctx->cq_overflow_flushed = 1;
1d7bb1d5 1046
c4a2ed72 1047 cqe = NULL;
1d7bb1d5
JA		 1048 while (!list_empty(&ctx->cq_overflow_list)) {
1049 cqe = io_get_cqring(ctx);
1050 if (!cqe && !force)
1051 break;
1052
1053 req = list_first_entry(&ctx->cq_overflow_list, struct io_kiocb,
1054 list);
1055 list_move(&req->list, &list);
1056 if (cqe) {
1057 WRITE_ONCE(cqe->user_data, req->user_data);
1058 WRITE_ONCE(cqe->res, req->result);
1059 WRITE_ONCE(cqe->flags, 0);
1060 } else {
1061 WRITE_ONCE(ctx->rings->cq_overflow,
1062 atomic_inc_return(&ctx->cached_cq_overflow));
1063 }
1064 }
1065
1066 io_commit_cqring(ctx);
ad3eb2c8
JA		 1067 if (cqe) {
1068 clear_bit(0, &ctx->sq_check_overflow);
1069 clear_bit(0, &ctx->cq_check_overflow);
1070 }
1d7bb1d5
JA		 1071 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1072 io_cqring_ev_posted(ctx);
1073
1074 while (!list_empty(&list)) {
1075 req = list_first_entry(&list, struct io_kiocb, list);
1076 list_del(&req->list);
ec9c02ad 1077 io_put_req(req);
1d7bb1d5 1078 }
c4a2ed72
JA		 1079
1080 return cqe != NULL;
1d7bb1d5
JA		 1081}
1082
78e19bbe 1083static void io_cqring_fill_event(struct io_kiocb *req, long res)
2b188cc1 1084{
78e19bbe 1085 struct io_ring_ctx *ctx = req->ctx;
2b188cc1
JA		 1086 struct io_uring_cqe *cqe;
1087
78e19bbe 1088 trace_io_uring_complete(ctx, req->user_data, res);
51c3ff62 1089
2b188cc1
JA		 1090 /*
1091 * If we can't get a cq entry, userspace overflowed the
1092 * submission (by quite a lot). Increment the overflow count in
1093 * the ring.
1094 */
1095 cqe = io_get_cqring(ctx);
1d7bb1d5 1096 if (likely(cqe)) {
78e19bbe 1097 WRITE_ONCE(cqe->user_data, req->user_data);
2b188cc1 1098 WRITE_ONCE(cqe->res, res);
c71ffb67 1099 WRITE_ONCE(cqe->flags, 0);
1d7bb1d5 1100 } else if (ctx->cq_overflow_flushed) {
498ccd9e
JA		 1101 WRITE_ONCE(ctx->rings->cq_overflow,
1102 atomic_inc_return(&ctx->cached_cq_overflow));
1d7bb1d5 1103 } else {
ad3eb2c8
JA		 1104 if (list_empty(&ctx->cq_overflow_list)) {
1105 set_bit(0, &ctx->sq_check_overflow);
1106 set_bit(0, &ctx->cq_check_overflow);
1107 }
1d7bb1d5
JA		 1108 refcount_inc(&req->refs);
1109 req->result = res;
1110 list_add_tail(&req->list, &ctx->cq_overflow_list);
2b188cc1
JA		 1111 }
1112}
1113
78e19bbe 1114static void io_cqring_add_event(struct io_kiocb *req, long res)
2b188cc1 1115{
78e19bbe 1116 struct io_ring_ctx *ctx = req->ctx;
2b188cc1
JA		 1117 unsigned long flags;
1118
1119 spin_lock_irqsave(&ctx->completion_lock, flags);
78e19bbe 1120 io_cqring_fill_event(req, res);
2b188cc1
JA		 1121 io_commit_cqring(ctx);
1122 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1123
8c838788 1124 io_cqring_ev_posted(ctx);
2b188cc1
JA		 1125}
1126
0ddf92e8
JA		 1127static inline bool io_is_fallback_req(struct io_kiocb *req)
1128{
1129 return req == (struct io_kiocb *)
1130 ((unsigned long) req->ctx->fallback_req & ~1UL);
1131}
1132
1133static struct io_kiocb *io_get_fallback_req(struct io_ring_ctx *ctx)
1134{
1135 struct io_kiocb *req;
1136
1137 req = ctx->fallback_req;
1138 if (!test_and_set_bit_lock(0, (unsigned long *) ctx->fallback_req))
1139 return req;
1140
1141 return NULL;
1142}
1143
2579f913
JA		 1144static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
1145 struct io_submit_state *state)
2b188cc1 1146{
fd6fab2c 1147 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
2b188cc1
JA		 1148 struct io_kiocb *req;
1149
2579f913 1150 if (!state) {
fd6fab2c 1151 req = kmem_cache_alloc(req_cachep, gfp);
2579f913 1152 if (unlikely(!req))
0ddf92e8 1153 goto fallback;
2579f913
JA		 1154 } else if (!state->free_reqs) {
1155 size_t sz;
1156 int ret;
1157
1158 sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
fd6fab2c
JA		 1159 ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs);
1160
1161 /*
1162 * Bulk alloc is all-or-nothing. If we fail to get a batch,
1163 * retry single alloc to be on the safe side.
1164 */
1165 if (unlikely(ret <= 0)) {
1166 state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
1167 if (!state->reqs[0])
0ddf92e8 1168 goto fallback;
fd6fab2c
JA		 1169 ret = 1;
1170 }
2579f913
JA		 1171 state->free_reqs = ret - 1;
1172 state->cur_req = 1;
1173 req = state->reqs[0];
1174 } else {
1175 req = state->reqs[state->cur_req];
1176 state->free_reqs--;
1177 state->cur_req++;
2b188cc1
JA		 1178 }
1179
0ddf92e8 1180got_it:
1a6b74fc 1181 req->io = NULL;
60c112b0 1182 req->file = NULL;
2579f913
JA		 1183 req->ctx = ctx;
1184 req->flags = 0;
e65ef56d
JA		 1185 /* one is dropped after submission, the other at completion */
1186 refcount_set(&req->refs, 2);
9e645e11 1187 req->result = 0;
561fb04a 1188 INIT_IO_WORK(&req->work, io_wq_submit_work);
2579f913 1189 return req;
0ddf92e8
JA		 1190fallback:
1191 req = io_get_fallback_req(ctx);
1192 if (req)
1193 goto got_it;
6805b32e 1194 percpu_ref_put(&ctx->refs);
2b188cc1
JA		 1195 return NULL;
1196}
1197
2b85edfc
PB		 1198static void __io_req_do_free(struct io_kiocb *req)
1199{
1200 if (likely(!io_is_fallback_req(req)))
1201 kmem_cache_free(req_cachep, req);
1202 else
1203 clear_bit_unlock(0, (unsigned long *) req->ctx->fallback_req);
1204}
1205
c6ca97b3 1206static void __io_req_aux_free(struct io_kiocb *req)
2b188cc1 1207{
fcb323cc
JA		 1208 struct io_ring_ctx *ctx = req->ctx;
1209
96fd84d8 1210 kfree(req->io);
05f3fb3c
JA		 1211 if (req->file) {
1212 if (req->flags & REQ_F_FIXED_FILE)
1213 percpu_ref_put(&ctx->file_data->refs);
1214 else
1215 fput(req->file);
1216 }
cccf0ee8
JA		 1217
1218 io_req_work_drop_env(req);
c6ca97b3
JA		 1219}
1220
1221static void __io_free_req(struct io_kiocb *req)
1222{
1223 __io_req_aux_free(req);
1224
fcb323cc 1225 if (req->flags & REQ_F_INFLIGHT) {
c6ca97b3 1226 struct io_ring_ctx *ctx = req->ctx;
fcb323cc
JA		 1227 unsigned long flags;
1228
1229 spin_lock_irqsave(&ctx->inflight_lock, flags);
1230 list_del(&req->inflight_entry);
1231 if (waitqueue_active(&ctx->inflight_wait))
1232 wake_up(&ctx->inflight_wait);
1233 spin_unlock_irqrestore(&ctx->inflight_lock, flags);
1234 }
2b85edfc
PB		 1235
1236 percpu_ref_put(&req->ctx->refs);
1237 __io_req_do_free(req);
e65ef56d
JA		 1238}
1239
c6ca97b3
JA		 1240struct req_batch {
1241 void *reqs[IO_IOPOLL_BATCH];
1242 int to_free;
1243 int need_iter;
1244};
1245
1246static void io_free_req_many(struct io_ring_ctx *ctx, struct req_batch *rb)
1247{
10fef4be
JA		 1248 int fixed_refs = rb->to_free;
1249
c6ca97b3
JA		 1250 if (!rb->to_free)
1251 return;
1252 if (rb->need_iter) {
1253 int i, inflight = 0;
1254 unsigned long flags;
1255
10fef4be 1256 fixed_refs = 0;
c6ca97b3
JA		 1257 for (i = 0; i < rb->to_free; i++) {
1258 struct io_kiocb *req = rb->reqs[i];
1259
10fef4be 1260 if (req->flags & REQ_F_FIXED_FILE) {
c6ca97b3 1261 req->file = NULL;
10fef4be
JA		 1262 fixed_refs++;
1263 }
c6ca97b3
JA		 1264 if (req->flags & REQ_F_INFLIGHT)
1265 inflight++;
c6ca97b3
JA		 1266 __io_req_aux_free(req);
1267 }
1268 if (!inflight)
1269 goto do_free;
1270
1271 spin_lock_irqsave(&ctx->inflight_lock, flags);
1272 for (i = 0; i < rb->to_free; i++) {
1273 struct io_kiocb *req = rb->reqs[i];
1274
10fef4be 1275 if (req->flags & REQ_F_INFLIGHT) {
c6ca97b3
JA		 1276 list_del(&req->inflight_entry);
1277 if (!--inflight)
1278 break;
1279 }
1280 }
1281 spin_unlock_irqrestore(&ctx->inflight_lock, flags);
1282
1283 if (waitqueue_active(&ctx->inflight_wait))
1284 wake_up(&ctx->inflight_wait);
1285 }
1286do_free:
1287 kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs);
10fef4be
JA		 1288 if (fixed_refs)
1289 percpu_ref_put_many(&ctx->file_data->refs, fixed_refs);
c6ca97b3 1290 percpu_ref_put_many(&ctx->refs, rb->to_free);
c6ca97b3
JA		 1291 rb->to_free = rb->need_iter = 0;
1292}
1293
a197f664 1294static bool io_link_cancel_timeout(struct io_kiocb *req)
2665abfd 1295{
a197f664 1296 struct io_ring_ctx *ctx = req->ctx;
2665abfd
JA		 1297 int ret;
1298
2d28390a 1299 ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
2665abfd 1300 if (ret != -1) {
78e19bbe 1301 io_cqring_fill_event(req, -ECANCELED);
2665abfd
JA		 1302 io_commit_cqring(ctx);
1303 req->flags &= ~REQ_F_LINK;
ec9c02ad 1304 io_put_req(req);
2665abfd
JA		 1305 return true;
1306 }
1307
1308 return false;
e65ef56d
JA		 1309}
1310
ba816ad6 1311static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
9e645e11 1312{
2665abfd 1313 struct io_ring_ctx *ctx = req->ctx;
2665abfd 1314 bool wake_ev = false;
9e645e11 1315
4d7dd462
JA		 1316 /* Already got next link */
1317 if (req->flags & REQ_F_LINK_NEXT)
1318 return;
1319
9e645e11
JA		 1320 /*
1321 * The list should never be empty when we are called here. But could
1322 * potentially happen if the chain is messed up, check to be on the
1323 * safe side.
1324 */
4493233e
PB		 1325 while (!list_empty(&req->link_list)) {
1326 struct io_kiocb *nxt = list_first_entry(&req->link_list,
1327 struct io_kiocb, link_list);
94ae5e77 1328
4493233e
PB		 1329 if (unlikely((req->flags & REQ_F_LINK_TIMEOUT) &&
1330 (nxt->flags & REQ_F_TIMEOUT))) {
1331 list_del_init(&nxt->link_list);
94ae5e77 1332 wake_ev |= io_link_cancel_timeout(nxt);
94ae5e77
JA		 1333 req->flags &= ~REQ_F_LINK_TIMEOUT;
1334 continue;
1335 }
9e645e11 1336
4493233e
PB		 1337 list_del_init(&req->link_list);
1338 if (!list_empty(&nxt->link_list))
1339 nxt->flags |= REQ_F_LINK;
b18fdf71 1340 *nxtptr = nxt;
94ae5e77 1341 break;
9e645e11 1342 }
2665abfd 1343
4d7dd462 1344 req->flags |= REQ_F_LINK_NEXT;
2665abfd
JA		 1345 if (wake_ev)
1346 io_cqring_ev_posted(ctx);
9e645e11
JA		 1347}
1348
1349/*
1350 * Called if REQ_F_LINK is set, and we fail the head request
1351 */
1352static void io_fail_links(struct io_kiocb *req)
1353{
2665abfd 1354 struct io_ring_ctx *ctx = req->ctx;
2665abfd
JA		 1355 unsigned long flags;
1356
1357 spin_lock_irqsave(&ctx->completion_lock, flags);
9e645e11
JA		 1358
1359 while (!list_empty(&req->link_list)) {
4493233e
PB		 1360 struct io_kiocb *link = list_first_entry(&req->link_list,
1361 struct io_kiocb, link_list);
9e645e11 1362
4493233e 1363 list_del_init(&link->link_list);
c826bd7a 1364 trace_io_uring_fail_link(req, link);
2665abfd
JA		 1365
1366 if ((req->flags & REQ_F_LINK_TIMEOUT) &&
d625c6ee 1367 link->opcode == IORING_OP_LINK_TIMEOUT) {
a197f664 1368 io_link_cancel_timeout(link);
2665abfd 1369 } else {
78e19bbe 1370 io_cqring_fill_event(link, -ECANCELED);
978db57e 1371 __io_double_put_req(link);
2665abfd 1372 }
5d960724 1373 req->flags &= ~REQ_F_LINK_TIMEOUT;
9e645e11 1374 }
2665abfd
JA		 1375
1376 io_commit_cqring(ctx);
1377 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1378 io_cqring_ev_posted(ctx);
9e645e11
JA		 1379}
1380
4d7dd462 1381static void io_req_find_next(struct io_kiocb *req, struct io_kiocb **nxt)
9e645e11 1382{
4d7dd462 1383 if (likely(!(req->flags & REQ_F_LINK)))
2665abfd 1384 return;
2665abfd 1385
9e645e11
JA		 1386 /*
1387 * If LINK is set, we have dependent requests in this chain. If we
1388 * didn't fail this request, queue the first one up, moving any other
1389 * dependencies to the next request. In case of failure, fail the rest
1390 * of the chain.
1391 */
2665abfd
JA		 1392 if (req->flags & REQ_F_FAIL_LINK) {
1393 io_fail_links(req);
7c9e7f0f
JA		 1394 } else if ((req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_COMP_LOCKED)) ==
1395 REQ_F_LINK_TIMEOUT) {
2665abfd
JA		 1396 struct io_ring_ctx *ctx = req->ctx;
1397 unsigned long flags;
1398
1399 /*
1400 * If this is a timeout link, we could be racing with the
1401 * timeout timer. Grab the completion lock for this case to
7c9e7f0f 1402 * protect against that.
2665abfd
JA		 1403 */
1404 spin_lock_irqsave(&ctx->completion_lock, flags);
1405 io_req_link_next(req, nxt);
1406 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1407 } else {
1408 io_req_link_next(req, nxt);
9e645e11 1409 }
4d7dd462 1410}
9e645e11 1411
c69f8dbe
JL		 1412static void io_free_req(struct io_kiocb *req)
1413{
944e58bf
PB		 1414 struct io_kiocb *nxt = NULL;
1415
1416 io_req_find_next(req, &nxt);
70cf9f32 1417 __io_free_req(req);
944e58bf
PB		 1418
1419 if (nxt)
1420 io_queue_async_work(nxt);
c69f8dbe
JL		 1421}
1422
ba816ad6
JA		 1423/*
1424 * Drop reference to request, return next in chain (if there is one) if this
1425 * was the last reference to this request.
1426 */
f9bd67f6 1427__attribute__((nonnull))
ec9c02ad 1428static void io_put_req_find_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
e65ef56d 1429{
f9bd67f6 1430 io_req_find_next(req, nxtptr);
4d7dd462 1431
e65ef56d 1432 if (refcount_dec_and_test(&req->refs))
4d7dd462 1433 __io_free_req(req);
2b188cc1
JA		 1434}
1435
e65ef56d
JA		 1436static void io_put_req(struct io_kiocb *req)
1437{
1438 if (refcount_dec_and_test(&req->refs))
1439 io_free_req(req);
2b188cc1
JA		 1440}
1441
978db57e
JA		 1442/*
1443 * Must only be used if we don't need to care about links, usually from
1444 * within the completion handling itself.
1445 */
1446static void __io_double_put_req(struct io_kiocb *req)
78e19bbe
JA		 1447{
1448 /* drop both submit and complete references */
1449 if (refcount_sub_and_test(2, &req->refs))
1450 __io_free_req(req);
1451}
1452
978db57e
JA		 1453static void io_double_put_req(struct io_kiocb *req)
1454{
1455 /* drop both submit and complete references */
1456 if (refcount_sub_and_test(2, &req->refs))
1457 io_free_req(req);
1458}
1459
1d7bb1d5 1460static unsigned io_cqring_events(struct io_ring_ctx *ctx, bool noflush)
a3a0e43f 1461{
84f97dc2
JA		 1462 struct io_rings *rings = ctx->rings;
1463
ad3eb2c8
JA		 1464 if (test_bit(0, &ctx->cq_check_overflow)) {
1465 /*
1466 * noflush == true is from the waitqueue handler, just ensure
1467 * we wake up the task, and the next invocation will flush the
1468 * entries. We cannot safely to it from here.
1469 */
1470 if (noflush && !list_empty(&ctx->cq_overflow_list))
1471 return -1U;
1d7bb1d5 1472
ad3eb2c8
JA		 1473 io_cqring_overflow_flush(ctx, false);
1474 }
1d7bb1d5 1475
a3a0e43f
JA		 1476 /* See comment at the top of this file */
1477 smp_rmb();
ad3eb2c8 1478 return ctx->cached_cq_tail - READ_ONCE(rings->cq.head);
a3a0e43f
JA		 1479}
1480
fb5ccc98
PB		 1481static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
1482{
1483 struct io_rings *rings = ctx->rings;
1484
1485 /* make sure SQ entry isn't read before tail */
1486 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
1487}
1488
8237e045 1489static inline bool io_req_multi_free(struct req_batch *rb, struct io_kiocb *req)
e94f141b 1490{
c6ca97b3
JA		 1491 if ((req->flags & REQ_F_LINK) || io_is_fallback_req(req))
1492 return false;
e94f141b 1493
c6ca97b3
JA		 1494 if (!(req->flags & REQ_F_FIXED_FILE) || req->io)
1495 rb->need_iter++;
1496
1497 rb->reqs[rb->to_free++] = req;
1498 if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs)))
1499 io_free_req_many(req->ctx, rb);
1500 return true;
e94f141b
JA		 1501}
1502
def596e9
JA		 1503/*
1504 * Find and free completed poll iocbs
1505 */
1506static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
1507 struct list_head *done)
1508{
8237e045 1509 struct req_batch rb;
def596e9 1510 struct io_kiocb *req;
def596e9 1511
c6ca97b3 1512 rb.to_free = rb.need_iter = 0;
def596e9
JA		 1513 while (!list_empty(done)) {
1514 req = list_first_entry(done, struct io_kiocb, list);
1515 list_del(&req->list);
1516
78e19bbe 1517 io_cqring_fill_event(req, req->result);
def596e9
JA		 1518 (*nr_events)++;
1519
8237e045
JA		 1520 if (refcount_dec_and_test(&req->refs) &&
1521 !io_req_multi_free(&rb, req))
1522 io_free_req(req);
def596e9 1523 }
def596e9 1524
09bb8394 1525 io_commit_cqring(ctx);
8237e045 1526 io_free_req_many(ctx, &rb);
def596e9
JA		 1527}
1528
1529static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
1530 long min)
1531{
1532 struct io_kiocb *req, *tmp;
1533 LIST_HEAD(done);
1534 bool spin;
1535 int ret;
1536
1537 /*
1538 * Only spin for completions if we don't have multiple devices hanging
1539 * off our complete list, and we're under the requested amount.
1540 */
1541 spin = !ctx->poll_multi_file && *nr_events < min;
1542
1543 ret = 0;
1544 list_for_each_entry_safe(req, tmp, &ctx->poll_list, list) {
9adbd45d 1545 struct kiocb *kiocb = &req->rw.kiocb;
def596e9
JA		 1546
1547 /*
1548 * Move completed entries to our local list. If we find a
1549 * request that requires polling, break out and complete
1550 * the done list first, if we have entries there.
1551 */
1552 if (req->flags & REQ_F_IOPOLL_COMPLETED) {
1553 list_move_tail(&req->list, &done);
1554 continue;
1555 }
1556 if (!list_empty(&done))
1557 break;
1558
1559 ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
1560 if (ret < 0)
1561 break;
1562
1563 if (ret && spin)
1564 spin = false;
1565 ret = 0;
1566 }
1567
1568 if (!list_empty(&done))
1569 io_iopoll_complete(ctx, nr_events, &done);
1570
1571 return ret;
1572}
1573
1574/*
d195a66e 1575 * Poll for a minimum of 'min' events. Note that if min == 0 we consider that a
def596e9
JA		 1576 * non-spinning poll check - we'll still enter the driver poll loop, but only
1577 * as a non-spinning completion check.
1578 */
1579static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
1580 long min)
1581{
08f5439f 1582 while (!list_empty(&ctx->poll_list) && !need_resched()) {
def596e9
JA		 1583 int ret;
1584
1585 ret = io_do_iopoll(ctx, nr_events, min);
1586 if (ret < 0)
1587 return ret;
1588 if (!min || *nr_events >= min)
1589 return 0;
1590 }
1591
1592 return 1;
1593}
1594
1595/*
1596 * We can't just wait for polled events to come to us, we have to actively
1597 * find and complete them.
1598 */
1599static void io_iopoll_reap_events(struct io_ring_ctx *ctx)
1600{
1601 if (!(ctx->flags & IORING_SETUP_IOPOLL))
1602 return;
1603
1604 mutex_lock(&ctx->uring_lock);
1605 while (!list_empty(&ctx->poll_list)) {
1606 unsigned int nr_events = 0;
1607
1608 io_iopoll_getevents(ctx, &nr_events, 1);
08f5439f
JA		 1609
1610 /*
1611 * Ensure we allow local-to-the-cpu processing to take place,
1612 * in this case we need to ensure that we reap all events.
1613 */
1614 cond_resched();
def596e9
JA		 1615 }
1616 mutex_unlock(&ctx->uring_lock);
1617}
1618
2b2ed975
JA		 1619static int __io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
1620 long min)
def596e9 1621{
2b2ed975 1622 int iters = 0, ret = 0;
500f9fba 1623
def596e9
JA		 1624 do {
1625 int tmin = 0;
1626
a3a0e43f
JA		 1627 /*
1628 * Don't enter poll loop if we already have events pending.
1629 * If we do, we can potentially be spinning for commands that
1630 * already triggered a CQE (eg in error).
1631 */
1d7bb1d5 1632 if (io_cqring_events(ctx, false))
a3a0e43f
JA		 1633 break;
1634
500f9fba
JA		 1635 /*
1636 * If a submit got punted to a workqueue, we can have the
1637 * application entering polling for a command before it gets
1638 * issued. That app will hold the uring_lock for the duration
1639 * of the poll right here, so we need to take a breather every
1640 * now and then to ensure that the issue has a chance to add
1641 * the poll to the issued list. Otherwise we can spin here
1642 * forever, while the workqueue is stuck trying to acquire the
1643 * very same mutex.
1644 */
1645 if (!(++iters & 7)) {
1646 mutex_unlock(&ctx->uring_lock);
1647 mutex_lock(&ctx->uring_lock);
1648 }
1649
def596e9
JA		 1650 if (*nr_events < min)
1651 tmin = min - *nr_events;
1652
1653 ret = io_iopoll_getevents(ctx, nr_events, tmin);
1654 if (ret <= 0)
1655 break;
1656 ret = 0;
1657 } while (min && !*nr_events && !need_resched());
1658
2b2ed975
JA		 1659 return ret;
1660}
1661
1662static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
1663 long min)
1664{
1665 int ret;
1666
1667 /*
1668 * We disallow the app entering submit/complete with polling, but we
1669 * still need to lock the ring to prevent racing with polled issue
1670 * that got punted to a workqueue.
1671 */
1672 mutex_lock(&ctx->uring_lock);
1673 ret = __io_iopoll_check(ctx, nr_events, min);
500f9fba 1674 mutex_unlock(&ctx->uring_lock);
def596e9
JA		 1675 return ret;
1676}
1677
491381ce 1678static void kiocb_end_write(struct io_kiocb *req)
2b188cc1 1679{
491381ce
JA		 1680 /*
1681 * Tell lockdep we inherited freeze protection from submission
1682 * thread.
1683 */
1684 if (req->flags & REQ_F_ISREG) {
1685 struct inode *inode = file_inode(req->file);
2b188cc1 1686
491381ce 1687 __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
2b188cc1 1688 }
491381ce 1689 file_end_write(req->file);
2b188cc1
JA		 1690}
1691
4e88d6e7
JA		 1692static inline void req_set_fail_links(struct io_kiocb *req)
1693{
1694 if ((req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) == REQ_F_LINK)
1695 req->flags |= REQ_F_FAIL_LINK;
1696}
1697
ba816ad6 1698static void io_complete_rw_common(struct kiocb *kiocb, long res)
2b188cc1 1699{
9adbd45d 1700 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
2b188cc1 1701
491381ce
JA		 1702 if (kiocb->ki_flags & IOCB_WRITE)
1703 kiocb_end_write(req);
2b188cc1 1704
4e88d6e7
JA		 1705 if (res != req->result)
1706 req_set_fail_links(req);
78e19bbe 1707 io_cqring_add_event(req, res);
ba816ad6
JA		 1708}
1709
1710static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
1711{
9adbd45d 1712 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
ba816ad6
JA		 1713
1714 io_complete_rw_common(kiocb, res);
e65ef56d 1715 io_put_req(req);
2b188cc1
JA		 1716}
1717
ba816ad6
JA		 1718static struct io_kiocb *__io_complete_rw(struct kiocb *kiocb, long res)
1719{
9adbd45d 1720 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
ec9c02ad 1721 struct io_kiocb *nxt = NULL;
ba816ad6
JA		 1722
1723 io_complete_rw_common(kiocb, res);
ec9c02ad
JL		 1724 io_put_req_find_next(req, &nxt);
1725
1726 return nxt;
2b188cc1
JA		 1727}
1728
def596e9
JA		 1729static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
1730{
9adbd45d 1731 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
def596e9 1732
491381ce
JA		 1733 if (kiocb->ki_flags & IOCB_WRITE)
1734 kiocb_end_write(req);
def596e9 1735
4e88d6e7
JA		 1736 if (res != req->result)
1737 req_set_fail_links(req);
9e645e11 1738 req->result = res;
def596e9
JA		 1739 if (res != -EAGAIN)
1740 req->flags |= REQ_F_IOPOLL_COMPLETED;
1741}
1742
1743/*
1744 * After the iocb has been issued, it's safe to be found on the poll list.
1745 * Adding the kiocb to the list AFTER submission ensures that we don't
1746 * find it from a io_iopoll_getevents() thread before the issuer is done
1747 * accessing the kiocb cookie.
1748 */
1749static void io_iopoll_req_issued(struct io_kiocb *req)
1750{
1751 struct io_ring_ctx *ctx = req->ctx;
1752
1753 /*
1754 * Track whether we have multiple files in our lists. This will impact
1755 * how we do polling eventually, not spinning if we're on potentially
1756 * different devices.
1757 */
1758 if (list_empty(&ctx->poll_list)) {
1759 ctx->poll_multi_file = false;
1760 } else if (!ctx->poll_multi_file) {
1761 struct io_kiocb *list_req;
1762
1763 list_req = list_first_entry(&ctx->poll_list, struct io_kiocb,
1764 list);
9adbd45d 1765 if (list_req->file != req->file)
def596e9
JA		 1766 ctx->poll_multi_file = true;
1767 }
1768
1769 /*
1770 * For fast devices, IO may have already completed. If it has, add
1771 * it to the front so we find it first.
1772 */
1773 if (req->flags & REQ_F_IOPOLL_COMPLETED)
1774 list_add(&req->list, &ctx->poll_list);
1775 else
1776 list_add_tail(&req->list, &ctx->poll_list);
1777}
1778
3d6770fb 1779static void io_file_put(struct io_submit_state *state)
9a56a232 1780{
3d6770fb 1781 if (state->file) {
9a56a232
JA		 1782 int diff = state->has_refs - state->used_refs;
1783
1784 if (diff)
1785 fput_many(state->file, diff);
1786 state->file = NULL;
1787 }
1788}
1789
1790/*
1791 * Get as many references to a file as we have IOs left in this submission,
1792 * assuming most submissions are for one file, or at least that each file
1793 * has more than one submission.
1794 */
1795static struct file *io_file_get(struct io_submit_state *state, int fd)
1796{
1797 if (!state)
1798 return fget(fd);
1799
1800 if (state->file) {
1801 if (state->fd == fd) {
1802 state->used_refs++;
1803 state->ios_left--;
1804 return state->file;
1805 }
3d6770fb 1806 io_file_put(state);
9a56a232
JA		 1807 }
1808 state->file = fget_many(fd, state->ios_left);
1809 if (!state->file)
1810 return NULL;
1811
1812 state->fd = fd;
1813 state->has_refs = state->ios_left;
1814 state->used_refs = 1;
1815 state->ios_left--;
1816 return state->file;
1817}
1818
2b188cc1
JA		 1819/*
1820 * If we tracked the file through the SCM inflight mechanism, we could support
1821 * any file. For now, just ensure that anything potentially problematic is done
1822 * inline.
1823 */
1824static bool io_file_supports_async(struct file *file)
1825{
1826 umode_t mode = file_inode(file)->i_mode;
1827
10d59345 1828 if (S_ISBLK(mode) || S_ISCHR(mode) || S_ISSOCK(mode))
2b188cc1
JA		 1829 return true;
1830 if (S_ISREG(mode) && file->f_op != &io_uring_fops)
1831 return true;
1832
1833 return false;
1834}
1835
3529d8c2
JA		 1836static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
1837 bool force_nonblock)
2b188cc1 1838{
def596e9 1839 struct io_ring_ctx *ctx = req->ctx;
9adbd45d 1840 struct kiocb *kiocb = &req->rw.kiocb;
09bb8394
JA		 1841 unsigned ioprio;
1842 int ret;
2b188cc1 1843
09bb8394
JA		 1844 if (!req->file)
1845 return -EBADF;
2b188cc1 1846
491381ce
JA		 1847 if (S_ISREG(file_inode(req->file)->i_mode))
1848 req->flags |= REQ_F_ISREG;
1849
2b188cc1 1850 kiocb->ki_pos = READ_ONCE(sqe->off);
ba04291e
JA		 1851 if (kiocb->ki_pos == -1 && !(req->file->f_mode & FMODE_STREAM)) {
1852 req->flags |= REQ_F_CUR_POS;
1853 kiocb->ki_pos = req->file->f_pos;
1854 }
2b188cc1
JA		 1855 kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
1856 kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
1857
1858 ioprio = READ_ONCE(sqe->ioprio);
1859 if (ioprio) {
1860 ret = ioprio_check_cap(ioprio);
1861 if (ret)
09bb8394 1862 return ret;
2b188cc1
JA		 1863
1864 kiocb->ki_ioprio = ioprio;
1865 } else
1866 kiocb->ki_ioprio = get_current_ioprio();
1867
1868 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
1869 if (unlikely(ret))
09bb8394 1870 return ret;
8449eeda
SB		 1871
1872 /* don't allow async punt if RWF_NOWAIT was requested */
491381ce
JA		 1873 if ((kiocb->ki_flags & IOCB_NOWAIT) ||
1874 (req->file->f_flags & O_NONBLOCK))
8449eeda
SB		 1875 req->flags |= REQ_F_NOWAIT;
1876
1877 if (force_nonblock)
2b188cc1 1878 kiocb->ki_flags |= IOCB_NOWAIT;
8449eeda 1879
def596e9 1880 if (ctx->flags & IORING_SETUP_IOPOLL) {
def596e9
JA		 1881 if (!(kiocb->ki_flags & IOCB_DIRECT) ||
1882 !kiocb->ki_filp->f_op->iopoll)
09bb8394 1883 return -EOPNOTSUPP;
2b188cc1 1884
def596e9
JA		 1885 kiocb->ki_flags |= IOCB_HIPRI;
1886 kiocb->ki_complete = io_complete_rw_iopoll;
6873e0bd 1887 req->result = 0;
def596e9 1888 } else {
09bb8394
JA		 1889 if (kiocb->ki_flags & IOCB_HIPRI)
1890 return -EINVAL;
def596e9
JA		 1891 kiocb->ki_complete = io_complete_rw;
1892 }
9adbd45d 1893
3529d8c2
JA		 1894 req->rw.addr = READ_ONCE(sqe->addr);
1895 req->rw.len = READ_ONCE(sqe->len);
9adbd45d
JA		 1896 /* we own ->private, reuse it for the buffer index */
1897 req->rw.kiocb.private = (void *) (unsigned long)
3529d8c2 1898 READ_ONCE(sqe->buf_index);
2b188cc1 1899 return 0;
2b188cc1
JA		 1900}
1901
1902static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
1903{
1904 switch (ret) {
1905 case -EIOCBQUEUED:
1906 break;
1907 case -ERESTARTSYS:
1908 case -ERESTARTNOINTR:
1909 case -ERESTARTNOHAND:
1910 case -ERESTART_RESTARTBLOCK:
1911 /*
1912 * We can't just restart the syscall, since previously
1913 * submitted sqes may already be in progress. Just fail this
1914 * IO with EINTR.
1915 */
1916 ret = -EINTR;
1917 /* fall through */
1918 default:
1919 kiocb->ki_complete(kiocb, ret, 0);
1920 }
1921}
1922
ba816ad6
JA		 1923static void kiocb_done(struct kiocb *kiocb, ssize_t ret, struct io_kiocb **nxt,
1924 bool in_async)
1925{
ba04291e
JA		 1926 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
1927
1928 if (req->flags & REQ_F_CUR_POS)
1929 req->file->f_pos = kiocb->ki_pos;
f9bd67f6 1930 if (in_async && ret >= 0 && kiocb->ki_complete == io_complete_rw)
ba816ad6
JA		 1931 *nxt = __io_complete_rw(kiocb, ret);
1932 else
1933 io_rw_done(kiocb, ret);
1934}
1935
9adbd45d 1936static ssize_t io_import_fixed(struct io_kiocb *req, int rw,
7d009165 1937 struct iov_iter *iter)
edafccee 1938{
9adbd45d
JA		 1939 struct io_ring_ctx *ctx = req->ctx;
1940 size_t len = req->rw.len;
edafccee
JA		 1941 struct io_mapped_ubuf *imu;
1942 unsigned index, buf_index;
1943 size_t offset;
1944 u64 buf_addr;
1945
1946 /* attempt to use fixed buffers without having provided iovecs */
1947 if (unlikely(!ctx->user_bufs))
1948 return -EFAULT;
1949
9adbd45d 1950 buf_index = (unsigned long) req->rw.kiocb.private;
edafccee
JA		 1951 if (unlikely(buf_index >= ctx->nr_user_bufs))
1952 return -EFAULT;
1953
1954 index = array_index_nospec(buf_index, ctx->nr_user_bufs);
1955 imu = &ctx->user_bufs[index];
9adbd45d 1956 buf_addr = req->rw.addr;
edafccee
JA		 1957
1958 /* overflow */
1959 if (buf_addr + len < buf_addr)
1960 return -EFAULT;
1961 /* not inside the mapped region */
1962 if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
1963 return -EFAULT;
1964
1965 /*
1966 * May not be a start of buffer, set size appropriately
1967 * and advance us to the beginning.
1968 */
1969 offset = buf_addr - imu->ubuf;
1970 iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
bd11b3a3
JA		 1971
1972 if (offset) {
1973 /*
1974 * Don't use iov_iter_advance() here, as it's really slow for
1975 * using the latter parts of a big fixed buffer - it iterates
1976 * over each segment manually. We can cheat a bit here, because
1977 * we know that:
1978 *
1979 * 1) it's a BVEC iter, we set it up
1980 * 2) all bvecs are PAGE_SIZE in size, except potentially the
1981 * first and last bvec
1982 *
1983 * So just find our index, and adjust the iterator afterwards.
1984 * If the offset is within the first bvec (or the whole first
1985 * bvec, just use iov_iter_advance(). This makes it easier
1986 * since we can just skip the first segment, which may not
1987 * be PAGE_SIZE aligned.
1988 */
1989 const struct bio_vec *bvec = imu->bvec;
1990
1991 if (offset <= bvec->bv_len) {
1992 iov_iter_advance(iter, offset);
1993 } else {
1994 unsigned long seg_skip;
1995
1996 /* skip first vec */
1997 offset -= bvec->bv_len;
1998 seg_skip = 1 + (offset >> PAGE_SHIFT);
1999
2000 iter->bvec = bvec + seg_skip;
2001 iter->nr_segs -= seg_skip;
99c79f66 2002 iter->count -= bvec->bv_len + offset;
bd11b3a3 2003 iter->iov_offset = offset & ~PAGE_MASK;
bd11b3a3
JA		 2004 }
2005 }
2006
5e559561 2007 return len;
edafccee
JA		 2008}
2009
cf6fd4bd
PB		 2010static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
2011 struct iovec **iovec, struct iov_iter *iter)
2b188cc1 2012{
9adbd45d
JA		 2013 void __user *buf = u64_to_user_ptr(req->rw.addr);
2014 size_t sqe_len = req->rw.len;
edafccee
JA		 2015 u8 opcode;
2016
d625c6ee 2017 opcode = req->opcode;
7d009165 2018 if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
edafccee 2019 *iovec = NULL;
9adbd45d 2020 return io_import_fixed(req, rw, iter);
edafccee 2021 }
2b188cc1 2022
9adbd45d
JA		 2023 /* buffer index only valid with fixed read/write */
2024 if (req->rw.kiocb.private)
2025 return -EINVAL;
2026
3a6820f2
JA		 2027 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
2028 ssize_t ret;
2029 ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
2030 *iovec = NULL;
2031 return ret;
2032 }
2033
f67676d1
JA		 2034 if (req->io) {
2035 struct io_async_rw *iorw = &req->io->rw;
2036
2037 *iovec = iorw->iov;
2038 iov_iter_init(iter, rw, *iovec, iorw->nr_segs, iorw->size);
2039 if (iorw->iov == iorw->fast_iov)
2040 *iovec = NULL;
2041 return iorw->size;
2042 }
2043
cf6fd4bd 2044 if (!req->has_user)
2b188cc1
JA		 2045 return -EFAULT;
2046
2047#ifdef CONFIG_COMPAT
cf6fd4bd 2048 if (req->ctx->compat)
2b188cc1
JA		 2049 return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,
2050 iovec, iter);
2051#endif
2052
2053 return import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter);
2054}
2055
31b51510 2056/*
32960613
JA		 2057 * For files that don't have ->read_iter() and ->write_iter(), handle them
2058 * by looping over ->read() or ->write() manually.
31b51510 2059 */
32960613
JA		 2060static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
2061 struct iov_iter *iter)
2062{
2063 ssize_t ret = 0;
2064
2065 /*
2066 * Don't support polled IO through this interface, and we can't
2067 * support non-blocking either. For the latter, this just causes
2068 * the kiocb to be handled from an async context.
2069 */
2070 if (kiocb->ki_flags & IOCB_HIPRI)
2071 return -EOPNOTSUPP;
2072 if (kiocb->ki_flags & IOCB_NOWAIT)
2073 return -EAGAIN;
2074
2075 while (iov_iter_count(iter)) {
311ae9e1 2076 struct iovec iovec;
32960613
JA		 2077 ssize_t nr;
2078
311ae9e1
PB		 2079 if (!iov_iter_is_bvec(iter)) {
2080 iovec = iov_iter_iovec(iter);
2081 } else {
2082 /* fixed buffers import bvec */
2083 iovec.iov_base = kmap(iter->bvec->bv_page)
2084 + iter->iov_offset;
2085 iovec.iov_len = min(iter->count,
2086 iter->bvec->bv_len - iter->iov_offset);
2087 }
2088
32960613
JA		 2089 if (rw == READ) {
2090 nr = file->f_op->read(file, iovec.iov_base,
2091 iovec.iov_len, &kiocb->ki_pos);
2092 } else {
2093 nr = file->f_op->write(file, iovec.iov_base,
2094 iovec.iov_len, &kiocb->ki_pos);
2095 }
2096
311ae9e1
PB		 2097 if (iov_iter_is_bvec(iter))
2098 kunmap(iter->bvec->bv_page);
2099
32960613
JA		 2100 if (nr < 0) {
2101 if (!ret)
2102 ret = nr;
2103 break;
2104 }
2105 ret += nr;
2106 if (nr != iovec.iov_len)
2107 break;
2108 iov_iter_advance(iter, nr);
2109 }
2110
2111 return ret;
2112}
2113
b7bb4f7d 2114static void io_req_map_rw(struct io_kiocb *req, ssize_t io_size,
f67676d1
JA		 2115 struct iovec *iovec, struct iovec *fast_iov,
2116 struct iov_iter *iter)
2117{
2118 req->io->rw.nr_segs = iter->nr_segs;
2119 req->io->rw.size = io_size;
2120 req->io->rw.iov = iovec;
2121 if (!req->io->rw.iov) {
2122 req->io->rw.iov = req->io->rw.fast_iov;
2123 memcpy(req->io->rw.iov, fast_iov,
2124 sizeof(struct iovec) * iter->nr_segs);
2125 }
2126}
2127
b7bb4f7d 2128static int io_alloc_async_ctx(struct io_kiocb *req)
f67676d1 2129{
d3656344
JA		 2130 if (!io_op_defs[req->opcode].async_ctx)
2131 return 0;
f67676d1 2132 req->io = kmalloc(sizeof(*req->io), GFP_KERNEL);
06b76d44 2133 return req->io == NULL;
b7bb4f7d
JA		 2134}
2135
2136static void io_rw_async(struct io_wq_work **workptr)
2137{
2138 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2139 struct iovec *iov = NULL;
2140
2141 if (req->io->rw.iov != req->io->rw.fast_iov)
2142 iov = req->io->rw.iov;
2143 io_wq_submit_work(workptr);
2144 kfree(iov);
2145}
2146
2147static int io_setup_async_rw(struct io_kiocb *req, ssize_t io_size,
2148 struct iovec *iovec, struct iovec *fast_iov,
2149 struct iov_iter *iter)
2150{
980ad263 2151 if (!io_op_defs[req->opcode].async_ctx)
74566df3 2152 return 0;
b7bb4f7d
JA		 2153 if (!req->io && io_alloc_async_ctx(req))
2154 return -ENOMEM;
2155
2156 io_req_map_rw(req, io_size, iovec, fast_iov, iter);
2157 req->work.func = io_rw_async;
2158 return 0;
f67676d1
JA		 2159}
2160
3529d8c2
JA		 2161static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
2162 bool force_nonblock)
f67676d1 2163{
3529d8c2
JA		 2164 struct io_async_ctx *io;
2165 struct iov_iter iter;
f67676d1
JA		 2166 ssize_t ret;
2167
3529d8c2
JA		 2168 ret = io_prep_rw(req, sqe, force_nonblock);
2169 if (ret)
2170 return ret;
f67676d1 2171
3529d8c2
JA		 2172 if (unlikely(!(req->file->f_mode & FMODE_READ)))
2173 return -EBADF;
f67676d1 2174
3529d8c2
JA		 2175 if (!req->io)
2176 return 0;
2177
2178 io = req->io;
2179 io->rw.iov = io->rw.fast_iov;
2180 req->io = NULL;
2181 ret = io_import_iovec(READ, req, &io->rw.iov, &iter);
2182 req->io = io;
2183 if (ret < 0)
2184 return ret;
2185
2186 io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
2187 return 0;
f67676d1
JA		 2188}
2189
267bc904 2190static int io_read(struct io_kiocb *req, struct io_kiocb **nxt,
8358e3a8 2191 bool force_nonblock)
2b188cc1
JA		 2192{
2193 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
9adbd45d 2194 struct kiocb *kiocb = &req->rw.kiocb;
2b188cc1 2195 struct iov_iter iter;
31b51510 2196 size_t iov_count;
f67676d1 2197 ssize_t io_size, ret;
2b188cc1 2198
3529d8c2 2199 ret = io_import_iovec(READ, req, &iovec, &iter);
06b76d44
JA		 2200 if (ret < 0)
2201 return ret;
2b188cc1 2202
fd6c2e4c
JA		 2203 /* Ensure we clear previously set non-block flag */
2204 if (!force_nonblock)
9adbd45d 2205 req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;
fd6c2e4c 2206
797f3f53 2207 req->result = 0;
f67676d1 2208 io_size = ret;
9e645e11 2209 if (req->flags & REQ_F_LINK)
f67676d1
JA		 2210 req->result = io_size;
2211
2212 /*
2213 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
2214 * we know to async punt it even if it was opened O_NONBLOCK
2215 */
9adbd45d 2216 if (force_nonblock && !io_file_supports_async(req->file)) {
f67676d1
JA		 2217 req->flags |= REQ_F_MUST_PUNT;
2218 goto copy_iov;
2219 }
9e645e11 2220
31b51510 2221 iov_count = iov_iter_count(&iter);
9adbd45d 2222 ret = rw_verify_area(READ, req->file, &kiocb->ki_pos, iov_count);
2b188cc1
JA		 2223 if (!ret) {
2224 ssize_t ret2;
2225
9adbd45d
JA		 2226 if (req->file->f_op->read_iter)
2227 ret2 = call_read_iter(req->file, kiocb, &iter);
32960613 2228 else
9adbd45d 2229 ret2 = loop_rw_iter(READ, req->file, kiocb, &iter);
32960613 2230
9d93a3f5 2231 /* Catch -EAGAIN return for forced non-blocking submission */
f67676d1 2232 if (!force_nonblock || ret2 != -EAGAIN) {
cf6fd4bd 2233 kiocb_done(kiocb, ret2, nxt, req->in_async);
f67676d1
JA		 2234 } else {
2235copy_iov:
b7bb4f7d 2236 ret = io_setup_async_rw(req, io_size, iovec,
f67676d1
JA		 2237 inline_vecs, &iter);
2238 if (ret)
2239 goto out_free;
2240 return -EAGAIN;
2241 }
2b188cc1 2242 }
f67676d1 2243out_free:
b7bb4f7d
JA		 2244 if (!io_wq_current_is_worker())
2245 kfree(iovec);
2b188cc1
JA		 2246 return ret;
2247}
2248
3529d8c2
JA		 2249static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
2250 bool force_nonblock)
f67676d1 2251{
3529d8c2
JA		 2252 struct io_async_ctx *io;
2253 struct iov_iter iter;
f67676d1
JA		 2254 ssize_t ret;
2255
3529d8c2
JA		 2256 ret = io_prep_rw(req, sqe, force_nonblock);
2257 if (ret)
2258 return ret;
f67676d1 2259
3529d8c2
JA		 2260 if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
2261 return -EBADF;
f67676d1 2262
3529d8c2
JA		 2263 if (!req->io)
2264 return 0;
2265
2266 io = req->io;
2267 io->rw.iov = io->rw.fast_iov;
2268 req->io = NULL;
2269 ret = io_import_iovec(WRITE, req, &io->rw.iov, &iter);
2270 req->io = io;
2271 if (ret < 0)
2272 return ret;
2273
2274 io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
2275 return 0;
f67676d1
JA		 2276}
2277
267bc904 2278static int io_write(struct io_kiocb *req, struct io_kiocb **nxt,
8358e3a8 2279 bool force_nonblock)
2b188cc1
JA		 2280{
2281 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
9adbd45d 2282 struct kiocb *kiocb = &req->rw.kiocb;
2b188cc1 2283 struct iov_iter iter;
31b51510 2284 size_t iov_count;
f67676d1 2285 ssize_t ret, io_size;
2b188cc1 2286
3529d8c2 2287 ret = io_import_iovec(WRITE, req, &iovec, &iter);
06b76d44
JA		 2288 if (ret < 0)
2289 return ret;
2b188cc1 2290
fd6c2e4c
JA		 2291 /* Ensure we clear previously set non-block flag */
2292 if (!force_nonblock)
9adbd45d 2293 req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;
fd6c2e4c 2294
797f3f53 2295 req->result = 0;
f67676d1 2296 io_size = ret;
9e645e11 2297 if (req->flags & REQ_F_LINK)
f67676d1 2298 req->result = io_size;
9e645e11 2299
f67676d1
JA		 2300 /*
2301 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
2302 * we know to async punt it even if it was opened O_NONBLOCK
2303 */
2304 if (force_nonblock && !io_file_supports_async(req->file)) {
2305 req->flags |= REQ_F_MUST_PUNT;
2306 goto copy_iov;
2307 }
31b51510 2308
10d59345
JA		 2309 /* file path doesn't support NOWAIT for non-direct_IO */
2310 if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
2311 (req->flags & REQ_F_ISREG))
f67676d1 2312 goto copy_iov;
31b51510 2313
f67676d1 2314 iov_count = iov_iter_count(&iter);
9adbd45d 2315 ret = rw_verify_area(WRITE, req->file, &kiocb->ki_pos, iov_count);
2b188cc1 2316 if (!ret) {
9bf7933f
RP		 2317 ssize_t ret2;
2318
2b188cc1
JA		 2319 /*
2320 * Open-code file_start_write here to grab freeze protection,
2321 * which will be released by another thread in
2322 * io_complete_rw(). Fool lockdep by telling it the lock got
2323 * released so that it doesn't complain about the held lock when
2324 * we return to userspace.
2325 */
491381ce 2326 if (req->flags & REQ_F_ISREG) {
9adbd45d 2327 __sb_start_write(file_inode(req->file)->i_sb,
2b188cc1 2328 SB_FREEZE_WRITE, true);
9adbd45d 2329 __sb_writers_release(file_inode(req->file)->i_sb,
2b188cc1
JA		 2330 SB_FREEZE_WRITE);
2331 }
2332 kiocb->ki_flags |= IOCB_WRITE;
9bf7933f 2333
9adbd45d
JA		 2334 if (req->file->f_op->write_iter)
2335 ret2 = call_write_iter(req->file, kiocb, &iter);
32960613 2336 else
9adbd45d 2337 ret2 = loop_rw_iter(WRITE, req->file, kiocb, &iter);
f67676d1 2338 if (!force_nonblock || ret2 != -EAGAIN) {
cf6fd4bd 2339 kiocb_done(kiocb, ret2, nxt, req->in_async);
f67676d1
JA		 2340 } else {
2341copy_iov:
b7bb4f7d 2342 ret = io_setup_async_rw(req, io_size, iovec,
f67676d1
JA		 2343 inline_vecs, &iter);
2344 if (ret)
2345 goto out_free;
2346 return -EAGAIN;
2347 }
2b188cc1 2348 }
31b51510 2349out_free:
b7bb4f7d
JA		 2350 if (!io_wq_current_is_worker())
2351 kfree(iovec);
2b188cc1
JA		 2352 return ret;
2353}
2354
2355/*
2356 * IORING_OP_NOP just posts a completion event, nothing else.
2357 */
78e19bbe 2358static int io_nop(struct io_kiocb *req)
2b188cc1
JA		 2359{
2360 struct io_ring_ctx *ctx = req->ctx;
2b188cc1 2361
def596e9
JA		 2362 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
2363 return -EINVAL;
2364
78e19bbe 2365 io_cqring_add_event(req, 0);
e65ef56d 2366 io_put_req(req);
2b188cc1
JA		 2367 return 0;
2368}
2369
3529d8c2 2370static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe)
c992fe29 2371{
6b06314c 2372 struct io_ring_ctx *ctx = req->ctx;
c992fe29 2373
09bb8394
JA		 2374 if (!req->file)
2375 return -EBADF;
c992fe29 2376
6b06314c 2377 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
def596e9 2378 return -EINVAL;
edafccee 2379 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
c992fe29
CH		 2380 return -EINVAL;
2381
8ed8d3c3
JA		 2382 req->sync.flags = READ_ONCE(sqe->fsync_flags);
2383 if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC))
2384 return -EINVAL;
2385
2386 req->sync.off = READ_ONCE(sqe->off);
2387 req->sync.len = READ_ONCE(sqe->len);
c992fe29
CH		 2388 return 0;
2389}
2390
8ed8d3c3
JA		 2391static bool io_req_cancelled(struct io_kiocb *req)
2392{
2393 if (req->work.flags & IO_WQ_WORK_CANCEL) {
2394 req_set_fail_links(req);
2395 io_cqring_add_event(req, -ECANCELED);
2396 io_put_req(req);
2397 return true;
2398 }
2399
2400 return false;
2401}
2402
78912934
JA		 2403static void io_link_work_cb(struct io_wq_work **workptr)
2404{
2405 struct io_wq_work *work = *workptr;
2406 struct io_kiocb *link = work->data;
2407
2408 io_queue_linked_timeout(link);
2409 work->func = io_wq_submit_work;
2410}
2411
2412static void io_wq_assign_next(struct io_wq_work **workptr, struct io_kiocb *nxt)
2413{
2414 struct io_kiocb *link;
2415
2416 io_prep_async_work(nxt, &link);
2417 *workptr = &nxt->work;
2418 if (link) {
2419 nxt->work.flags |= IO_WQ_WORK_CB;
2420 nxt->work.func = io_link_work_cb;
2421 nxt->work.data = link;
2422 }
2423}
2424
8ed8d3c3
JA		 2425static void io_fsync_finish(struct io_wq_work **workptr)
2426{
2427 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2428 loff_t end = req->sync.off + req->sync.len;
2429 struct io_kiocb *nxt = NULL;
2430 int ret;
2431
2432 if (io_req_cancelled(req))
2433 return;
2434
9adbd45d 2435 ret = vfs_fsync_range(req->file, req->sync.off,
8ed8d3c3
JA		 2436 end > 0 ? end : LLONG_MAX,
2437 req->sync.flags & IORING_FSYNC_DATASYNC);
2438 if (ret < 0)
2439 req_set_fail_links(req);
2440 io_cqring_add_event(req, ret);
2441 io_put_req_find_next(req, &nxt);
2442 if (nxt)
78912934 2443 io_wq_assign_next(workptr, nxt);
8ed8d3c3
JA		 2444}
2445
fc4df999
JA		 2446static int io_fsync(struct io_kiocb *req, struct io_kiocb **nxt,
2447 bool force_nonblock)
c992fe29 2448{
8ed8d3c3 2449 struct io_wq_work *work, *old_work;
c992fe29
CH		 2450
2451 /* fsync always requires a blocking context */
8ed8d3c3
JA		 2452 if (force_nonblock) {
2453 io_put_req(req);
2454 req->work.func = io_fsync_finish;
c992fe29 2455 return -EAGAIN;
8ed8d3c3 2456 }
c992fe29 2457
8ed8d3c3
JA		 2458 work = old_work = &req->work;
2459 io_fsync_finish(&work);
2460 if (work && work != old_work)
2461 *nxt = container_of(work, struct io_kiocb, work);
c992fe29
CH		 2462 return 0;
2463}
2464
d63d1b5e
JA		 2465static void io_fallocate_finish(struct io_wq_work **workptr)
2466{
2467 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2468 struct io_kiocb *nxt = NULL;
2469 int ret;
2470
2471 ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
2472 req->sync.len);
2473 if (ret < 0)
2474 req_set_fail_links(req);
2475 io_cqring_add_event(req, ret);
2476 io_put_req_find_next(req, &nxt);
2477 if (nxt)
2478 io_wq_assign_next(workptr, nxt);
2479}
2480
2481static int io_fallocate_prep(struct io_kiocb *req,
2482 const struct io_uring_sqe *sqe)
2483{
2484 if (sqe->ioprio || sqe->buf_index || sqe->rw_flags)
2485 return -EINVAL;
2486
2487 req->sync.off = READ_ONCE(sqe->off);
2488 req->sync.len = READ_ONCE(sqe->addr);
2489 req->sync.mode = READ_ONCE(sqe->len);
2490 return 0;
2491}
2492
2493static int io_fallocate(struct io_kiocb *req, struct io_kiocb **nxt,
2494 bool force_nonblock)
2495{
2496 struct io_wq_work *work, *old_work;
2497
2498 /* fallocate always requiring blocking context */
2499 if (force_nonblock) {
2500 io_put_req(req);
2501 req->work.func = io_fallocate_finish;
2502 return -EAGAIN;
2503 }
2504
2505 work = old_work = &req->work;
2506 io_fallocate_finish(&work);
2507 if (work && work != old_work)
2508 *nxt = container_of(work, struct io_kiocb, work);
2509
2510 return 0;
2511}
2512
15b71abe
JA		 2513static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2514{
f8748881 2515 const char __user *fname;
15b71abe
JA		 2516 int ret;
2517
2518 if (sqe->ioprio || sqe->buf_index)
2519 return -EINVAL;
2520
2521 req->open.dfd = READ_ONCE(sqe->fd);
c12cedf2 2522 req->open.how.mode = READ_ONCE(sqe->len);
f8748881 2523 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
c12cedf2 2524 req->open.how.flags = READ_ONCE(sqe->open_flags);
15b71abe 2525
f8748881 2526 req->open.filename = getname(fname);
15b71abe
JA		 2527 if (IS_ERR(req->open.filename)) {
2528 ret = PTR_ERR(req->open.filename);
2529 req->open.filename = NULL;
2530 return ret;
2531 }
2532
2533 return 0;
2534}
2535
cebdb986
JA		 2536static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2537{
2538 struct open_how __user *how;
2539 const char __user *fname;
2540 size_t len;
2541 int ret;
2542
2543 if (sqe->ioprio || sqe->buf_index)
2544 return -EINVAL;
2545
2546 req->open.dfd = READ_ONCE(sqe->fd);
2547 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
2548 how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
2549 len = READ_ONCE(sqe->len);
2550
2551 if (len < OPEN_HOW_SIZE_VER0)
2552 return -EINVAL;
2553
2554 ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how,
2555 len);
2556 if (ret)
2557 return ret;
2558
2559 if (!(req->open.how.flags & O_PATH) && force_o_largefile())
2560 req->open.how.flags |= O_LARGEFILE;
2561
2562 req->open.filename = getname(fname);
2563 if (IS_ERR(req->open.filename)) {
2564 ret = PTR_ERR(req->open.filename);
2565 req->open.filename = NULL;
2566 return ret;
2567 }
2568
2569 return 0;
2570}
2571
2572static int io_openat2(struct io_kiocb *req, struct io_kiocb **nxt,
2573 bool force_nonblock)
15b71abe
JA		 2574{
2575 struct open_flags op;
15b71abe
JA		 2576 struct file *file;
2577 int ret;
2578
f86cd20c 2579 if (force_nonblock)
15b71abe 2580 return -EAGAIN;
15b71abe 2581
cebdb986 2582 ret = build_open_flags(&req->open.how, &op);
15b71abe
JA		 2583 if (ret)
2584 goto err;
2585
cebdb986 2586 ret = get_unused_fd_flags(req->open.how.flags);
15b71abe
JA		 2587 if (ret < 0)
2588 goto err;
2589
2590 file = do_filp_open(req->open.dfd, req->open.filename, &op);
2591 if (IS_ERR(file)) {
2592 put_unused_fd(ret);
2593 ret = PTR_ERR(file);
2594 } else {
2595 fsnotify_open(file);
2596 fd_install(ret, file);
2597 }
2598err:
2599 putname(req->open.filename);
2600 if (ret < 0)
2601 req_set_fail_links(req);
2602 io_cqring_add_event(req, ret);
2603 io_put_req_find_next(req, nxt);
2604 return 0;
2605}
2606
cebdb986
JA		 2607static int io_openat(struct io_kiocb *req, struct io_kiocb **nxt,
2608 bool force_nonblock)
2609{
2610 req->open.how = build_open_how(req->open.how.flags, req->open.how.mode);
2611 return io_openat2(req, nxt, force_nonblock);
2612}
2613
c1ca757b
JA		 2614static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2615{
2616#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
2617 if (sqe->ioprio || sqe->buf_index || sqe->off)
2618 return -EINVAL;
2619
2620 req->madvise.addr = READ_ONCE(sqe->addr);
2621 req->madvise.len = READ_ONCE(sqe->len);
2622 req->madvise.advice = READ_ONCE(sqe->fadvise_advice);
2623 return 0;
2624#else
2625 return -EOPNOTSUPP;
2626#endif
2627}
2628
2629static int io_madvise(struct io_kiocb *req, struct io_kiocb **nxt,
2630 bool force_nonblock)
2631{
2632#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
2633 struct io_madvise *ma = &req->madvise;
2634 int ret;
2635
2636 if (force_nonblock)
2637 return -EAGAIN;
2638
2639 ret = do_madvise(ma->addr, ma->len, ma->advice);
2640 if (ret < 0)
2641 req_set_fail_links(req);
2642 io_cqring_add_event(req, ret);
2643 io_put_req_find_next(req, nxt);
2644 return 0;
2645#else
2646 return -EOPNOTSUPP;
2647#endif
2648}
2649
4840e418
JA		 2650static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2651{
2652 if (sqe->ioprio || sqe->buf_index || sqe->addr)
2653 return -EINVAL;
2654
2655 req->fadvise.offset = READ_ONCE(sqe->off);
2656 req->fadvise.len = READ_ONCE(sqe->len);
2657 req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
2658 return 0;
2659}
2660
2661static int io_fadvise(struct io_kiocb *req, struct io_kiocb **nxt,
2662 bool force_nonblock)
2663{
2664 struct io_fadvise *fa = &req->fadvise;
2665 int ret;
2666
2667 /* DONTNEED may block, others _should_ not */
2668 if (fa->advice == POSIX_FADV_DONTNEED && force_nonblock)
2669 return -EAGAIN;
2670
2671 ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
2672 if (ret < 0)
2673 req_set_fail_links(req);
2674 io_cqring_add_event(req, ret);
2675 io_put_req_find_next(req, nxt);
2676 return 0;
2677}
2678
eddc7ef5
JA		 2679static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2680{
f8748881 2681 const char __user *fname;
eddc7ef5
JA		 2682 unsigned lookup_flags;
2683 int ret;
2684
2685 if (sqe->ioprio || sqe->buf_index)
2686 return -EINVAL;
2687
2688 req->open.dfd = READ_ONCE(sqe->fd);
2689 req->open.mask = READ_ONCE(sqe->len);
f8748881 2690 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
eddc7ef5 2691 req->open.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
c12cedf2 2692 req->open.how.flags = READ_ONCE(sqe->statx_flags);
eddc7ef5 2693
c12cedf2 2694 if (vfs_stat_set_lookup_flags(&lookup_flags, req->open.how.flags))
eddc7ef5
JA		 2695 return -EINVAL;
2696
f8748881 2697 req->open.filename = getname_flags(fname, lookup_flags, NULL);
eddc7ef5
JA		 2698 if (IS_ERR(req->open.filename)) {
2699 ret = PTR_ERR(req->open.filename);
2700 req->open.filename = NULL;
2701 return ret;
2702 }
2703
2704 return 0;
2705}
2706
2707static int io_statx(struct io_kiocb *req, struct io_kiocb **nxt,
2708 bool force_nonblock)
2709{
2710 struct io_open *ctx = &req->open;
2711 unsigned lookup_flags;
2712 struct path path;
2713 struct kstat stat;
2714 int ret;
2715
2716 if (force_nonblock)
2717 return -EAGAIN;
2718
c12cedf2 2719 if (vfs_stat_set_lookup_flags(&lookup_flags, ctx->how.flags))
eddc7ef5
JA		 2720 return -EINVAL;
2721
2722retry:
2723 /* filename_lookup() drops it, keep a reference */
2724 ctx->filename->refcnt++;
2725
2726 ret = filename_lookup(ctx->dfd, ctx->filename, lookup_flags, &path,
2727 NULL);
2728 if (ret)
2729 goto err;
2730
c12cedf2 2731 ret = vfs_getattr(&path, &stat, ctx->mask, ctx->how.flags);
eddc7ef5
JA		 2732 path_put(&path);
2733 if (retry_estale(ret, lookup_flags)) {
2734 lookup_flags |= LOOKUP_REVAL;
2735 goto retry;
2736 }
2737 if (!ret)
2738 ret = cp_statx(&stat, ctx->buffer);
2739err:
2740 putname(ctx->filename);
2741 if (ret < 0)
2742 req_set_fail_links(req);
2743 io_cqring_add_event(req, ret);
2744 io_put_req_find_next(req, nxt);
2745 return 0;
2746}
2747
b5dba59e
JA		 2748static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2749{
2750 /*
2751 * If we queue this for async, it must not be cancellable. That would
2752 * leave the 'file' in an undeterminate state.
2753 */
2754 req->work.flags |= IO_WQ_WORK_NO_CANCEL;
2755
2756 if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
2757 sqe->rw_flags || sqe->buf_index)
2758 return -EINVAL;
2759 if (sqe->flags & IOSQE_FIXED_FILE)
2760 return -EINVAL;
2761
2762 req->close.fd = READ_ONCE(sqe->fd);
2763 if (req->file->f_op == &io_uring_fops ||
b14cca0c 2764 req->close.fd == req->ctx->ring_fd)
b5dba59e
JA		 2765 return -EBADF;
2766
2767 return 0;
2768}
2769
2770static void io_close_finish(struct io_wq_work **workptr)
2771{
2772 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2773 struct io_kiocb *nxt = NULL;
2774
2775 /* Invoked with files, we need to do the close */
2776 if (req->work.files) {
2777 int ret;
2778
2779 ret = filp_close(req->close.put_file, req->work.files);
2780 if (ret < 0) {
2781 req_set_fail_links(req);
2782 }
2783 io_cqring_add_event(req, ret);
2784 }
2785
2786 fput(req->close.put_file);
2787
2788 /* we bypassed the re-issue, drop the submission reference */
2789 io_put_req(req);
2790 io_put_req_find_next(req, &nxt);
2791 if (nxt)
2792 io_wq_assign_next(workptr, nxt);
2793}
2794
2795static int io_close(struct io_kiocb *req, struct io_kiocb **nxt,
2796 bool force_nonblock)
2797{
2798 int ret;
2799
2800 req->close.put_file = NULL;
2801 ret = __close_fd_get_file(req->close.fd, &req->close.put_file);
2802 if (ret < 0)
2803 return ret;
2804
2805 /* if the file has a flush method, be safe and punt to async */
f86cd20c 2806 if (req->close.put_file->f_op->flush && !io_wq_current_is_worker())
b5dba59e 2807 goto eagain;
b5dba59e
JA		 2808
2809 /*
2810 * No ->flush(), safely close from here and just punt the
2811 * fput() to async context.
2812 */
2813 ret = filp_close(req->close.put_file, current->files);
2814
2815 if (ret < 0)
2816 req_set_fail_links(req);
2817 io_cqring_add_event(req, ret);
2818
2819 if (io_wq_current_is_worker()) {
2820 struct io_wq_work *old_work, *work;
2821
2822 old_work = work = &req->work;
2823 io_close_finish(&work);
2824 if (work && work != old_work)
2825 *nxt = container_of(work, struct io_kiocb, work);
2826 return 0;
2827 }
2828
2829eagain:
2830 req->work.func = io_close_finish;
2831 return -EAGAIN;
2832}
2833
3529d8c2 2834static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5d17b4a4
JA		 2835{
2836 struct io_ring_ctx *ctx = req->ctx;
5d17b4a4
JA		 2837
2838 if (!req->file)
2839 return -EBADF;
5d17b4a4
JA		 2840
2841 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
2842 return -EINVAL;
2843 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
2844 return -EINVAL;
2845
8ed8d3c3
JA		 2846 req->sync.off = READ_ONCE(sqe->off);
2847 req->sync.len = READ_ONCE(sqe->len);
2848 req->sync.flags = READ_ONCE(sqe->sync_range_flags);
8ed8d3c3
JA		 2849 return 0;
2850}
2851
2852static void io_sync_file_range_finish(struct io_wq_work **workptr)
2853{
2854 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2855 struct io_kiocb *nxt = NULL;
2856 int ret;
2857
2858 if (io_req_cancelled(req))
2859 return;
2860
9adbd45d 2861 ret = sync_file_range(req->file, req->sync.off, req->sync.len,
8ed8d3c3
JA		 2862 req->sync.flags);
2863 if (ret < 0)
2864 req_set_fail_links(req);
2865 io_cqring_add_event(req, ret);
2866 io_put_req_find_next(req, &nxt);
2867 if (nxt)
78912934 2868 io_wq_assign_next(workptr, nxt);
5d17b4a4
JA		 2869}
2870
fc4df999 2871static int io_sync_file_range(struct io_kiocb *req, struct io_kiocb **nxt,
5d17b4a4
JA		 2872 bool force_nonblock)
2873{
8ed8d3c3 2874 struct io_wq_work *work, *old_work;
5d17b4a4
JA		 2875
2876 /* sync_file_range always requires a blocking context */
8ed8d3c3
JA		 2877 if (force_nonblock) {
2878 io_put_req(req);
2879 req->work.func = io_sync_file_range_finish;
5d17b4a4 2880 return -EAGAIN;
8ed8d3c3 2881 }
5d17b4a4 2882
8ed8d3c3
JA		 2883 work = old_work = &req->work;
2884 io_sync_file_range_finish(&work);
2885 if (work && work != old_work)
2886 *nxt = container_of(work, struct io_kiocb, work);
5d17b4a4
JA		 2887 return 0;
2888}
2889
b7bb4f7d
JA		 2890#if defined(CONFIG_NET)
2891static void io_sendrecv_async(struct io_wq_work **workptr)
2892{
2893 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2894 struct iovec *iov = NULL;
2895
2896 if (req->io->rw.iov != req->io->rw.fast_iov)
2897 iov = req->io->msg.iov;
2898 io_wq_submit_work(workptr);
2899 kfree(iov);
2900}
2901#endif
2902
3529d8c2 2903static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
03b1230c 2904{
0fa03c62 2905#if defined(CONFIG_NET)
e47293fd 2906 struct io_sr_msg *sr = &req->sr_msg;
3529d8c2 2907 struct io_async_ctx *io = req->io;
03b1230c 2908
e47293fd
JA		 2909 sr->msg_flags = READ_ONCE(sqe->msg_flags);
2910 sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
fddaface 2911 sr->len = READ_ONCE(sqe->len);
3529d8c2 2912
fddaface 2913 if (!io || req->opcode == IORING_OP_SEND)
3529d8c2
JA		 2914 return 0;
2915
d9688565 2916 io->msg.iov = io->msg.fast_iov;
3529d8c2 2917 return sendmsg_copy_msghdr(&io->msg.msg, sr->msg, sr->msg_flags,
e47293fd 2918 &io->msg.iov);
03b1230c 2919#else
e47293fd 2920 return -EOPNOTSUPP;
03b1230c
JA		 2921#endif
2922}
2923
fc4df999
JA		 2924static int io_sendmsg(struct io_kiocb *req, struct io_kiocb **nxt,
2925 bool force_nonblock)
aa1fa28f 2926{
03b1230c 2927#if defined(CONFIG_NET)
0b416c3e 2928 struct io_async_msghdr *kmsg = NULL;
0fa03c62
JA		 2929 struct socket *sock;
2930 int ret;
2931
2932 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
2933 return -EINVAL;
2934
2935 sock = sock_from_file(req->file, &ret);
2936 if (sock) {
b7bb4f7d 2937 struct io_async_ctx io;
03b1230c 2938 struct sockaddr_storage addr;
0fa03c62
JA		 2939 unsigned flags;
2940
03b1230c 2941 if (req->io) {
0b416c3e
JA		 2942 kmsg = &req->io->msg;
2943 kmsg->msg.msg_name = &addr;
2944 /* if iov is set, it's allocated already */
2945 if (!kmsg->iov)
2946 kmsg->iov = kmsg->fast_iov;
2947 kmsg->msg.msg_iter.iov = kmsg->iov;
03b1230c 2948 } else {
3529d8c2
JA		 2949 struct io_sr_msg *sr = &req->sr_msg;
2950
0b416c3e
JA		 2951 kmsg = &io.msg;
2952 kmsg->msg.msg_name = &addr;
3529d8c2
JA		 2953
2954 io.msg.iov = io.msg.fast_iov;
2955 ret = sendmsg_copy_msghdr(&io.msg.msg, sr->msg,
2956 sr->msg_flags, &io.msg.iov);
03b1230c 2957 if (ret)
3529d8c2 2958 return ret;
03b1230c 2959 }
0fa03c62 2960
e47293fd
JA		 2961 flags = req->sr_msg.msg_flags;
2962 if (flags & MSG_DONTWAIT)
2963 req->flags |= REQ_F_NOWAIT;
2964 else if (force_nonblock)
2965 flags |= MSG_DONTWAIT;
2966
0b416c3e 2967 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
03b1230c 2968 if (force_nonblock && ret == -EAGAIN) {
b7bb4f7d
JA		 2969 if (req->io)
2970 return -EAGAIN;
2971 if (io_alloc_async_ctx(req))
2972 return -ENOMEM;
2973 memcpy(&req->io->msg, &io.msg, sizeof(io.msg));
2974 req->work.func = io_sendrecv_async;
0b416c3e 2975 return -EAGAIN;
03b1230c 2976 }
441cdbd5
JA		 2977 if (ret == -ERESTARTSYS)
2978 ret = -EINTR;
0fa03c62
JA		 2979 }
2980
b7bb4f7d 2981 if (!io_wq_current_is_worker() && kmsg && kmsg->iov != kmsg->fast_iov)
0b416c3e 2982 kfree(kmsg->iov);
78e19bbe 2983 io_cqring_add_event(req, ret);
4e88d6e7
JA		 2984 if (ret < 0)
2985 req_set_fail_links(req);
ec9c02ad 2986 io_put_req_find_next(req, nxt);
5d17b4a4 2987 return 0;
03b1230c
JA		 2988#else
2989 return -EOPNOTSUPP;
aa1fa28f 2990#endif
03b1230c 2991}
aa1fa28f 2992
fddaface
JA		 2993static int io_send(struct io_kiocb *req, struct io_kiocb **nxt,
2994 bool force_nonblock)
2995{
2996#if defined(CONFIG_NET)
2997 struct socket *sock;
2998 int ret;
2999
3000 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3001 return -EINVAL;
3002
3003 sock = sock_from_file(req->file, &ret);
3004 if (sock) {
3005 struct io_sr_msg *sr = &req->sr_msg;
3006 struct msghdr msg;
3007 struct iovec iov;
3008 unsigned flags;
3009
3010 ret = import_single_range(WRITE, sr->buf, sr->len, &iov,
3011 &msg.msg_iter);
3012 if (ret)
3013 return ret;
3014
3015 msg.msg_name = NULL;
3016 msg.msg_control = NULL;
3017 msg.msg_controllen = 0;
3018 msg.msg_namelen = 0;
3019
3020 flags = req->sr_msg.msg_flags;
3021 if (flags & MSG_DONTWAIT)
3022 req->flags |= REQ_F_NOWAIT;
3023 else if (force_nonblock)
3024 flags |= MSG_DONTWAIT;
3025
3026 ret = __sys_sendmsg_sock(sock, &msg, flags);
3027 if (force_nonblock && ret == -EAGAIN)
3028 return -EAGAIN;
3029 if (ret == -ERESTARTSYS)
3030 ret = -EINTR;
3031 }
3032
3033 io_cqring_add_event(req, ret);
3034 if (ret < 0)
3035 req_set_fail_links(req);
3036 io_put_req_find_next(req, nxt);
3037 return 0;
3038#else
3039 return -EOPNOTSUPP;
3040#endif
3041}
3042
3529d8c2
JA		 3043static int io_recvmsg_prep(struct io_kiocb *req,
3044 const struct io_uring_sqe *sqe)
aa1fa28f
JA		 3045{
3046#if defined(CONFIG_NET)
e47293fd 3047 struct io_sr_msg *sr = &req->sr_msg;
3529d8c2
JA		 3048 struct io_async_ctx *io = req->io;
3049
3050 sr->msg_flags = READ_ONCE(sqe->msg_flags);
3051 sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
06b76d44 3052
fddaface 3053 if (!io || req->opcode == IORING_OP_RECV)
06b76d44 3054 return 0;
03b1230c 3055
d9688565 3056 io->msg.iov = io->msg.fast_iov;
3529d8c2 3057 return recvmsg_copy_msghdr(&io->msg.msg, sr->msg, sr->msg_flags,
e47293fd 3058 &io->msg.uaddr, &io->msg.iov);
aa1fa28f 3059#else
e47293fd 3060 return -EOPNOTSUPP;
aa1fa28f
JA		 3061#endif
3062}
3063
fc4df999
JA		 3064static int io_recvmsg(struct io_kiocb *req, struct io_kiocb **nxt,
3065 bool force_nonblock)
aa1fa28f
JA		 3066{
3067#if defined(CONFIG_NET)
0b416c3e 3068 struct io_async_msghdr *kmsg = NULL;
03b1230c
JA		 3069 struct socket *sock;
3070 int ret;
3071
3072 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3073 return -EINVAL;
3074
3075 sock = sock_from_file(req->file, &ret);
3076 if (sock) {
b7bb4f7d 3077 struct io_async_ctx io;
03b1230c 3078 struct sockaddr_storage addr;
03b1230c
JA		 3079 unsigned flags;
3080
03b1230c 3081 if (req->io) {
0b416c3e
JA		 3082 kmsg = &req->io->msg;
3083 kmsg->msg.msg_name = &addr;
3084 /* if iov is set, it's allocated already */
3085 if (!kmsg->iov)
3086 kmsg->iov = kmsg->fast_iov;
3087 kmsg->msg.msg_iter.iov = kmsg->iov;
03b1230c 3088 } else {
3529d8c2
JA		 3089 struct io_sr_msg *sr = &req->sr_msg;
3090
0b416c3e
JA		 3091 kmsg = &io.msg;
3092 kmsg->msg.msg_name = &addr;
3529d8c2
JA		 3093
3094 io.msg.iov = io.msg.fast_iov;
3095 ret = recvmsg_copy_msghdr(&io.msg.msg, sr->msg,
3096 sr->msg_flags, &io.msg.uaddr,
3097 &io.msg.iov);
03b1230c 3098 if (ret)
3529d8c2 3099 return ret;
03b1230c
JA		 3100 }
3101
e47293fd
JA		 3102 flags = req->sr_msg.msg_flags;
3103 if (flags & MSG_DONTWAIT)
3104 req->flags |= REQ_F_NOWAIT;
3105 else if (force_nonblock)
3106 flags |= MSG_DONTWAIT;
3107
3108 ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.msg,
3109 kmsg->uaddr, flags);
03b1230c 3110 if (force_nonblock && ret == -EAGAIN) {
b7bb4f7d
JA		 3111 if (req->io)
3112 return -EAGAIN;
3113 if (io_alloc_async_ctx(req))
3114 return -ENOMEM;
3115 memcpy(&req->io->msg, &io.msg, sizeof(io.msg));
3116 req->work.func = io_sendrecv_async;
0b416c3e 3117 return -EAGAIN;
03b1230c
JA		 3118 }
3119 if (ret == -ERESTARTSYS)
3120 ret = -EINTR;
3121 }
3122
b7bb4f7d 3123 if (!io_wq_current_is_worker() && kmsg && kmsg->iov != kmsg->fast_iov)
0b416c3e 3124 kfree(kmsg->iov);
03b1230c 3125 io_cqring_add_event(req, ret);
4e88d6e7
JA		 3126 if (ret < 0)
3127 req_set_fail_links(req);
03b1230c
JA		 3128 io_put_req_find_next(req, nxt);
3129 return 0;
0fa03c62
JA		 3130#else
3131 return -EOPNOTSUPP;
3132#endif
3133}
5d17b4a4 3134
fddaface
JA		 3135static int io_recv(struct io_kiocb *req, struct io_kiocb **nxt,
3136 bool force_nonblock)
3137{
3138#if defined(CONFIG_NET)
3139 struct socket *sock;
3140 int ret;
3141
3142 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3143 return -EINVAL;
3144
3145 sock = sock_from_file(req->file, &ret);
3146 if (sock) {
3147 struct io_sr_msg *sr = &req->sr_msg;
3148 struct msghdr msg;
3149 struct iovec iov;
3150 unsigned flags;
3151
3152 ret = import_single_range(READ, sr->buf, sr->len, &iov,
3153 &msg.msg_iter);
3154 if (ret)
3155 return ret;
3156
3157 msg.msg_name = NULL;
3158 msg.msg_control = NULL;
3159 msg.msg_controllen = 0;
3160 msg.msg_namelen = 0;
3161 msg.msg_iocb = NULL;
3162 msg.msg_flags = 0;
3163
3164 flags = req->sr_msg.msg_flags;
3165 if (flags & MSG_DONTWAIT)
3166 req->flags |= REQ_F_NOWAIT;
3167 else if (force_nonblock)
3168 flags |= MSG_DONTWAIT;
3169
3170 ret = __sys_recvmsg_sock(sock, &msg, NULL, NULL, flags);
3171 if (force_nonblock && ret == -EAGAIN)
3172 return -EAGAIN;
3173 if (ret == -ERESTARTSYS)
3174 ret = -EINTR;
3175 }
3176
3177 io_cqring_add_event(req, ret);
3178 if (ret < 0)
3179 req_set_fail_links(req);
3180 io_put_req_find_next(req, nxt);
3181 return 0;
3182#else
3183 return -EOPNOTSUPP;
3184#endif
3185}
3186
3187
3529d8c2 3188static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
17f2fe35
JA		 3189{
3190#if defined(CONFIG_NET)
8ed8d3c3
JA		 3191 struct io_accept *accept = &req->accept;
3192
17f2fe35
JA		 3193 if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
3194 return -EINVAL;
8042d6ce 3195 if (sqe->ioprio || sqe->len || sqe->buf_index)
17f2fe35
JA		 3196 return -EINVAL;
3197
d55e5f5b
JA		 3198 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
3199 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
8ed8d3c3 3200 accept->flags = READ_ONCE(sqe->accept_flags);
8ed8d3c3
JA		 3201 return 0;
3202#else
3203 return -EOPNOTSUPP;
3204#endif
3205}
17f2fe35 3206
8ed8d3c3
JA		 3207#if defined(CONFIG_NET)
3208static int __io_accept(struct io_kiocb *req, struct io_kiocb **nxt,
3209 bool force_nonblock)
3210{
3211 struct io_accept *accept = &req->accept;
3212 unsigned file_flags;
3213 int ret;
3214
3215 file_flags = force_nonblock ? O_NONBLOCK : 0;
3216 ret = __sys_accept4_file(req->file, file_flags, accept->addr,
3217 accept->addr_len, accept->flags);
3218 if (ret == -EAGAIN && force_nonblock)
17f2fe35 3219 return -EAGAIN;
8e3cca12
JA		 3220 if (ret == -ERESTARTSYS)
3221 ret = -EINTR;
4e88d6e7
JA		 3222 if (ret < 0)
3223 req_set_fail_links(req);
78e19bbe 3224 io_cqring_add_event(req, ret);
ec9c02ad 3225 io_put_req_find_next(req, nxt);
17f2fe35 3226 return 0;
8ed8d3c3
JA		 3227}
3228
3229static void io_accept_finish(struct io_wq_work **workptr)
3230{
3231 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
3232 struct io_kiocb *nxt = NULL;
3233
3234 if (io_req_cancelled(req))
3235 return;
3236 __io_accept(req, &nxt, false);
3237 if (nxt)
78912934 3238 io_wq_assign_next(workptr, nxt);
8ed8d3c3
JA		 3239}
3240#endif
3241
3242static int io_accept(struct io_kiocb *req, struct io_kiocb **nxt,
3243 bool force_nonblock)
3244{
3245#if defined(CONFIG_NET)
3246 int ret;
3247
8ed8d3c3
JA		 3248 ret = __io_accept(req, nxt, force_nonblock);
3249 if (ret == -EAGAIN && force_nonblock) {
3250 req->work.func = io_accept_finish;
8ed8d3c3
JA		 3251 io_put_req(req);
3252 return -EAGAIN;
3253 }
3254 return 0;
0fa03c62
JA		 3255#else
3256 return -EOPNOTSUPP;
3257#endif
3258}
5d17b4a4 3259
3529d8c2 3260static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
f499a021
JA		 3261{
3262#if defined(CONFIG_NET)
3529d8c2
JA		 3263 struct io_connect *conn = &req->connect;
3264 struct io_async_ctx *io = req->io;
f499a021 3265
3fbb51c1
JA		 3266 if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
3267 return -EINVAL;
3268 if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags)
3269 return -EINVAL;
3270
3529d8c2
JA		 3271 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
3272 conn->addr_len = READ_ONCE(sqe->addr2);
3273
3274 if (!io)
3275 return 0;
3276
3277 return move_addr_to_kernel(conn->addr, conn->addr_len,
3fbb51c1 3278 &io->connect.address);
f499a021 3279#else
3fbb51c1 3280 return -EOPNOTSUPP;
f499a021
JA		 3281#endif
3282}
3283
fc4df999
JA		 3284static int io_connect(struct io_kiocb *req, struct io_kiocb **nxt,
3285 bool force_nonblock)
f8e85cf2
JA		 3286{
3287#if defined(CONFIG_NET)
f499a021 3288 struct io_async_ctx __io, *io;
f8e85cf2 3289 unsigned file_flags;
3fbb51c1 3290 int ret;
f8e85cf2 3291
f499a021
JA		 3292 if (req->io) {
3293 io = req->io;
3294 } else {
3529d8c2
JA		 3295 ret = move_addr_to_kernel(req->connect.addr,
3296 req->connect.addr_len,
3297 &__io.connect.address);
f499a021
JA		 3298 if (ret)
3299 goto out;
3300 io = &__io;
3301 }
3302
3fbb51c1
JA		 3303 file_flags = force_nonblock ? O_NONBLOCK : 0;
3304
3305 ret = __sys_connect_file(req->file, &io->connect.address,
3306 req->connect.addr_len, file_flags);
87f80d62 3307 if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
b7bb4f7d
JA		 3308 if (req->io)
3309 return -EAGAIN;
3310 if (io_alloc_async_ctx(req)) {
f499a021
JA		 3311 ret = -ENOMEM;
3312 goto out;
3313 }
b7bb4f7d 3314 memcpy(&req->io->connect, &__io.connect, sizeof(__io.connect));
f8e85cf2 3315 return -EAGAIN;
f499a021 3316 }
f8e85cf2
JA		 3317 if (ret == -ERESTARTSYS)
3318 ret = -EINTR;
f499a021 3319out:
4e88d6e7
JA		 3320 if (ret < 0)
3321 req_set_fail_links(req);
f8e85cf2
JA		 3322 io_cqring_add_event(req, ret);
3323 io_put_req_find_next(req, nxt);
3324 return 0;
3325#else
3326 return -EOPNOTSUPP;
3327#endif
3328}
3329
221c5eb2
JA		 3330static void io_poll_remove_one(struct io_kiocb *req)
3331{
3332 struct io_poll_iocb *poll = &req->poll;
3333
3334 spin_lock(&poll->head->lock);
3335 WRITE_ONCE(poll->canceled, true);
392edb45
JA		 3336 if (!list_empty(&poll->wait.entry)) {
3337 list_del_init(&poll->wait.entry);
a197f664 3338 io_queue_async_work(req);
221c5eb2
JA		 3339 }
3340 spin_unlock(&poll->head->lock);
78076bb6 3341 hash_del(&req->hash_node);
221c5eb2
JA		 3342}
3343
3344static void io_poll_remove_all(struct io_ring_ctx *ctx)
3345{
78076bb6 3346 struct hlist_node *tmp;
221c5eb2 3347 struct io_kiocb *req;
78076bb6 3348 int i;
221c5eb2
JA		 3349
3350 spin_lock_irq(&ctx->completion_lock);
78076bb6
JA		 3351 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
3352 struct hlist_head *list;
3353
3354 list = &ctx->cancel_hash[i];
3355 hlist_for_each_entry_safe(req, tmp, list, hash_node)
3356 io_poll_remove_one(req);
221c5eb2
JA		 3357 }
3358 spin_unlock_irq(&ctx->completion_lock);
3359}
3360
47f46768
JA		 3361static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr)
3362{
78076bb6 3363 struct hlist_head *list;
47f46768
JA		 3364 struct io_kiocb *req;
3365
78076bb6
JA		 3366 list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
3367 hlist_for_each_entry(req, list, hash_node) {
3368 if (sqe_addr == req->user_data) {
eac406c6
JA		 3369 io_poll_remove_one(req);
3370 return 0;
3371 }
47f46768
JA		 3372 }
3373
3374 return -ENOENT;
3375}
3376
3529d8c2
JA		 3377static int io_poll_remove_prep(struct io_kiocb *req,
3378 const struct io_uring_sqe *sqe)
0969e783 3379{
0969e783
JA		 3380 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3381 return -EINVAL;
3382 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
3383 sqe->poll_events)
3384 return -EINVAL;
3385
3386 req->poll.addr = READ_ONCE(sqe->addr);
0969e783
JA		 3387 return 0;
3388}
3389
221c5eb2
JA		 3390/*
3391 * Find a running poll command that matches one specified in sqe->addr,
3392 * and remove it if found.
3393 */
fc4df999 3394static int io_poll_remove(struct io_kiocb *req)
221c5eb2
JA		 3395{
3396 struct io_ring_ctx *ctx = req->ctx;
0969e783 3397 u64 addr;
47f46768 3398 int ret;
221c5eb2 3399
0969e783 3400 addr = req->poll.addr;
221c5eb2 3401 spin_lock_irq(&ctx->completion_lock);
0969e783 3402 ret = io_poll_cancel(ctx, addr);
221c5eb2
JA		 3403 spin_unlock_irq(&ctx->completion_lock);
3404
78e19bbe 3405 io_cqring_add_event(req, ret);
4e88d6e7
JA		 3406 if (ret < 0)
3407 req_set_fail_links(req);
e65ef56d 3408 io_put_req(req);
221c5eb2
JA		 3409 return 0;
3410}
3411
b0dd8a41 3412static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error)
221c5eb2 3413{
a197f664
JL		 3414 struct io_ring_ctx *ctx = req->ctx;
3415
8c838788 3416 req->poll.done = true;
b0dd8a41
JA		 3417 if (error)
3418 io_cqring_fill_event(req, error);
3419 else
3420 io_cqring_fill_event(req, mangle_poll(mask));
8c838788 3421 io_commit_cqring(ctx);
221c5eb2
JA		 3422}
3423
561fb04a 3424static void io_poll_complete_work(struct io_wq_work **workptr)
221c5eb2 3425{
561fb04a 3426 struct io_wq_work *work = *workptr;
221c5eb2
JA		 3427 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
3428 struct io_poll_iocb *poll = &req->poll;
3429 struct poll_table_struct pt = { ._key = poll->events };
3430 struct io_ring_ctx *ctx = req->ctx;
89723d0b 3431 struct io_kiocb *nxt = NULL;
221c5eb2 3432 __poll_t mask = 0;
b0dd8a41 3433 int ret = 0;
221c5eb2 3434
b0dd8a41 3435 if (work->flags & IO_WQ_WORK_CANCEL) {
561fb04a 3436 WRITE_ONCE(poll->canceled, true);
b0dd8a41
JA		 3437 ret = -ECANCELED;
3438 } else if (READ_ONCE(poll->canceled)) {
3439 ret = -ECANCELED;
3440 }
561fb04a 3441
b0dd8a41 3442 if (ret != -ECANCELED)
221c5eb2
JA		 3443 mask = vfs_poll(poll->file, &pt) & poll->events;
3444
3445 /*
3446 * Note that ->ki_cancel callers also delete iocb from active_reqs after
3447 * calling ->ki_cancel. We need the ctx_lock roundtrip here to
3448 * synchronize with them. In the cancellation case the list_del_init
3449 * itself is not actually needed, but harmless so we keep it in to
3450 * avoid further branches in the fast path.
3451 */
3452 spin_lock_irq(&ctx->completion_lock);
b0dd8a41 3453 if (!mask && ret != -ECANCELED) {
392edb45 3454 add_wait_queue(poll->head, &poll->wait);
221c5eb2
JA		 3455 spin_unlock_irq(&ctx->completion_lock);
3456 return;
3457 }
78076bb6 3458 hash_del(&req->hash_node);
b0dd8a41 3459 io_poll_complete(req, mask, ret);
221c5eb2
JA		 3460 spin_unlock_irq(&ctx->completion_lock);
3461
8c838788 3462 io_cqring_ev_posted(ctx);
89723d0b 3463
4e88d6e7
JA		 3464 if (ret < 0)
3465 req_set_fail_links(req);
ec9c02ad 3466 io_put_req_find_next(req, &nxt);
89723d0b 3467 if (nxt)
78912934 3468 io_wq_assign_next(workptr, nxt);
221c5eb2
JA		 3469}
3470
e94f141b
JA		 3471static void __io_poll_flush(struct io_ring_ctx *ctx, struct llist_node *nodes)
3472{
e94f141b 3473 struct io_kiocb *req, *tmp;
8237e045 3474 struct req_batch rb;
e94f141b 3475
c6ca97b3 3476 rb.to_free = rb.need_iter = 0;
e94f141b
JA		 3477 spin_lock_irq(&ctx->completion_lock);
3478 llist_for_each_entry_safe(req, tmp, nodes, llist_node) {
3479 hash_del(&req->hash_node);
3480 io_poll_complete(req, req->result, 0);
3481
8237e045
JA		 3482 if (refcount_dec_and_test(&req->refs) &&
3483 !io_req_multi_free(&rb, req)) {
3484 req->flags |= REQ_F_COMP_LOCKED;
3485 io_free_req(req);
e94f141b
JA		 3486 }
3487 }
3488 spin_unlock_irq(&ctx->completion_lock);
3489
3490 io_cqring_ev_posted(ctx);
8237e045 3491 io_free_req_many(ctx, &rb);
e94f141b
JA		 3492}
3493
3494static void io_poll_flush(struct io_wq_work **workptr)
3495{
3496 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
3497 struct llist_node *nodes;
3498
3499 nodes = llist_del_all(&req->ctx->poll_llist);
3500 if (nodes)
3501 __io_poll_flush(req->ctx, nodes);
3502}
3503
221c5eb2
JA		 3504static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
3505 void *key)
3506{
e944475e 3507 struct io_poll_iocb *poll = wait->private;
221c5eb2
JA		 3508 struct io_kiocb *req = container_of(poll, struct io_kiocb, poll);
3509 struct io_ring_ctx *ctx = req->ctx;
3510 __poll_t mask = key_to_poll(key);
221c5eb2
JA		 3511
3512 /* for instances that support it check for an event match first: */
8c838788
JA		 3513 if (mask && !(mask & poll->events))
3514 return 0;
221c5eb2 3515
392edb45 3516 list_del_init(&poll->wait.entry);
221c5eb2 3517
7c9e7f0f
JA		 3518 /*
3519 * Run completion inline if we can. We're using trylock here because
3520 * we are violating the completion_lock -> poll wq lock ordering.
3521 * If we have a link timeout we're going to need the completion_lock
3522 * for finalizing the request, mark us as having grabbed that already.
3523 */
e94f141b
JA		 3524 if (mask) {
3525 unsigned long flags;
221c5eb2 3526
e94f141b
JA		 3527 if (llist_empty(&ctx->poll_llist) &&
3528 spin_trylock_irqsave(&ctx->completion_lock, flags)) {
3529 hash_del(&req->hash_node);
3530 io_poll_complete(req, mask, 0);
3531 req->flags |= REQ_F_COMP_LOCKED;
3532 io_put_req(req);
3533 spin_unlock_irqrestore(&ctx->completion_lock, flags);
3534
3535 io_cqring_ev_posted(ctx);
3536 req = NULL;
3537 } else {
3538 req->result = mask;
3539 req->llist_node.next = NULL;
3540 /* if the list wasn't empty, we're done */
3541 if (!llist_add(&req->llist_node, &ctx->poll_llist))
3542 req = NULL;
3543 else
3544 req->work.func = io_poll_flush;
3545 }
221c5eb2 3546 }
e94f141b
JA		 3547 if (req)
3548 io_queue_async_work(req);
221c5eb2 3549
221c5eb2
JA		 3550 return 1;
3551}
3552
3553struct io_poll_table {
3554 struct poll_table_struct pt;
3555 struct io_kiocb *req;
3556 int error;
3557};
3558
3559static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
3560 struct poll_table_struct *p)
3561{
3562 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
3563
3564 if (unlikely(pt->req->poll.head)) {
3565 pt->error = -EINVAL;
3566 return;
3567 }
3568
3569 pt->error = 0;
3570 pt->req->poll.head = head;
392edb45 3571 add_wait_queue(head, &pt->req->poll.wait);
221c5eb2
JA		 3572}
3573
eac406c6
JA		 3574static void io_poll_req_insert(struct io_kiocb *req)
3575{
3576 struct io_ring_ctx *ctx = req->ctx;
78076bb6
JA		 3577 struct hlist_head *list;
3578
3579 list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
3580 hlist_add_head(&req->hash_node, list);
eac406c6
JA		 3581}
3582
3529d8c2 3583static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
221c5eb2
JA		 3584{
3585 struct io_poll_iocb *poll = &req->poll;
221c5eb2 3586 u16 events;
221c5eb2
JA		 3587
3588 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3589 return -EINVAL;
3590 if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
3591 return -EINVAL;
09bb8394
JA		 3592 if (!poll->file)
3593 return -EBADF;
221c5eb2 3594
221c5eb2
JA		 3595 events = READ_ONCE(sqe->poll_events);
3596 poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;
0969e783
JA		 3597 return 0;
3598}
3599
3600static int io_poll_add(struct io_kiocb *req, struct io_kiocb **nxt)
3601{
3602 struct io_poll_iocb *poll = &req->poll;
3603 struct io_ring_ctx *ctx = req->ctx;
3604 struct io_poll_table ipt;
3605 bool cancel = false;
3606 __poll_t mask;
0969e783
JA		 3607
3608 INIT_IO_WORK(&req->work, io_poll_complete_work);
78076bb6 3609 INIT_HLIST_NODE(&req->hash_node);
221c5eb2 3610
221c5eb2 3611 poll->head = NULL;
8c838788 3612 poll->done = false;
221c5eb2
JA		 3613 poll->canceled = false;
3614
3615 ipt.pt._qproc = io_poll_queue_proc;
3616 ipt.pt._key = poll->events;
3617 ipt.req = req;
3618 ipt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */
3619
3620 /* initialized the list so that we can do list_empty checks */
392edb45
JA		 3621 INIT_LIST_HEAD(&poll->wait.entry);
3622 init_waitqueue_func_entry(&poll->wait, io_poll_wake);
3623 poll->wait.private = poll;
221c5eb2 3624
36703247
JA		 3625 INIT_LIST_HEAD(&req->list);
3626
221c5eb2 3627 mask = vfs_poll(poll->file, &ipt.pt) & poll->events;
221c5eb2
JA		 3628
3629 spin_lock_irq(&ctx->completion_lock);
8c838788
JA		 3630 if (likely(poll->head)) {
3631 spin_lock(&poll->head->lock);
392edb45 3632 if (unlikely(list_empty(&poll->wait.entry))) {
8c838788
JA		 3633 if (ipt.error)
3634 cancel = true;
3635 ipt.error = 0;
3636 mask = 0;
3637 }
3638 if (mask || ipt.error)
392edb45 3639 list_del_init(&poll->wait.entry);
8c838788
JA		 3640 else if (cancel)
3641 WRITE_ONCE(poll->canceled, true);
3642 else if (!poll->done) /* actually waiting for an event */
eac406c6 3643 io_poll_req_insert(req);
8c838788
JA		 3644 spin_unlock(&poll->head->lock);
3645 }
3646 if (mask) { /* no async, we'd stolen it */
221c5eb2 3647 ipt.error = 0;
b0dd8a41 3648 io_poll_complete(req, mask, 0);
221c5eb2 3649 }
221c5eb2
JA		 3650 spin_unlock_irq(&ctx->completion_lock);
3651
8c838788
JA		 3652 if (mask) {
3653 io_cqring_ev_posted(ctx);
ec9c02ad 3654 io_put_req_find_next(req, nxt);
221c5eb2 3655 }
8c838788 3656 return ipt.error;
221c5eb2
JA		 3657}
3658
5262f567
JA		 3659static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
3660{
ad8a48ac
JA		 3661 struct io_timeout_data *data = container_of(timer,
3662 struct io_timeout_data, timer);
3663 struct io_kiocb *req = data->req;
3664 struct io_ring_ctx *ctx = req->ctx;
5262f567
JA		 3665 unsigned long flags;
3666
5262f567
JA		 3667 atomic_inc(&ctx->cq_timeouts);
3668
3669 spin_lock_irqsave(&ctx->completion_lock, flags);
ef03681a 3670 /*
11365043
JA		 3671 * We could be racing with timeout deletion. If the list is empty,
3672 * then timeout lookup already found it and will be handling it.
ef03681a 3673 */
842f9612 3674 if (!list_empty(&req->list)) {
11365043 3675 struct io_kiocb *prev;
5262f567 3676
11365043
JA		 3677 /*
3678 * Adjust the reqs sequence before the current one because it
d195a66e 3679 * will consume a slot in the cq_ring and the cq_tail
11365043
JA		 3680 * pointer will be increased, otherwise other timeout reqs may
3681 * return in advance without waiting for enough wait_nr.
3682 */
3683 prev = req;
3684 list_for_each_entry_continue_reverse(prev, &ctx->timeout_list, list)
3685 prev->sequence++;
11365043 3686 list_del_init(&req->list);
11365043 3687 }
5262f567 3688
78e19bbe 3689 io_cqring_fill_event(req, -ETIME);
5262f567
JA		 3690 io_commit_cqring(ctx);
3691 spin_unlock_irqrestore(&ctx->completion_lock, flags);
3692
3693 io_cqring_ev_posted(ctx);
4e88d6e7 3694 req_set_fail_links(req);
5262f567
JA		 3695 io_put_req(req);
3696 return HRTIMER_NORESTART;
3697}
3698
47f46768
JA		 3699static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
3700{
3701 struct io_kiocb *req;
3702 int ret = -ENOENT;
3703
3704 list_for_each_entry(req, &ctx->timeout_list, list) {
3705 if (user_data == req->user_data) {
3706 list_del_init(&req->list);
3707 ret = 0;
3708 break;
3709 }
3710 }
3711
3712 if (ret == -ENOENT)
3713 return ret;
3714
2d28390a 3715 ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
47f46768
JA		 3716 if (ret == -1)
3717 return -EALREADY;
3718
4e88d6e7 3719 req_set_fail_links(req);
47f46768
JA		 3720 io_cqring_fill_event(req, -ECANCELED);
3721 io_put_req(req);
3722 return 0;
3723}
3724
3529d8c2
JA		 3725static int io_timeout_remove_prep(struct io_kiocb *req,
3726 const struct io_uring_sqe *sqe)
b29472ee 3727{
b29472ee
JA		 3728 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3729 return -EINVAL;
3730 if (sqe->flags || sqe->ioprio || sqe->buf_index || sqe->len)
3731 return -EINVAL;
3732
3733 req->timeout.addr = READ_ONCE(sqe->addr);
3734 req->timeout.flags = READ_ONCE(sqe->timeout_flags);
3735 if (req->timeout.flags)
3736 return -EINVAL;
3737
b29472ee
JA		 3738 return 0;
3739}
3740
11365043
JA		 3741/*
3742 * Remove or update an existing timeout command
3743 */
fc4df999 3744static int io_timeout_remove(struct io_kiocb *req)
11365043
JA		 3745{
3746 struct io_ring_ctx *ctx = req->ctx;
47f46768 3747 int ret;
11365043 3748
11365043 3749 spin_lock_irq(&ctx->completion_lock);
b29472ee 3750 ret = io_timeout_cancel(ctx, req->timeout.addr);
11365043 3751
47f46768 3752 io_cqring_fill_event(req, ret);
11365043
JA		 3753 io_commit_cqring(ctx);
3754 spin_unlock_irq(&ctx->completion_lock);
5262f567 3755 io_cqring_ev_posted(ctx);
4e88d6e7
JA		 3756 if (ret < 0)
3757 req_set_fail_links(req);
ec9c02ad 3758 io_put_req(req);
11365043 3759 return 0;
5262f567
JA		 3760}
3761
3529d8c2 3762static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
2d28390a 3763 bool is_timeout_link)
5262f567 3764{
ad8a48ac 3765 struct io_timeout_data *data;
a41525ab 3766 unsigned flags;
5262f567 3767
ad8a48ac 3768 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5262f567 3769 return -EINVAL;
ad8a48ac 3770 if (sqe->ioprio || sqe->buf_index || sqe->len != 1)
a41525ab 3771 return -EINVAL;
2d28390a
JA		 3772 if (sqe->off && is_timeout_link)
3773 return -EINVAL;
a41525ab
JA		 3774 flags = READ_ONCE(sqe->timeout_flags);
3775 if (flags & ~IORING_TIMEOUT_ABS)
5262f567 3776 return -EINVAL;
bdf20073 3777
26a61679
JA		 3778 req->timeout.count = READ_ONCE(sqe->off);
3779
3529d8c2 3780 if (!req->io && io_alloc_async_ctx(req))
26a61679
JA		 3781 return -ENOMEM;
3782
3783 data = &req->io->timeout;
ad8a48ac 3784 data->req = req;
ad8a48ac
JA		 3785 req->flags |= REQ_F_TIMEOUT;
3786
3787 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
5262f567
JA		 3788 return -EFAULT;
3789
11365043 3790 if (flags & IORING_TIMEOUT_ABS)
ad8a48ac 3791 data->mode = HRTIMER_MODE_ABS;
11365043 3792 else
ad8a48ac 3793 data->mode = HRTIMER_MODE_REL;
11365043 3794
ad8a48ac
JA		 3795 hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode);
3796 return 0;
3797}
3798
fc4df999 3799static int io_timeout(struct io_kiocb *req)
ad8a48ac
JA		 3800{
3801 unsigned count;
3802 struct io_ring_ctx *ctx = req->ctx;
3803 struct io_timeout_data *data;
3804 struct list_head *entry;
3805 unsigned span = 0;
ad8a48ac 3806
2d28390a 3807 data = &req->io->timeout;
93bd25bb 3808
5262f567
JA		 3809 /*
3810 * sqe->off holds how many events that need to occur for this
93bd25bb
JA		 3811 * timeout event to be satisfied. If it isn't set, then this is
3812 * a pure timeout request, sequence isn't used.
5262f567 3813 */
26a61679 3814 count = req->timeout.count;
93bd25bb
JA		 3815 if (!count) {
3816 req->flags |= REQ_F_TIMEOUT_NOSEQ;
3817 spin_lock_irq(&ctx->completion_lock);
3818 entry = ctx->timeout_list.prev;
3819 goto add;
3820 }
5262f567
JA		 3821
3822 req->sequence = ctx->cached_sq_head + count - 1;
2d28390a 3823 data->seq_offset = count;
5262f567
JA		 3824
3825 /*
3826 * Insertion sort, ensuring the first entry in the list is always
3827 * the one we need first.
3828 */
5262f567
JA		 3829 spin_lock_irq(&ctx->completion_lock);
3830 list_for_each_prev(entry, &ctx->timeout_list) {
3831 struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, list);
5da0fb1a 3832 unsigned nxt_sq_head;
3833 long long tmp, tmp_nxt;
2d28390a 3834 u32 nxt_offset = nxt->io->timeout.seq_offset;
5262f567 3835
93bd25bb
JA		 3836 if (nxt->flags & REQ_F_TIMEOUT_NOSEQ)
3837 continue;
3838
5da0fb1a 3839 /*
3840 * Since cached_sq_head + count - 1 can overflow, use type long
3841 * long to store it.
3842 */
3843 tmp = (long long)ctx->cached_sq_head + count - 1;
cc42e0ac
PB		 3844 nxt_sq_head = nxt->sequence - nxt_offset + 1;
3845 tmp_nxt = (long long)nxt_sq_head + nxt_offset - 1;
5da0fb1a 3846
3847 /*
3848 * cached_sq_head may overflow, and it will never overflow twice
3849 * once there is some timeout req still be valid.
3850 */
3851 if (ctx->cached_sq_head < nxt_sq_head)
8b07a65a 3852 tmp += UINT_MAX;
5da0fb1a 3853
a1f58ba4 3854 if (tmp > tmp_nxt)
5262f567 3855 break;
a1f58ba4 3856
3857 /*
3858 * Sequence of reqs after the insert one and itself should
3859 * be adjusted because each timeout req consumes a slot.
3860 */
3861 span++;
3862 nxt->sequence++;
5262f567 3863 }
a1f58ba4 3864 req->sequence -= span;
93bd25bb 3865add:
5262f567 3866 list_add(&req->list, entry);
ad8a48ac
JA		 3867 data->timer.function = io_timeout_fn;
3868 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
5262f567 3869 spin_unlock_irq(&ctx->completion_lock);
5262f567
JA		 3870 return 0;
3871}
5262f567 3872
62755e35
JA		 3873static bool io_cancel_cb(struct io_wq_work *work, void *data)
3874{
3875 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
3876
3877 return req->user_data == (unsigned long) data;
3878}
3879
e977d6d3 3880static int io_async_cancel_one(struct io_ring_ctx *ctx, void *sqe_addr)
62755e35 3881{
62755e35 3882 enum io_wq_cancel cancel_ret;
62755e35
JA		 3883 int ret = 0;
3884
62755e35
JA		 3885 cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr);
3886 switch (cancel_ret) {
3887 case IO_WQ_CANCEL_OK:
3888 ret = 0;
3889 break;
3890 case IO_WQ_CANCEL_RUNNING:
3891 ret = -EALREADY;
3892 break;
3893 case IO_WQ_CANCEL_NOTFOUND:
3894 ret = -ENOENT;
3895 break;
3896 }
3897
e977d6d3
JA		 3898 return ret;
3899}
3900
47f46768
JA		 3901static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
3902 struct io_kiocb *req, __u64 sqe_addr,
b0dd8a41 3903 struct io_kiocb **nxt, int success_ret)
47f46768
JA		 3904{
3905 unsigned long flags;
3906 int ret;
3907
3908 ret = io_async_cancel_one(ctx, (void *) (unsigned long) sqe_addr);
3909 if (ret != -ENOENT) {
3910 spin_lock_irqsave(&ctx->completion_lock, flags);
3911 goto done;
3912 }
3913
3914 spin_lock_irqsave(&ctx->completion_lock, flags);
3915 ret = io_timeout_cancel(ctx, sqe_addr);
3916 if (ret != -ENOENT)
3917 goto done;
3918 ret = io_poll_cancel(ctx, sqe_addr);
3919done:
b0dd8a41
JA		 3920 if (!ret)
3921 ret = success_ret;
47f46768
JA		 3922 io_cqring_fill_event(req, ret);
3923 io_commit_cqring(ctx);
3924 spin_unlock_irqrestore(&ctx->completion_lock, flags);
3925 io_cqring_ev_posted(ctx);
3926
4e88d6e7
JA		 3927 if (ret < 0)
3928 req_set_fail_links(req);
47f46768
JA		 3929 io_put_req_find_next(req, nxt);
3930}
3931
3529d8c2
JA		 3932static int io_async_cancel_prep(struct io_kiocb *req,
3933 const struct io_uring_sqe *sqe)
e977d6d3 3934{
fbf23849 3935 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
e977d6d3
JA		 3936 return -EINVAL;
3937 if (sqe->flags || sqe->ioprio || sqe->off || sqe->len ||
3938 sqe->cancel_flags)
3939 return -EINVAL;
3940
fbf23849
JA		 3941 req->cancel.addr = READ_ONCE(sqe->addr);
3942 return 0;
3943}
3944
3945static int io_async_cancel(struct io_kiocb *req, struct io_kiocb **nxt)
3946{
3947 struct io_ring_ctx *ctx = req->ctx;
fbf23849
JA		 3948
3949 io_async_find_and_cancel(ctx, req, req->cancel.addr, nxt, 0);
5262f567
JA		 3950 return 0;
3951}
3952
05f3fb3c
JA		 3953static int io_files_update_prep(struct io_kiocb *req,
3954 const struct io_uring_sqe *sqe)
3955{
3956 if (sqe->flags || sqe->ioprio || sqe->rw_flags)
3957 return -EINVAL;
3958
3959 req->files_update.offset = READ_ONCE(sqe->off);
3960 req->files_update.nr_args = READ_ONCE(sqe->len);
3961 if (!req->files_update.nr_args)
3962 return -EINVAL;
3963 req->files_update.arg = READ_ONCE(sqe->addr);
3964 return 0;
3965}
3966
3967static int io_files_update(struct io_kiocb *req, bool force_nonblock)
3968{
3969 struct io_ring_ctx *ctx = req->ctx;
3970 struct io_uring_files_update up;
3971 int ret;
3972
f86cd20c 3973 if (force_nonblock)
05f3fb3c 3974 return -EAGAIN;
05f3fb3c
JA		 3975
3976 up.offset = req->files_update.offset;
3977 up.fds = req->files_update.arg;
3978
3979 mutex_lock(&ctx->uring_lock);
3980 ret = __io_sqe_files_update(ctx, &up, req->files_update.nr_args);
3981 mutex_unlock(&ctx->uring_lock);
3982
3983 if (ret < 0)
3984 req_set_fail_links(req);
3985 io_cqring_add_event(req, ret);
3986 io_put_req(req);
3987 return 0;
3988}
3989
3529d8c2
JA		 3990static int io_req_defer_prep(struct io_kiocb *req,
3991 const struct io_uring_sqe *sqe)
f67676d1 3992{
e781573e 3993 ssize_t ret = 0;
f67676d1 3994
f86cd20c
JA		 3995 if (io_op_defs[req->opcode].file_table) {
3996 ret = io_grab_files(req);
3997 if (unlikely(ret))
3998 return ret;
3999 }
4000
cccf0ee8
JA		 4001 io_req_work_grab_env(req, &io_op_defs[req->opcode]);
4002
d625c6ee 4003 switch (req->opcode) {
e781573e
JA		 4004 case IORING_OP_NOP:
4005 break;
f67676d1
JA		 4006 case IORING_OP_READV:
4007 case IORING_OP_READ_FIXED:
3a6820f2 4008 case IORING_OP_READ:
3529d8c2 4009 ret = io_read_prep(req, sqe, true);
f67676d1
JA		 4010 break;
4011 case IORING_OP_WRITEV:
4012 case IORING_OP_WRITE_FIXED:
3a6820f2 4013 case IORING_OP_WRITE:
3529d8c2 4014 ret = io_write_prep(req, sqe, true);
f67676d1 4015 break;
0969e783 4016 case IORING_OP_POLL_ADD:
3529d8c2 4017 ret = io_poll_add_prep(req, sqe);
0969e783
JA		 4018 break;
4019 case IORING_OP_POLL_REMOVE:
3529d8c2 4020 ret = io_poll_remove_prep(req, sqe);
0969e783 4021 break;
8ed8d3c3 4022 case IORING_OP_FSYNC:
3529d8c2 4023 ret = io_prep_fsync(req, sqe);
8ed8d3c3
JA		 4024 break;
4025 case IORING_OP_SYNC_FILE_RANGE:
3529d8c2 4026 ret = io_prep_sfr(req, sqe);
8ed8d3c3 4027 break;
03b1230c 4028 case IORING_OP_SENDMSG:
fddaface 4029 case IORING_OP_SEND:
3529d8c2 4030 ret = io_sendmsg_prep(req, sqe);
03b1230c
JA		 4031 break;
4032 case IORING_OP_RECVMSG:
fddaface 4033 case IORING_OP_RECV:
3529d8c2 4034 ret = io_recvmsg_prep(req, sqe);
03b1230c 4035 break;
f499a021 4036 case IORING_OP_CONNECT:
3529d8c2 4037 ret = io_connect_prep(req, sqe);
f499a021 4038 break;
2d28390a 4039 case IORING_OP_TIMEOUT:
3529d8c2 4040 ret = io_timeout_prep(req, sqe, false);
b7bb4f7d 4041 break;
b29472ee 4042 case IORING_OP_TIMEOUT_REMOVE:
3529d8c2 4043 ret = io_timeout_remove_prep(req, sqe);
b29472ee 4044 break;
fbf23849 4045 case IORING_OP_ASYNC_CANCEL:
3529d8c2 4046 ret = io_async_cancel_prep(req, sqe);
fbf23849 4047 break;
2d28390a 4048 case IORING_OP_LINK_TIMEOUT:
3529d8c2 4049 ret = io_timeout_prep(req, sqe, true);
b7bb4f7d 4050 break;
8ed8d3c3 4051 case IORING_OP_ACCEPT:
3529d8c2 4052 ret = io_accept_prep(req, sqe);
8ed8d3c3 4053 break;
d63d1b5e
JA		 4054 case IORING_OP_FALLOCATE:
4055 ret = io_fallocate_prep(req, sqe);
4056 break;
15b71abe
JA		 4057 case IORING_OP_OPENAT:
4058 ret = io_openat_prep(req, sqe);
4059 break;
b5dba59e
JA		 4060 case IORING_OP_CLOSE:
4061 ret = io_close_prep(req, sqe);
4062 break;
05f3fb3c
JA		 4063 case IORING_OP_FILES_UPDATE:
4064 ret = io_files_update_prep(req, sqe);
4065 break;
eddc7ef5
JA		 4066 case IORING_OP_STATX:
4067 ret = io_statx_prep(req, sqe);
4068 break;
4840e418
JA		 4069 case IORING_OP_FADVISE:
4070 ret = io_fadvise_prep(req, sqe);
4071 break;
c1ca757b
JA		 4072 case IORING_OP_MADVISE:
4073 ret = io_madvise_prep(req, sqe);
4074 break;
cebdb986
JA		 4075 case IORING_OP_OPENAT2:
4076 ret = io_openat2_prep(req, sqe);
4077 break;
f67676d1 4078 default:
e781573e
JA		 4079 printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
4080 req->opcode);
4081 ret = -EINVAL;
b7bb4f7d 4082 break;
f67676d1
JA		 4083 }
4084
b7bb4f7d 4085 return ret;
f67676d1
JA		 4086}
4087
3529d8c2 4088static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe)
de0617e4 4089{
a197f664 4090 struct io_ring_ctx *ctx = req->ctx;
f67676d1 4091 int ret;
de0617e4 4092
9d858b21
BL		 4093 /* Still need defer if there is pending req in defer list. */
4094 if (!req_need_defer(req) && list_empty(&ctx->defer_list))
de0617e4
JA		 4095 return 0;
4096
3529d8c2 4097 if (!req->io && io_alloc_async_ctx(req))
de0617e4
JA		 4098 return -EAGAIN;
4099
3529d8c2 4100 ret = io_req_defer_prep(req, sqe);
b7bb4f7d 4101 if (ret < 0)
2d28390a 4102 return ret;
2d28390a 4103
de0617e4 4104 spin_lock_irq(&ctx->completion_lock);
9d858b21 4105 if (!req_need_defer(req) && list_empty(&ctx->defer_list)) {
de0617e4 4106 spin_unlock_irq(&ctx->completion_lock);
de0617e4
JA		 4107 return 0;
4108 }
4109
915967f6 4110 trace_io_uring_defer(ctx, req, req->user_data);
de0617e4
JA		 4111 list_add_tail(&req->list, &ctx->defer_list);
4112 spin_unlock_irq(&ctx->completion_lock);
4113 return -EIOCBQUEUED;
4114}
4115
3529d8c2
JA		 4116static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
4117 struct io_kiocb **nxt, bool force_nonblock)
2b188cc1 4118{
a197f664 4119 struct io_ring_ctx *ctx = req->ctx;
d625c6ee 4120 int ret;
2b188cc1 4121
d625c6ee 4122 switch (req->opcode) {
2b188cc1 4123 case IORING_OP_NOP:
78e19bbe 4124 ret = io_nop(req);
2b188cc1
JA		 4125 break;
4126 case IORING_OP_READV:
edafccee 4127 case IORING_OP_READ_FIXED:
3a6820f2 4128 case IORING_OP_READ:
3529d8c2
JA		 4129 if (sqe) {
4130 ret = io_read_prep(req, sqe, force_nonblock);
4131 if (ret < 0)
4132 break;
4133 }
267bc904 4134 ret = io_read(req, nxt, force_nonblock);
edafccee 4135 break;
3529d8c2 4136 case IORING_OP_WRITEV:
edafccee 4137 case IORING_OP_WRITE_FIXED:
3a6820f2 4138 case IORING_OP_WRITE:
3529d8c2
JA		 4139 if (sqe) {
4140 ret = io_write_prep(req, sqe, force_nonblock);
4141 if (ret < 0)
4142 break;
4143 }
267bc904 4144 ret = io_write(req, nxt, force_nonblock);
2b188cc1 4145 break;
c992fe29 4146 case IORING_OP_FSYNC:
3529d8c2
JA		 4147 if (sqe) {
4148 ret = io_prep_fsync(req, sqe);
4149 if (ret < 0)
4150 break;
4151 }
fc4df999 4152 ret = io_fsync(req, nxt, force_nonblock);
c992fe29 4153 break;
221c5eb2 4154 case IORING_OP_POLL_ADD:
3529d8c2
JA		 4155 if (sqe) {
4156 ret = io_poll_add_prep(req, sqe);
4157 if (ret)
4158 break;
4159 }
fc4df999 4160 ret = io_poll_add(req, nxt);
221c5eb2
JA		 4161 break;
4162 case IORING_OP_POLL_REMOVE:
3529d8c2
JA		 4163 if (sqe) {
4164 ret = io_poll_remove_prep(req, sqe);
4165 if (ret < 0)
4166 break;
4167 }
fc4df999 4168 ret = io_poll_remove(req);
221c5eb2 4169 break;
5d17b4a4 4170 case IORING_OP_SYNC_FILE_RANGE:
3529d8c2
JA		 4171 if (sqe) {
4172 ret = io_prep_sfr(req, sqe);
4173 if (ret < 0)
4174 break;
4175 }
fc4df999 4176 ret = io_sync_file_range(req, nxt, force_nonblock);
5d17b4a4 4177 break;
0fa03c62 4178 case IORING_OP_SENDMSG:
fddaface 4179 case IORING_OP_SEND:
3529d8c2
JA		 4180 if (sqe) {
4181 ret = io_sendmsg_prep(req, sqe);
4182 if (ret < 0)
4183 break;
4184 }
fddaface
JA		 4185 if (req->opcode == IORING_OP_SENDMSG)
4186 ret = io_sendmsg(req, nxt, force_nonblock);
4187 else
4188 ret = io_send(req, nxt, force_nonblock);
0fa03c62 4189 break;
aa1fa28f 4190 case IORING_OP_RECVMSG:
fddaface 4191 case IORING_OP_RECV:
3529d8c2
JA		 4192 if (sqe) {
4193 ret = io_recvmsg_prep(req, sqe);
4194 if (ret)
4195 break;
4196 }
fddaface
JA		 4197 if (req->opcode == IORING_OP_RECVMSG)
4198 ret = io_recvmsg(req, nxt, force_nonblock);
4199 else
4200 ret = io_recv(req, nxt, force_nonblock);
aa1fa28f 4201 break;
5262f567 4202 case IORING_OP_TIMEOUT:
3529d8c2
JA		 4203 if (sqe) {
4204 ret = io_timeout_prep(req, sqe, false);
4205 if (ret)
4206 break;
4207 }
fc4df999 4208 ret = io_timeout(req);
5262f567 4209 break;
11365043 4210 case IORING_OP_TIMEOUT_REMOVE:
3529d8c2
JA		 4211 if (sqe) {
4212 ret = io_timeout_remove_prep(req, sqe);
4213 if (ret)
4214 break;
4215 }
fc4df999 4216 ret = io_timeout_remove(req);
11365043 4217 break;
17f2fe35 4218 case IORING_OP_ACCEPT:
3529d8c2
JA		 4219 if (sqe) {
4220 ret = io_accept_prep(req, sqe);
4221 if (ret)
4222 break;
4223 }
fc4df999 4224 ret = io_accept(req, nxt, force_nonblock);
17f2fe35 4225 break;
f8e85cf2 4226 case IORING_OP_CONNECT:
3529d8c2
JA		 4227 if (sqe) {
4228 ret = io_connect_prep(req, sqe);
4229 if (ret)
4230 break;
4231 }
fc4df999 4232 ret = io_connect(req, nxt, force_nonblock);
f8e85cf2 4233 break;
62755e35 4234 case IORING_OP_ASYNC_CANCEL:
3529d8c2
JA		 4235 if (sqe) {
4236 ret = io_async_cancel_prep(req, sqe);
4237 if (ret)
4238 break;
4239 }
fc4df999 4240 ret = io_async_cancel(req, nxt);
62755e35 4241 break;
d63d1b5e
JA		 4242 case IORING_OP_FALLOCATE:
4243 if (sqe) {
4244 ret = io_fallocate_prep(req, sqe);
4245 if (ret)
4246 break;
4247 }
4248 ret = io_fallocate(req, nxt, force_nonblock);
4249 break;
15b71abe
JA		 4250 case IORING_OP_OPENAT:
4251 if (sqe) {
4252 ret = io_openat_prep(req, sqe);
4253 if (ret)
4254 break;
4255 }
4256 ret = io_openat(req, nxt, force_nonblock);
4257 break;
b5dba59e
JA		 4258 case IORING_OP_CLOSE:
4259 if (sqe) {
4260 ret = io_close_prep(req, sqe);
4261 if (ret)
4262 break;
4263 }
4264 ret = io_close(req, nxt, force_nonblock);
4265 break;
05f3fb3c
JA		 4266 case IORING_OP_FILES_UPDATE:
4267 if (sqe) {
4268 ret = io_files_update_prep(req, sqe);
4269 if (ret)
4270 break;
4271 }
4272 ret = io_files_update(req, force_nonblock);
4273 break;
eddc7ef5
JA		 4274 case IORING_OP_STATX:
4275 if (sqe) {
4276 ret = io_statx_prep(req, sqe);
4277 if (ret)
4278 break;
4279 }
4280 ret = io_statx(req, nxt, force_nonblock);
4281 break;
4840e418
JA		 4282 case IORING_OP_FADVISE:
4283 if (sqe) {
4284 ret = io_fadvise_prep(req, sqe);
4285 if (ret)
4286 break;
4287 }
4288 ret = io_fadvise(req, nxt, force_nonblock);
4289 break;
c1ca757b
JA		 4290 case IORING_OP_MADVISE:
4291 if (sqe) {
4292 ret = io_madvise_prep(req, sqe);
4293 if (ret)
4294 break;
4295 }
4296 ret = io_madvise(req, nxt, force_nonblock);
4297 break;
cebdb986
JA		 4298 case IORING_OP_OPENAT2:
4299 if (sqe) {
4300 ret = io_openat2_prep(req, sqe);
4301 if (ret)
4302 break;
4303 }
4304 ret = io_openat2(req, nxt, force_nonblock);
4305 break;
2b188cc1
JA		 4306 default:
4307 ret = -EINVAL;
4308 break;
4309 }
4310
def596e9
JA		 4311 if (ret)
4312 return ret;
4313
4314 if (ctx->flags & IORING_SETUP_IOPOLL) {
11ba820b
JA		 4315 const bool in_async = io_wq_current_is_worker();
4316
9e645e11 4317 if (req->result == -EAGAIN)
def596e9
JA		 4318 return -EAGAIN;
4319
11ba820b
JA		 4320 /* workqueue context doesn't hold uring_lock, grab it now */
4321 if (in_async)
4322 mutex_lock(&ctx->uring_lock);
4323
def596e9 4324 io_iopoll_req_issued(req);
11ba820b
JA		 4325
4326 if (in_async)
4327 mutex_unlock(&ctx->uring_lock);
def596e9
JA		 4328 }
4329
4330 return 0;
2b188cc1
JA		 4331}
4332
561fb04a 4333static void io_wq_submit_work(struct io_wq_work **workptr)
2b188cc1 4334{
561fb04a 4335 struct io_wq_work *work = *workptr;
2b188cc1 4336 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
561fb04a
JA		 4337 struct io_kiocb *nxt = NULL;
4338 int ret = 0;
2b188cc1 4339
0c9d5ccd
JA		 4340 /* if NO_CANCEL is set, we must still run the work */
4341 if ((work->flags & (IO_WQ_WORK_CANCEL|IO_WQ_WORK_NO_CANCEL)) ==
4342 IO_WQ_WORK_CANCEL) {
561fb04a 4343 ret = -ECANCELED;
0c9d5ccd 4344 }
31b51510 4345
561fb04a 4346 if (!ret) {
cf6fd4bd
PB		 4347 req->has_user = (work->flags & IO_WQ_WORK_HAS_MM) != 0;
4348 req->in_async = true;
561fb04a 4349 do {
3529d8c2 4350 ret = io_issue_sqe(req, NULL, &nxt, false);
561fb04a
JA		 4351 /*
4352 * We can get EAGAIN for polled IO even though we're
4353 * forcing a sync submission from here, since we can't
4354 * wait for request slots on the block side.
4355 */
4356 if (ret != -EAGAIN)
4357 break;
4358 cond_resched();
4359 } while (1);
4360 }
31b51510 4361
561fb04a 4362 /* drop submission reference */
ec9c02ad 4363 io_put_req(req);
817869d2 4364
561fb04a 4365 if (ret) {
4e88d6e7 4366 req_set_fail_links(req);
78e19bbe 4367 io_cqring_add_event(req, ret);
817869d2 4368 io_put_req(req);
edafccee 4369 }
2b188cc1 4370
561fb04a 4371 /* if a dependent link is ready, pass it back */
78912934
JA		 4372 if (!ret && nxt)
4373 io_wq_assign_next(workptr, nxt);
2b188cc1
JA		 4374}
4375
15b71abe 4376static int io_req_needs_file(struct io_kiocb *req, int fd)
09bb8394 4377{
d3656344 4378 if (!io_op_defs[req->opcode].needs_file)
9e3aa61a 4379 return 0;
d3656344
JA		 4380 if (fd == -1 && io_op_defs[req->opcode].fd_non_neg)
4381 return 0;
4382 return 1;
09bb8394
JA		 4383}
4384
65e19f54
JA		 4385static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
4386 int index)
4387{
4388 struct fixed_file_table *table;
4389
05f3fb3c
JA		 4390 table = &ctx->file_data->table[index >> IORING_FILE_TABLE_SHIFT];
4391 return table->files[index & IORING_FILE_TABLE_MASK];;
65e19f54
JA		 4392}
4393
3529d8c2
JA		 4394static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req,
4395 const struct io_uring_sqe *sqe)
09bb8394 4396{
a197f664 4397 struct io_ring_ctx *ctx = req->ctx;
09bb8394 4398 unsigned flags;
d3656344 4399 int fd;
09bb8394 4400
3529d8c2
JA		 4401 flags = READ_ONCE(sqe->flags);
4402 fd = READ_ONCE(sqe->fd);
09bb8394 4403
d3656344
JA		 4404 if (!io_req_needs_file(req, fd))
4405 return 0;
09bb8394
JA		 4406
4407 if (flags & IOSQE_FIXED_FILE) {
05f3fb3c 4408 if (unlikely(!ctx->file_data ||
09bb8394
JA		 4409 (unsigned) fd >= ctx->nr_user_files))
4410 return -EBADF;
b7620121 4411 fd = array_index_nospec(fd, ctx->nr_user_files);
65e19f54
JA		 4412 req->file = io_file_from_index(ctx, fd);
4413 if (!req->file)
08a45173 4414 return -EBADF;
09bb8394 4415 req->flags |= REQ_F_FIXED_FILE;
05f3fb3c 4416 percpu_ref_get(&ctx->file_data->refs);
09bb8394 4417 } else {
cf6fd4bd 4418 if (req->needs_fixed_file)
09bb8394 4419 return -EBADF;
c826bd7a 4420 trace_io_uring_file_get(ctx, fd);
09bb8394
JA		 4421 req->file = io_file_get(state, fd);
4422 if (unlikely(!req->file))
4423 return -EBADF;
4424 }
4425
4426 return 0;
4427}
4428
a197f664 4429static int io_grab_files(struct io_kiocb *req)
fcb323cc
JA		 4430{
4431 int ret = -EBADF;
a197f664 4432 struct io_ring_ctx *ctx = req->ctx;
fcb323cc 4433
f86cd20c
JA		 4434 if (req->work.files)
4435 return 0;
b14cca0c 4436 if (!ctx->ring_file)
b5dba59e
JA		 4437 return -EBADF;
4438
fcb323cc
JA		 4439 rcu_read_lock();
4440 spin_lock_irq(&ctx->inflight_lock);
4441 /*
4442 * We use the f_ops->flush() handler to ensure that we can flush
4443 * out work accessing these files if the fd is closed. Check if
4444 * the fd has changed since we started down this path, and disallow
4445 * this operation if it has.
4446 */
b14cca0c 4447 if (fcheck(ctx->ring_fd) == ctx->ring_file) {
fcb323cc
JA		 4448 list_add(&req->inflight_entry, &ctx->inflight_list);
4449 req->flags |= REQ_F_INFLIGHT;
4450 req->work.files = current->files;
4451 ret = 0;
4452 }
4453 spin_unlock_irq(&ctx->inflight_lock);
4454 rcu_read_unlock();
4455
4456 return ret;
4457}
4458
2665abfd 4459static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
2b188cc1 4460{
ad8a48ac
JA		 4461 struct io_timeout_data *data = container_of(timer,
4462 struct io_timeout_data, timer);
4463 struct io_kiocb *req = data->req;
2665abfd
JA		 4464 struct io_ring_ctx *ctx = req->ctx;
4465 struct io_kiocb *prev = NULL;
4466 unsigned long flags;
2665abfd
JA		 4467
4468 spin_lock_irqsave(&ctx->completion_lock, flags);
4469
4470 /*
4471 * We don't expect the list to be empty, that will only happen if we
4472 * race with the completion of the linked work.
4473 */
4493233e
PB		 4474 if (!list_empty(&req->link_list)) {
4475 prev = list_entry(req->link_list.prev, struct io_kiocb,
4476 link_list);
5d960724 4477 if (refcount_inc_not_zero(&prev->refs)) {
4493233e 4478 list_del_init(&req->link_list);
5d960724
JA		 4479 prev->flags &= ~REQ_F_LINK_TIMEOUT;
4480 } else
76a46e06 4481 prev = NULL;
2665abfd
JA		 4482 }
4483
4484 spin_unlock_irqrestore(&ctx->completion_lock, flags);
4485
4486 if (prev) {
4e88d6e7 4487 req_set_fail_links(prev);
b0dd8a41
JA		 4488 io_async_find_and_cancel(ctx, req, prev->user_data, NULL,
4489 -ETIME);
76a46e06 4490 io_put_req(prev);
47f46768
JA		 4491 } else {
4492 io_cqring_add_event(req, -ETIME);
4493 io_put_req(req);
2665abfd 4494 }
2665abfd
JA		 4495 return HRTIMER_NORESTART;
4496}
4497
ad8a48ac 4498static void io_queue_linked_timeout(struct io_kiocb *req)
2665abfd 4499{
76a46e06 4500 struct io_ring_ctx *ctx = req->ctx;
2665abfd 4501
76a46e06
JA		 4502 /*
4503 * If the list is now empty, then our linked request finished before
4504 * we got a chance to setup the timer
4505 */
4506 spin_lock_irq(&ctx->completion_lock);
4493233e 4507 if (!list_empty(&req->link_list)) {
2d28390a 4508 struct io_timeout_data *data = &req->io->timeout;
94ae5e77 4509
ad8a48ac
JA		 4510 data->timer.function = io_link_timeout_fn;
4511 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
4512 data->mode);
2665abfd 4513 }
76a46e06 4514 spin_unlock_irq(&ctx->completion_lock);
2665abfd 4515
2665abfd 4516 /* drop submission reference */
76a46e06
JA		 4517 io_put_req(req);
4518}
2665abfd 4519
ad8a48ac 4520static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
2665abfd
JA		 4521{
4522 struct io_kiocb *nxt;
4523
4524 if (!(req->flags & REQ_F_LINK))
4525 return NULL;
4526
4493233e
PB		 4527 nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb,
4528 link_list);
d625c6ee 4529 if (!nxt || nxt->opcode != IORING_OP_LINK_TIMEOUT)
76a46e06 4530 return NULL;
2665abfd 4531
76a46e06 4532 req->flags |= REQ_F_LINK_TIMEOUT;
76a46e06 4533 return nxt;
2665abfd
JA		 4534}
4535
3529d8c2 4536static void __io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2b188cc1 4537{
4a0a7a18 4538 struct io_kiocb *linked_timeout;
f9bd67f6 4539 struct io_kiocb *nxt = NULL;
e0c5c576 4540 int ret;
2b188cc1 4541
4a0a7a18
JA		 4542again:
4543 linked_timeout = io_prep_linked_timeout(req);
4544
3529d8c2 4545 ret = io_issue_sqe(req, sqe, &nxt, true);
491381ce
JA		 4546
4547 /*
4548 * We async punt it if the file wasn't marked NOWAIT, or if the file
4549 * doesn't support non-blocking read/write attempts
4550 */
4551 if (ret == -EAGAIN && (!(req->flags & REQ_F_NOWAIT) ||
4552 (req->flags & REQ_F_MUST_PUNT))) {
86a761f8 4553punt:
f86cd20c 4554 if (io_op_defs[req->opcode].file_table) {
bbad27b2
PB		 4555 ret = io_grab_files(req);
4556 if (ret)
4557 goto err;
2b188cc1 4558 }
bbad27b2
PB		 4559
4560 /*
4561 * Queued up for async execution, worker will release
4562 * submit reference when the iocb is actually submitted.
4563 */
4564 io_queue_async_work(req);
4a0a7a18 4565 goto done_req;
2b188cc1 4566 }
e65ef56d 4567
fcb323cc 4568err:
76a46e06 4569 /* drop submission reference */
ec9c02ad 4570 io_put_req(req);
e65ef56d 4571
f9bd67f6 4572 if (linked_timeout) {
76a46e06 4573 if (!ret)
f9bd67f6 4574 io_queue_linked_timeout(linked_timeout);
76a46e06 4575 else
f9bd67f6 4576 io_put_req(linked_timeout);
76a46e06
JA		 4577 }
4578
e65ef56d 4579 /* and drop final reference, if we failed */
9e645e11 4580 if (ret) {
78e19bbe 4581 io_cqring_add_event(req, ret);
4e88d6e7 4582 req_set_fail_links(req);
e65ef56d 4583 io_put_req(req);
9e645e11 4584 }
4a0a7a18
JA		 4585done_req:
4586 if (nxt) {
4587 req = nxt;
4588 nxt = NULL;
86a761f8
PB		 4589
4590 if (req->flags & REQ_F_FORCE_ASYNC)
4591 goto punt;
4a0a7a18
JA		 4592 goto again;
4593 }
2b188cc1
JA		 4594}
4595
3529d8c2 4596static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4fe2c963
JL		 4597{
4598 int ret;
4599
3529d8c2 4600 ret = io_req_defer(req, sqe);
4fe2c963
JL		 4601 if (ret) {
4602 if (ret != -EIOCBQUEUED) {
1118591a 4603fail_req:
78e19bbe 4604 io_cqring_add_event(req, ret);
4e88d6e7 4605 req_set_fail_links(req);
78e19bbe 4606 io_double_put_req(req);
4fe2c963 4607 }
2550878f 4608 } else if (req->flags & REQ_F_FORCE_ASYNC) {
1118591a
PB		 4609 ret = io_req_defer_prep(req, sqe);
4610 if (unlikely(ret < 0))
4611 goto fail_req;
ce35a47a
JA		 4612 /*
4613 * Never try inline submit of IOSQE_ASYNC is set, go straight
4614 * to async execution.
4615 */
4616 req->work.flags |= IO_WQ_WORK_CONCURRENT;
4617 io_queue_async_work(req);
4618 } else {
3529d8c2 4619 __io_queue_sqe(req, sqe);
ce35a47a 4620 }
4fe2c963
JL		 4621}
4622
1b4a51b6 4623static inline void io_queue_link_head(struct io_kiocb *req)
4fe2c963 4624{
94ae5e77 4625 if (unlikely(req->flags & REQ_F_FAIL_LINK)) {
1b4a51b6
PB		 4626 io_cqring_add_event(req, -ECANCELED);
4627 io_double_put_req(req);
4628 } else
3529d8c2 4629 io_queue_sqe(req, NULL);
4fe2c963
JL		 4630}
4631
4e88d6e7 4632#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \
ce35a47a 4633 IOSQE_IO_HARDLINK | IOSQE_ASYNC)
9e645e11 4634
3529d8c2
JA		 4635static bool io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
4636 struct io_submit_state *state, struct io_kiocb **link)
9e645e11 4637{
75c6a039 4638 const struct cred *old_creds = NULL;
a197f664 4639 struct io_ring_ctx *ctx = req->ctx;
32fe525b 4640 unsigned int sqe_flags;
75c6a039 4641 int ret, id;
9e645e11 4642
32fe525b
PB		 4643 sqe_flags = READ_ONCE(sqe->flags);
4644
9e645e11 4645 /* enforce forwards compatibility on users */
32fe525b 4646 if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) {
9e645e11 4647 ret = -EINVAL;
196be95c 4648 goto err_req;
9e645e11 4649 }
75c6a039
JA		 4650
4651 id = READ_ONCE(sqe->personality);
4652 if (id) {
4653 const struct cred *personality_creds;
4654
4655 personality_creds = idr_find(&ctx->personality_idr, id);
4656 if (unlikely(!personality_creds)) {
4657 ret = -EINVAL;
4658 goto err_req;
4659 }
4660 old_creds = override_creds(personality_creds);
4661 }
4662
6b47ee6e
PB		 4663 /* same numerical values with corresponding REQ_F_*, safe to copy */
4664 req->flags |= sqe_flags & (IOSQE_IO_DRAIN|IOSQE_IO_HARDLINK|
4665 IOSQE_ASYNC);
9e645e11 4666
3529d8c2 4667 ret = io_req_set_file(state, req, sqe);
9e645e11
JA		 4668 if (unlikely(ret)) {
4669err_req:
78e19bbe
JA		 4670 io_cqring_add_event(req, ret);
4671 io_double_put_req(req);
75c6a039
JA		 4672 if (old_creds)
4673 revert_creds(old_creds);
2e6e1fde 4674 return false;
9e645e11
JA		 4675 }
4676
9e645e11
JA		 4677 /*
4678 * If we already have a head request, queue this one for async
4679 * submittal once the head completes. If we don't have a head but
4680 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
4681 * submitted sync once the chain is complete. If none of those
4682 * conditions are true (normal request), then just queue it.
4683 */
4684 if (*link) {
9d76377f 4685 struct io_kiocb *head = *link;
9e645e11 4686
8cdf2193
PB		 4687 /*
4688 * Taking sequential execution of a link, draining both sides
4689 * of the link also fullfils IOSQE_IO_DRAIN semantics for all
4690 * requests in the link. So, it drains the head and the
4691 * next after the link request. The last one is done via
4692 * drain_next flag to persist the effect across calls.
4693 */
711be031
PB		 4694 if (sqe_flags & IOSQE_IO_DRAIN) {
4695 head->flags |= REQ_F_IO_DRAIN;
4696 ctx->drain_next = 1;
4697 }
b7bb4f7d 4698 if (io_alloc_async_ctx(req)) {
9e645e11
JA		 4699 ret = -EAGAIN;
4700 goto err_req;
4701 }
4702
3529d8c2 4703 ret = io_req_defer_prep(req, sqe);
2d28390a 4704 if (ret) {
4e88d6e7 4705 /* fail even hard links since we don't submit */
9d76377f 4706 head->flags |= REQ_F_FAIL_LINK;
f67676d1 4707 goto err_req;
2d28390a 4708 }
9d76377f
PB		 4709 trace_io_uring_link(ctx, req, head);
4710 list_add_tail(&req->link_list, &head->link_list);
32fe525b
PB		 4711
4712 /* last request of a link, enqueue the link */
4713 if (!(sqe_flags & (IOSQE_IO_LINK|IOSQE_IO_HARDLINK))) {
4714 io_queue_link_head(head);
4715 *link = NULL;
4716 }
9e645e11 4717 } else {
711be031
PB		 4718 if (unlikely(ctx->drain_next)) {
4719 req->flags |= REQ_F_IO_DRAIN;
4720 req->ctx->drain_next = 0;
4721 }
4722 if (sqe_flags & (IOSQE_IO_LINK|IOSQE_IO_HARDLINK)) {
4723 req->flags |= REQ_F_LINK;
711be031
PB		 4724 INIT_LIST_HEAD(&req->link_list);
4725 ret = io_req_defer_prep(req, sqe);
4726 if (ret)
4727 req->flags |= REQ_F_FAIL_LINK;
4728 *link = req;
4729 } else {
4730 io_queue_sqe(req, sqe);
4731 }
9e645e11 4732 }
2e6e1fde 4733
75c6a039
JA		 4734 if (old_creds)
4735 revert_creds(old_creds);
2e6e1fde 4736 return true;
9e645e11
JA		 4737}
4738
9a56a232
JA		 4739/*
4740 * Batched submission is done, ensure local IO is flushed out.
4741 */
4742static void io_submit_state_end(struct io_submit_state *state)
4743{
4744 blk_finish_plug(&state->plug);
3d6770fb 4745 io_file_put(state);
2579f913
JA		 4746 if (state->free_reqs)
4747 kmem_cache_free_bulk(req_cachep, state->free_reqs,
4748 &state->reqs[state->cur_req]);
9a56a232
JA		 4749}
4750
4751/*
4752 * Start submission side cache.
4753 */
4754static void io_submit_state_start(struct io_submit_state *state,
22efde59 4755 unsigned int max_ios)
9a56a232
JA		 4756{
4757 blk_start_plug(&state->plug);
2579f913 4758 state->free_reqs = 0;
9a56a232
JA		 4759 state->file = NULL;
4760 state->ios_left = max_ios;
4761}
4762
2b188cc1
JA		 4763static void io_commit_sqring(struct io_ring_ctx *ctx)
4764{
75b28aff 4765 struct io_rings *rings = ctx->rings;
2b188cc1 4766
caf582c6
PB		 4767 /*
4768 * Ensure any loads from the SQEs are done at this point,
4769 * since once we write the new head, the application could
4770 * write new data to them.
4771 */
4772 smp_store_release(&rings->sq.head, ctx->cached_sq_head);
2b188cc1
JA		 4773}
4774
2b188cc1 4775/*
3529d8c2 4776 * Fetch an sqe, if one is available. Note that sqe_ptr will point to memory
2b188cc1
JA		 4777 * that is mapped by userspace. This means that care needs to be taken to
4778 * ensure that reads are stable, as we cannot rely on userspace always
4779 * being a good citizen. If members of the sqe are validated and then later
4780 * used, it's important that those reads are done through READ_ONCE() to
4781 * prevent a re-load down the line.
4782 */
3529d8c2
JA		 4783static bool io_get_sqring(struct io_ring_ctx *ctx, struct io_kiocb *req,
4784 const struct io_uring_sqe **sqe_ptr)
2b188cc1 4785{
75b28aff 4786 u32 *sq_array = ctx->sq_array;
2b188cc1
JA		 4787 unsigned head;
4788
4789 /*
4790 * The cached sq head (or cq tail) serves two purposes:
4791 *
4792 * 1) allows us to batch the cost of updating the user visible
4793 * head updates.
4794 * 2) allows the kernel side to track the head on its own, even
4795 * though the application is the one updating it.
4796 */
ee7d46d9 4797 head = READ_ONCE(sq_array[ctx->cached_sq_head & ctx->sq_mask]);
9835d6fa 4798 if (likely(head < ctx->sq_entries)) {
cf6fd4bd
PB		 4799 /*
4800 * All io need record the previous position, if LINK vs DARIN,
4801 * it can be used to mark the position of the first IO in the
4802 * link list.
4803 */
4804 req->sequence = ctx->cached_sq_head;
3529d8c2
JA		 4805 *sqe_ptr = &ctx->sq_sqes[head];
4806 req->opcode = READ_ONCE((*sqe_ptr)->opcode);
4807 req->user_data = READ_ONCE((*sqe_ptr)->user_data);
2b188cc1
JA		 4808 ctx->cached_sq_head++;
4809 return true;
4810 }
4811
4812 /* drop invalid entries */
4813 ctx->cached_sq_head++;
498ccd9e 4814 ctx->cached_sq_dropped++;
ee7d46d9 4815 WRITE_ONCE(ctx->rings->sq_dropped, ctx->cached_sq_dropped);
2b188cc1
JA		 4816 return false;
4817}
4818
fb5ccc98 4819static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
ae9428ca
PB		 4820 struct file *ring_file, int ring_fd,
4821 struct mm_struct **mm, bool async)
6c271ce2
JA		 4822{
4823 struct io_submit_state state, *statep = NULL;
9e645e11 4824 struct io_kiocb *link = NULL;
9e645e11 4825 int i, submitted = 0;
95a1b3ff 4826 bool mm_fault = false;
6c271ce2 4827
c4a2ed72 4828 /* if we have a backlog and couldn't flush it all, return BUSY */
ad3eb2c8
JA		 4829 if (test_bit(0, &ctx->sq_check_overflow)) {
4830 if (!list_empty(&ctx->cq_overflow_list) &&
4831 !io_cqring_overflow_flush(ctx, false))
4832 return -EBUSY;
4833 }
6c271ce2 4834
ee7d46d9
PB		 4835 /* make sure SQ entry isn't read before tail */
4836 nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));
9ef4f124 4837
2b85edfc
PB		 4838 if (!percpu_ref_tryget_many(&ctx->refs, nr))
4839 return -EAGAIN;
4840
6c271ce2 4841 if (nr > IO_PLUG_THRESHOLD) {
22efde59 4842 io_submit_state_start(&state, nr);
6c271ce2
JA		 4843 statep = &state;
4844 }
4845
b14cca0c
PB		 4846 ctx->ring_fd = ring_fd;
4847 ctx->ring_file = ring_file;
4848
6c271ce2 4849 for (i = 0; i < nr; i++) {
3529d8c2 4850 const struct io_uring_sqe *sqe;
196be95c 4851 struct io_kiocb *req;
fb5ccc98 4852
196be95c
PB		 4853 req = io_get_req(ctx, statep);
4854 if (unlikely(!req)) {
4855 if (!submitted)
4856 submitted = -EAGAIN;
fb5ccc98 4857 break;
196be95c 4858 }
3529d8c2 4859 if (!io_get_sqring(ctx, req, &sqe)) {
2b85edfc 4860 __io_req_do_free(req);
196be95c
PB		 4861 break;
4862 }
fb5ccc98 4863
d3656344
JA		 4864 /* will complete beyond this point, count as submitted */
4865 submitted++;
4866
4867 if (unlikely(req->opcode >= IORING_OP_LAST)) {
4868 io_cqring_add_event(req, -EINVAL);
4869 io_double_put_req(req);
4870 break;
4871 }
4872
4873 if (io_op_defs[req->opcode].needs_mm && !*mm) {
95a1b3ff
PB		 4874 mm_fault = mm_fault || !mmget_not_zero(ctx->sqo_mm);
4875 if (!mm_fault) {
4876 use_mm(ctx->sqo_mm);
4877 *mm = ctx->sqo_mm;
4878 }
9e645e11 4879 }
9e645e11 4880
cf6fd4bd
PB		 4881 req->has_user = *mm != NULL;
4882 req->in_async = async;
4883 req->needs_fixed_file = async;
354420f7
JA		 4884 trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,
4885 true, async);
3529d8c2 4886 if (!io_submit_sqe(req, sqe, statep, &link))
2e6e1fde 4887 break;
6c271ce2
JA		 4888 }
4889
9466f437
PB		 4890 if (unlikely(submitted != nr)) {
4891 int ref_used = (submitted == -EAGAIN) ? 0 : submitted;
4892
4893 percpu_ref_put_many(&ctx->refs, nr - ref_used);
4894 }
9e645e11 4895 if (link)
1b4a51b6 4896 io_queue_link_head(link);
6c271ce2
JA		 4897 if (statep)
4898 io_submit_state_end(&state);
4899
ae9428ca
PB		 4900 /* Commit SQ ring head once we've consumed and submitted all SQEs */
4901 io_commit_sqring(ctx);
4902
6c271ce2
JA		 4903 return submitted;
4904}
4905
4906static int io_sq_thread(void *data)
4907{
6c271ce2
JA		 4908 struct io_ring_ctx *ctx = data;
4909 struct mm_struct *cur_mm = NULL;
181e448d 4910 const struct cred *old_cred;
6c271ce2
JA		 4911 mm_segment_t old_fs;
4912 DEFINE_WAIT(wait);
4913 unsigned inflight;
4914 unsigned long timeout;
c1edbf5f 4915 int ret;
6c271ce2 4916
206aefde 4917 complete(&ctx->completions[1]);
a4c0b3de 4918
6c271ce2
JA		 4919 old_fs = get_fs();
4920 set_fs(USER_DS);
181e448d 4921 old_cred = override_creds(ctx->creds);
6c271ce2 4922
c1edbf5f 4923 ret = timeout = inflight = 0;
2bbcd6d3 4924 while (!kthread_should_park()) {
fb5ccc98 4925 unsigned int to_submit;
6c271ce2
JA		 4926
4927 if (inflight) {
4928 unsigned nr_events = 0;
4929
4930 if (ctx->flags & IORING_SETUP_IOPOLL) {
2b2ed975
JA		 4931 /*
4932 * inflight is the count of the maximum possible
4933 * entries we submitted, but it can be smaller
4934 * if we dropped some of them. If we don't have
4935 * poll entries available, then we know that we
4936 * have nothing left to poll for. Reset the
4937 * inflight count to zero in that case.
4938 */
4939 mutex_lock(&ctx->uring_lock);
4940 if (!list_empty(&ctx->poll_list))
4941 __io_iopoll_check(ctx, &nr_events, 0);
4942 else
4943 inflight = 0;
4944 mutex_unlock(&ctx->uring_lock);
6c271ce2
JA		 4945 } else {
4946 /*
4947 * Normal IO, just pretend everything completed.
4948 * We don't have to poll completions for that.
4949 */
4950 nr_events = inflight;
4951 }
4952
4953 inflight -= nr_events;
4954 if (!inflight)
4955 timeout = jiffies + ctx->sq_thread_idle;
4956 }
4957
fb5ccc98 4958 to_submit = io_sqring_entries(ctx);
c1edbf5f
JA		 4959
4960 /*
4961 * If submit got -EBUSY, flag us as needing the application
4962 * to enter the kernel to reap and flush events.
4963 */
4964 if (!to_submit || ret == -EBUSY) {
6c271ce2
JA		 4965 /*
4966 * We're polling. If we're within the defined idle
4967 * period, then let us spin without work before going
c1edbf5f
JA		 4968 * to sleep. The exception is if we got EBUSY doing
4969 * more IO, we should wait for the application to
4970 * reap events and wake us up.
6c271ce2 4971 */
c1edbf5f
JA		 4972 if (inflight ||
4973 (!time_after(jiffies, timeout) && ret != -EBUSY)) {
9831a90c 4974 cond_resched();
6c271ce2
JA		 4975 continue;
4976 }
4977
4978 /*
4979 * Drop cur_mm before scheduling, we can't hold it for
4980 * long periods (or over schedule()). Do this before
4981 * adding ourselves to the waitqueue, as the unuse/drop
4982 * may sleep.
4983 */
4984 if (cur_mm) {
4985 unuse_mm(cur_mm);
4986 mmput(cur_mm);
4987 cur_mm = NULL;
4988 }
4989
4990 prepare_to_wait(&ctx->sqo_wait, &wait,
4991 TASK_INTERRUPTIBLE);
4992
4993 /* Tell userspace we may need a wakeup call */
75b28aff 4994 ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
0d7bae69
SB		 4995 /* make sure to read SQ tail after writing flags */
4996 smp_mb();
6c271ce2 4997
fb5ccc98 4998 to_submit = io_sqring_entries(ctx);
c1edbf5f 4999 if (!to_submit || ret == -EBUSY) {
2bbcd6d3 5000 if (kthread_should_park()) {
6c271ce2
JA		 5001 finish_wait(&ctx->sqo_wait, &wait);
5002 break;
5003 }
5004 if (signal_pending(current))
5005 flush_signals(current);
5006 schedule();
5007 finish_wait(&ctx->sqo_wait, &wait);
5008
75b28aff 5009 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
6c271ce2
JA		 5010 continue;
5011 }
5012 finish_wait(&ctx->sqo_wait, &wait);
5013
75b28aff 5014 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
6c271ce2
JA		 5015 }
5016
8a4955ff 5017 mutex_lock(&ctx->uring_lock);
1d7bb1d5 5018 ret = io_submit_sqes(ctx, to_submit, NULL, -1, &cur_mm, true);
8a4955ff 5019 mutex_unlock(&ctx->uring_lock);
1d7bb1d5
JA		 5020 if (ret > 0)
5021 inflight += ret;
6c271ce2
JA		 5022 }
5023
5024 set_fs(old_fs);
5025 if (cur_mm) {
5026 unuse_mm(cur_mm);
5027 mmput(cur_mm);
5028 }
181e448d 5029 revert_creds(old_cred);
06058632 5030
2bbcd6d3 5031 kthread_parkme();
06058632 5032
6c271ce2
JA		 5033 return 0;
5034}
5035
bda52162
JA		 5036struct io_wait_queue {
5037 struct wait_queue_entry wq;
5038 struct io_ring_ctx *ctx;
5039 unsigned to_wait;
5040 unsigned nr_timeouts;
5041};
5042
1d7bb1d5 5043static inline bool io_should_wake(struct io_wait_queue *iowq, bool noflush)
bda52162
JA		 5044{
5045 struct io_ring_ctx *ctx = iowq->ctx;
5046
5047 /*
d195a66e 5048 * Wake up if we have enough events, or if a timeout occurred since we
bda52162
JA		 5049 * started waiting. For timeouts, we always want to return to userspace,
5050 * regardless of event count.
5051 */
1d7bb1d5 5052 return io_cqring_events(ctx, noflush) >= iowq->to_wait ||
bda52162
JA		 5053 atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
5054}
5055
5056static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
5057 int wake_flags, void *key)
5058{
5059 struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
5060 wq);
5061
1d7bb1d5
JA		 5062 /* use noflush == true, as we can't safely rely on locking context */
5063 if (!io_should_wake(iowq, true))
bda52162
JA		 5064 return -1;
5065
5066 return autoremove_wake_function(curr, mode, wake_flags, key);
5067}
5068
2b188cc1
JA		 5069/*
5070 * Wait until events become available, if we don't already have some. The
5071 * application must reap them itself, as they reside on the shared cq ring.
5072 */
5073static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
5074 const sigset_t __user *sig, size_t sigsz)
5075{
bda52162
JA		 5076 struct io_wait_queue iowq = {
5077 .wq = {
5078 .private = current,
5079 .func = io_wake_function,
5080 .entry = LIST_HEAD_INIT(iowq.wq.entry),
5081 },
5082 .ctx = ctx,
5083 .to_wait = min_events,
5084 };
75b28aff 5085 struct io_rings *rings = ctx->rings;
e9ffa5c2 5086 int ret = 0;
2b188cc1 5087
1d7bb1d5 5088 if (io_cqring_events(ctx, false) >= min_events)
2b188cc1
JA		 5089 return 0;
5090
5091 if (sig) {
9e75ad5d
AB		 5092#ifdef CONFIG_COMPAT
5093 if (in_compat_syscall())
5094 ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
b772434b 5095 sigsz);
9e75ad5d
AB		 5096 else
5097#endif
b772434b 5098 ret = set_user_sigmask(sig, sigsz);
9e75ad5d 5099
2b188cc1
JA		 5100 if (ret)
5101 return ret;
5102 }
5103
bda52162 5104 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
c826bd7a 5105 trace_io_uring_cqring_wait(ctx, min_events);
bda52162
JA		 5106 do {
5107 prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
5108 TASK_INTERRUPTIBLE);
1d7bb1d5 5109 if (io_should_wake(&iowq, false))
bda52162
JA		 5110 break;
5111 schedule();
5112 if (signal_pending(current)) {
e9ffa5c2 5113 ret = -EINTR;
bda52162
JA		 5114 break;
5115 }
5116 } while (1);
5117 finish_wait(&ctx->wait, &iowq.wq);
5118
e9ffa5c2 5119 restore_saved_sigmask_unless(ret == -EINTR);
2b188cc1 5120
75b28aff 5121 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
2b188cc1
JA		 5122}
5123
6b06314c
JA		 5124static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
5125{
5126#if defined(CONFIG_UNIX)
5127 if (ctx->ring_sock) {
5128 struct sock *sock = ctx->ring_sock->sk;
5129 struct sk_buff *skb;
5130
5131 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
5132 kfree_skb(skb);
5133 }
5134#else
5135 int i;
5136
65e19f54
JA		 5137 for (i = 0; i < ctx->nr_user_files; i++) {
5138 struct file *file;
5139
5140 file = io_file_from_index(ctx, i);
5141 if (file)
5142 fput(file);
5143 }
6b06314c
JA		 5144#endif
5145}
5146
05f3fb3c
JA		 5147static void io_file_ref_kill(struct percpu_ref *ref)
5148{
5149 struct fixed_file_data *data;
5150
5151 data = container_of(ref, struct fixed_file_data, refs);
5152 complete(&data->done);
5153}
5154
6b06314c
JA		 5155static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
5156{
05f3fb3c 5157 struct fixed_file_data *data = ctx->file_data;
65e19f54
JA		 5158 unsigned nr_tables, i;
5159
05f3fb3c 5160 if (!data)
6b06314c
JA		 5161 return -ENXIO;
5162
05f3fb3c 5163 /* protect against inflight atomic switch, which drops the ref */
05f3fb3c 5164 percpu_ref_get(&data->refs);
e46a7950
JA		 5165 /* wait for existing switches */
5166 flush_work(&data->ref_work);
05f3fb3c
JA		 5167 percpu_ref_kill_and_confirm(&data->refs, io_file_ref_kill);
5168 wait_for_completion(&data->done);
5169 percpu_ref_put(&data->refs);
e46a7950
JA		 5170 /* flush potential new switch */
5171 flush_work(&data->ref_work);
05f3fb3c
JA		 5172 percpu_ref_exit(&data->refs);
5173
6b06314c 5174 __io_sqe_files_unregister(ctx);
65e19f54
JA		 5175 nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE);
5176 for (i = 0; i < nr_tables; i++)
05f3fb3c
JA		 5177 kfree(data->table[i].files);
5178 kfree(data->table);
5179 kfree(data);
5180 ctx->file_data = NULL;
6b06314c
JA		 5181 ctx->nr_user_files = 0;
5182 return 0;
5183}
5184
6c271ce2
JA		 5185static void io_sq_thread_stop(struct io_ring_ctx *ctx)
5186{
5187 if (ctx->sqo_thread) {
206aefde 5188 wait_for_completion(&ctx->completions[1]);
2bbcd6d3
RP		 5189 /*
5190 * The park is a bit of a work-around, without it we get
5191 * warning spews on shutdown with SQPOLL set and affinity
5192 * set to a single CPU.
5193 */
06058632 5194 kthread_park(ctx->sqo_thread);
6c271ce2
JA		 5195 kthread_stop(ctx->sqo_thread);
5196 ctx->sqo_thread = NULL;
5197 }
5198}
5199
6b06314c
JA		 5200static void io_finish_async(struct io_ring_ctx *ctx)
5201{
6c271ce2
JA		 5202 io_sq_thread_stop(ctx);
5203
561fb04a
JA		 5204 if (ctx->io_wq) {
5205 io_wq_destroy(ctx->io_wq);
5206 ctx->io_wq = NULL;
6b06314c
JA		 5207 }
5208}
5209
5210#if defined(CONFIG_UNIX)
6b06314c
JA		 5211/*
5212 * Ensure the UNIX gc is aware of our file set, so we are certain that
5213 * the io_uring can be safely unregistered on process exit, even if we have
5214 * loops in the file referencing.
5215 */
5216static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
5217{
5218 struct sock *sk = ctx->ring_sock->sk;
5219 struct scm_fp_list *fpl;
5220 struct sk_buff *skb;
08a45173 5221 int i, nr_files;
6b06314c
JA		 5222
5223 if (!capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) {
5224 unsigned long inflight = ctx->user->unix_inflight + nr;
5225
5226 if (inflight > task_rlimit(current, RLIMIT_NOFILE))
5227 return -EMFILE;
5228 }
5229
5230 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
5231 if (!fpl)
5232 return -ENOMEM;
5233
5234 skb = alloc_skb(0, GFP_KERNEL);
5235 if (!skb) {
5236 kfree(fpl);
5237 return -ENOMEM;
5238 }
5239
5240 skb->sk = sk;
6b06314c 5241
08a45173 5242 nr_files = 0;
6b06314c
JA		 5243 fpl->user = get_uid(ctx->user);
5244 for (i = 0; i < nr; i++) {
65e19f54
JA		 5245 struct file *file = io_file_from_index(ctx, i + offset);
5246
5247 if (!file)
08a45173 5248 continue;
65e19f54 5249 fpl->fp[nr_files] = get_file(file);
08a45173
JA		 5250 unix_inflight(fpl->user, fpl->fp[nr_files]);
5251 nr_files++;
6b06314c
JA		 5252 }
5253
08a45173
JA		 5254 if (nr_files) {
5255 fpl->max = SCM_MAX_FD;
5256 fpl->count = nr_files;
5257 UNIXCB(skb).fp = fpl;
05f3fb3c 5258 skb->destructor = unix_destruct_scm;
08a45173
JA		 5259 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
5260 skb_queue_head(&sk->sk_receive_queue, skb);
6b06314c 5261
08a45173
JA		 5262 for (i = 0; i < nr_files; i++)
5263 fput(fpl->fp[i]);
5264 } else {
5265 kfree_skb(skb);
5266 kfree(fpl);
5267 }
6b06314c
JA		 5268
5269 return 0;
5270}
5271
5272/*
5273 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
5274 * causes regular reference counting to break down. We rely on the UNIX
5275 * garbage collection to take care of this problem for us.
5276 */
5277static int io_sqe_files_scm(struct io_ring_ctx *ctx)
5278{
5279 unsigned left, total;
5280 int ret = 0;
5281
5282 total = 0;
5283 left = ctx->nr_user_files;
5284 while (left) {
5285 unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);
6b06314c
JA		 5286
5287 ret = __io_sqe_files_scm(ctx, this_files, total);
5288 if (ret)
5289 break;
5290 left -= this_files;
5291 total += this_files;
5292 }
5293
5294 if (!ret)
5295 return 0;
5296
5297 while (total < ctx->nr_user_files) {
65e19f54
JA		 5298 struct file *file = io_file_from_index(ctx, total);
5299
5300 if (file)
5301 fput(file);
6b06314c
JA		 5302 total++;
5303 }
5304
5305 return ret;
5306}
5307#else
5308static int io_sqe_files_scm(struct io_ring_ctx *ctx)
5309{
5310 return 0;
5311}
5312#endif
5313
65e19f54
JA		 5314static int io_sqe_alloc_file_tables(struct io_ring_ctx *ctx, unsigned nr_tables,
5315 unsigned nr_files)
5316{
5317 int i;
5318
5319 for (i = 0; i < nr_tables; i++) {
05f3fb3c 5320 struct fixed_file_table *table = &ctx->file_data->table[i];
65e19f54
JA		 5321 unsigned this_files;
5322
5323 this_files = min(nr_files, IORING_MAX_FILES_TABLE);
5324 table->files = kcalloc(this_files, sizeof(struct file *),
5325 GFP_KERNEL);
5326 if (!table->files)
5327 break;
5328 nr_files -= this_files;
5329 }
5330
5331 if (i == nr_tables)
5332 return 0;
5333
5334 for (i = 0; i < nr_tables; i++) {
05f3fb3c 5335 struct fixed_file_table *table = &ctx->file_data->table[i];
65e19f54
JA		 5336 kfree(table->files);
5337 }
5338 return 1;
5339}
5340
05f3fb3c
JA		 5341static void io_ring_file_put(struct io_ring_ctx *ctx, struct file *file)
5342{
5343#if defined(CONFIG_UNIX)
5344 struct sock *sock = ctx->ring_sock->sk;
5345 struct sk_buff_head list, *head = &sock->sk_receive_queue;
5346 struct sk_buff *skb;
5347 int i;
5348
5349 __skb_queue_head_init(&list);
5350
5351 /*
5352 * Find the skb that holds this file in its SCM_RIGHTS. When found,
5353 * remove this entry and rearrange the file array.
5354 */
5355 skb = skb_dequeue(head);
5356 while (skb) {
5357 struct scm_fp_list *fp;
5358
5359 fp = UNIXCB(skb).fp;
5360 for (i = 0; i < fp->count; i++) {
5361 int left;
5362
5363 if (fp->fp[i] != file)
5364 continue;
5365
5366 unix_notinflight(fp->user, fp->fp[i]);
5367 left = fp->count - 1 - i;
5368 if (left) {
5369 memmove(&fp->fp[i], &fp->fp[i + 1],
5370 left * sizeof(struct file *));
5371 }
5372 fp->count--;
5373 if (!fp->count) {
5374 kfree_skb(skb);
5375 skb = NULL;
5376 } else {
5377 __skb_queue_tail(&list, skb);
5378 }
5379 fput(file);
5380 file = NULL;
5381 break;
5382 }
5383
5384 if (!file)
5385 break;
5386
5387 __skb_queue_tail(&list, skb);
5388
5389 skb = skb_dequeue(head);
5390 }
5391
5392 if (skb_peek(&list)) {
5393 spin_lock_irq(&head->lock);
5394 while ((skb = __skb_dequeue(&list)) != NULL)
5395 __skb_queue_tail(head, skb);
5396 spin_unlock_irq(&head->lock);
5397 }
5398#else
5399 fput(file);
5400#endif
5401}
5402
5403struct io_file_put {
5404 struct llist_node llist;
5405 struct file *file;
5406 struct completion *done;
5407};
5408
5409static void io_ring_file_ref_switch(struct work_struct *work)
5410{
5411 struct io_file_put *pfile, *tmp;
5412 struct fixed_file_data *data;
5413 struct llist_node *node;
5414
5415 data = container_of(work, struct fixed_file_data, ref_work);
5416
5417 while ((node = llist_del_all(&data->put_llist)) != NULL) {
5418 llist_for_each_entry_safe(pfile, tmp, node, llist) {
5419 io_ring_file_put(data->ctx, pfile->file);
5420 if (pfile->done)
5421 complete(pfile->done);
5422 else
5423 kfree(pfile);
5424 }
5425 }
5426
5427 percpu_ref_get(&data->refs);
5428 percpu_ref_switch_to_percpu(&data->refs);
5429}
5430
5431static void io_file_data_ref_zero(struct percpu_ref *ref)
5432{
5433 struct fixed_file_data *data;
5434
5435 data = container_of(ref, struct fixed_file_data, refs);
5436
5437 /* we can't safely switch from inside this context, punt to wq */
5438 queue_work(system_wq, &data->ref_work);
5439}
5440
6b06314c
JA		 5441static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
5442 unsigned nr_args)
5443{
5444 __s32 __user *fds = (__s32 __user *) arg;
65e19f54 5445 unsigned nr_tables;
05f3fb3c 5446 struct file *file;
6b06314c
JA		 5447 int fd, ret = 0;
5448 unsigned i;
5449
05f3fb3c 5450 if (ctx->file_data)
6b06314c
JA		 5451 return -EBUSY;
5452 if (!nr_args)
5453 return -EINVAL;
5454 if (nr_args > IORING_MAX_FIXED_FILES)
5455 return -EMFILE;
5456
05f3fb3c
JA		 5457 ctx->file_data = kzalloc(sizeof(*ctx->file_data), GFP_KERNEL);
5458 if (!ctx->file_data)
5459 return -ENOMEM;
5460 ctx->file_data->ctx = ctx;
5461 init_completion(&ctx->file_data->done);
5462
65e19f54 5463 nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE);
05f3fb3c
JA		 5464 ctx->file_data->table = kcalloc(nr_tables,
5465 sizeof(struct fixed_file_table),
65e19f54 5466 GFP_KERNEL);
05f3fb3c
JA		 5467 if (!ctx->file_data->table) {
5468 kfree(ctx->file_data);
5469 ctx->file_data = NULL;
6b06314c 5470 return -ENOMEM;
05f3fb3c
JA		 5471 }
5472
5473 if (percpu_ref_init(&ctx->file_data->refs, io_file_data_ref_zero,
5474 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) {
5475 kfree(ctx->file_data->table);
5476 kfree(ctx->file_data);
5477 ctx->file_data = NULL;
5478 return -ENOMEM;
5479 }
5480 ctx->file_data->put_llist.first = NULL;
5481 INIT_WORK(&ctx->file_data->ref_work, io_ring_file_ref_switch);
6b06314c 5482
65e19f54 5483 if (io_sqe_alloc_file_tables(ctx, nr_tables, nr_args)) {
05f3fb3c
JA		 5484 percpu_ref_exit(&ctx->file_data->refs);
5485 kfree(ctx->file_data->table);
5486 kfree(ctx->file_data);
5487 ctx->file_data = NULL;
65e19f54
JA		 5488 return -ENOMEM;
5489 }
5490
08a45173 5491 for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
65e19f54
JA		 5492 struct fixed_file_table *table;
5493 unsigned index;
5494
6b06314c
JA		 5495 ret = -EFAULT;
5496 if (copy_from_user(&fd, &fds[i], sizeof(fd)))
5497 break;
08a45173
JA		 5498 /* allow sparse sets */
5499 if (fd == -1) {
5500 ret = 0;
5501 continue;
5502 }
6b06314c 5503
05f3fb3c 5504 table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
65e19f54 5505 index = i & IORING_FILE_TABLE_MASK;
05f3fb3c 5506 file = fget(fd);
6b06314c
JA		 5507
5508 ret = -EBADF;
05f3fb3c 5509 if (!file)
6b06314c 5510 break;
05f3fb3c 5511
6b06314c
JA		 5512 /*
5513 * Don't allow io_uring instances to be registered. If UNIX
5514 * isn't enabled, then this causes a reference cycle and this
5515 * instance can never get freed. If UNIX is enabled we'll
5516 * handle it just fine, but there's still no point in allowing
5517 * a ring fd as it doesn't support regular read/write anyway.
5518 */
05f3fb3c
JA		 5519 if (file->f_op == &io_uring_fops) {
5520 fput(file);
6b06314c
JA		 5521 break;
5522 }
6b06314c 5523 ret = 0;
05f3fb3c 5524 table->files[index] = file;
6b06314c
JA		 5525 }
5526
5527 if (ret) {
65e19f54 5528 for (i = 0; i < ctx->nr_user_files; i++) {
65e19f54
JA		 5529 file = io_file_from_index(ctx, i);
5530 if (file)
5531 fput(file);
5532 }
5533 for (i = 0; i < nr_tables; i++)
05f3fb3c 5534 kfree(ctx->file_data->table[i].files);
6b06314c 5535
05f3fb3c
JA		 5536 kfree(ctx->file_data->table);
5537 kfree(ctx->file_data);
5538 ctx->file_data = NULL;
6b06314c
JA		 5539 ctx->nr_user_files = 0;
5540 return ret;
5541 }
5542
5543 ret = io_sqe_files_scm(ctx);
5544 if (ret)
5545 io_sqe_files_unregister(ctx);
5546
5547 return ret;
5548}
5549
c3a31e60
JA		 5550static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
5551 int index)
5552{
5553#if defined(CONFIG_UNIX)
5554 struct sock *sock = ctx->ring_sock->sk;
5555 struct sk_buff_head *head = &sock->sk_receive_queue;
5556 struct sk_buff *skb;
5557
5558 /*
5559 * See if we can merge this file into an existing skb SCM_RIGHTS
5560 * file set. If there's no room, fall back to allocating a new skb
5561 * and filling it in.
5562 */
5563 spin_lock_irq(&head->lock);
5564 skb = skb_peek(head);
5565 if (skb) {
5566 struct scm_fp_list *fpl = UNIXCB(skb).fp;
5567
5568 if (fpl->count < SCM_MAX_FD) {
5569 __skb_unlink(skb, head);
5570 spin_unlock_irq(&head->lock);
5571 fpl->fp[fpl->count] = get_file(file);
5572 unix_inflight(fpl->user, fpl->fp[fpl->count]);
5573 fpl->count++;
5574 spin_lock_irq(&head->lock);
5575 __skb_queue_head(head, skb);
5576 } else {
5577 skb = NULL;
5578 }
5579 }
5580 spin_unlock_irq(&head->lock);
5581
5582 if (skb) {
5583 fput(file);
5584 return 0;
5585 }
5586
5587 return __io_sqe_files_scm(ctx, 1, index);
5588#else
5589 return 0;
5590#endif
5591}
5592
05f3fb3c 5593static void io_atomic_switch(struct percpu_ref *ref)
c3a31e60 5594{
05f3fb3c
JA		 5595 struct fixed_file_data *data;
5596
5597 data = container_of(ref, struct fixed_file_data, refs);
5598 clear_bit(FFD_F_ATOMIC, &data->state);
5599}
5600
5601static bool io_queue_file_removal(struct fixed_file_data *data,
5602 struct file *file)
5603{
5604 struct io_file_put *pfile, pfile_stack;
5605 DECLARE_COMPLETION_ONSTACK(done);
5606
5607 /*
5608 * If we fail allocating the struct we need for doing async reomval
5609 * of this file, just punt to sync and wait for it.
5610 */
5611 pfile = kzalloc(sizeof(*pfile), GFP_KERNEL);
5612 if (!pfile) {
5613 pfile = &pfile_stack;
5614 pfile->done = &done;
5615 }
5616
5617 pfile->file = file;
5618 llist_add(&pfile->llist, &data->put_llist);
5619
5620 if (pfile == &pfile_stack) {
5621 if (!test_and_set_bit(FFD_F_ATOMIC, &data->state)) {
5622 percpu_ref_put(&data->refs);
5623 percpu_ref_switch_to_atomic(&data->refs,
5624 io_atomic_switch);
5625 }
5626 wait_for_completion(&done);
5627 flush_work(&data->ref_work);
5628 return false;
5629 }
5630
5631 return true;
5632}
5633
5634static int __io_sqe_files_update(struct io_ring_ctx *ctx,
5635 struct io_uring_files_update *up,
5636 unsigned nr_args)
5637{
5638 struct fixed_file_data *data = ctx->file_data;
5639 bool ref_switch = false;
5640 struct file *file;
c3a31e60
JA		 5641 __s32 __user *fds;
5642 int fd, i, err;
5643 __u32 done;
5644
05f3fb3c 5645 if (check_add_overflow(up->offset, nr_args, &done))
c3a31e60
JA		 5646 return -EOVERFLOW;
5647 if (done > ctx->nr_user_files)
5648 return -EINVAL;
5649
5650 done = 0;
05f3fb3c 5651 fds = u64_to_user_ptr(up->fds);
c3a31e60 5652 while (nr_args) {
65e19f54
JA		 5653 struct fixed_file_table *table;
5654 unsigned index;
5655
c3a31e60
JA		 5656 err = 0;
5657 if (copy_from_user(&fd, &fds[done], sizeof(fd))) {
5658 err = -EFAULT;
5659 break;
5660 }
05f3fb3c
JA		 5661 i = array_index_nospec(up->offset, ctx->nr_user_files);
5662 table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
65e19f54
JA		 5663 index = i & IORING_FILE_TABLE_MASK;
5664 if (table->files[index]) {
05f3fb3c 5665 file = io_file_from_index(ctx, index);
65e19f54 5666 table->files[index] = NULL;
05f3fb3c
JA		 5667 if (io_queue_file_removal(data, file))
5668 ref_switch = true;
c3a31e60
JA		 5669 }
5670 if (fd != -1) {
c3a31e60
JA		 5671 file = fget(fd);
5672 if (!file) {
5673 err = -EBADF;
5674 break;
5675 }
5676 /*
5677 * Don't allow io_uring instances to be registered. If
5678 * UNIX isn't enabled, then this causes a reference
5679 * cycle and this instance can never get freed. If UNIX
5680 * is enabled we'll handle it just fine, but there's
5681 * still no point in allowing a ring fd as it doesn't
5682 * support regular read/write anyway.
5683 */
5684 if (file->f_op == &io_uring_fops) {
5685 fput(file);
5686 err = -EBADF;
5687 break;
5688 }
65e19f54 5689 table->files[index] = file;
c3a31e60
JA		 5690 err = io_sqe_file_register(ctx, file, i);
5691 if (err)
5692 break;
5693 }
5694 nr_args--;
5695 done++;
05f3fb3c
JA		 5696 up->offset++;
5697 }
5698
5699 if (ref_switch && !test_and_set_bit(FFD_F_ATOMIC, &data->state)) {
5700 percpu_ref_put(&data->refs);
5701 percpu_ref_switch_to_atomic(&data->refs, io_atomic_switch);
c3a31e60
JA		 5702 }
5703
5704 return done ? done : err;
5705}
05f3fb3c
JA		 5706static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg,
5707 unsigned nr_args)
5708{
5709 struct io_uring_files_update up;
5710
5711 if (!ctx->file_data)
5712 return -ENXIO;
5713 if (!nr_args)
5714 return -EINVAL;
5715 if (copy_from_user(&up, arg, sizeof(up)))
5716 return -EFAULT;
5717 if (up.resv)
5718 return -EINVAL;
5719
5720 return __io_sqe_files_update(ctx, &up, nr_args);
5721}
c3a31e60 5722
7d723065
JA		 5723static void io_put_work(struct io_wq_work *work)
5724{
5725 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
5726
5727 io_put_req(req);
5728}
5729
5730static void io_get_work(struct io_wq_work *work)
5731{
5732 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
5733
5734 refcount_inc(&req->refs);
5735}
5736
24369c2e
PB		 5737static int io_init_wq_offload(struct io_ring_ctx *ctx,
5738 struct io_uring_params *p)
5739{
5740 struct io_wq_data data;
5741 struct fd f;
5742 struct io_ring_ctx *ctx_attach;
5743 unsigned int concurrency;
5744 int ret = 0;
5745
5746 data.user = ctx->user;
5747 data.get_work = io_get_work;
5748 data.put_work = io_put_work;
5749
5750 if (!(p->flags & IORING_SETUP_ATTACH_WQ)) {
5751 /* Do QD, or 4 * CPUS, whatever is smallest */
5752 concurrency = min(ctx->sq_entries, 4 * num_online_cpus());
5753
5754 ctx->io_wq = io_wq_create(concurrency, &data);
5755 if (IS_ERR(ctx->io_wq)) {
5756 ret = PTR_ERR(ctx->io_wq);
5757 ctx->io_wq = NULL;
5758 }
5759 return ret;
5760 }
5761
5762 f = fdget(p->wq_fd);
5763 if (!f.file)
5764 return -EBADF;
5765
5766 if (f.file->f_op != &io_uring_fops) {
5767 ret = -EINVAL;
5768 goto out_fput;
5769 }
5770
5771 ctx_attach = f.file->private_data;
5772 /* @io_wq is protected by holding the fd */
5773 if (!io_wq_get(ctx_attach->io_wq, &data)) {
5774 ret = -EINVAL;
5775 goto out_fput;
5776 }
5777
5778 ctx->io_wq = ctx_attach->io_wq;
5779out_fput:
5780 fdput(f);
5781 return ret;
5782}
5783
6c271ce2
JA		 5784static int io_sq_offload_start(struct io_ring_ctx *ctx,
5785 struct io_uring_params *p)
2b188cc1
JA		 5786{
5787 int ret;
5788
6c271ce2 5789 init_waitqueue_head(&ctx->sqo_wait);
2b188cc1
JA		 5790 mmgrab(current->mm);
5791 ctx->sqo_mm = current->mm;
5792
6c271ce2 5793 if (ctx->flags & IORING_SETUP_SQPOLL) {
3ec482d1
JA		 5794 ret = -EPERM;
5795 if (!capable(CAP_SYS_ADMIN))
5796 goto err;
5797
917257da
JA		 5798 ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
5799 if (!ctx->sq_thread_idle)
5800 ctx->sq_thread_idle = HZ;
5801
6c271ce2 5802 if (p->flags & IORING_SETUP_SQ_AFF) {
44a9bd18 5803 int cpu = p->sq_thread_cpu;
6c271ce2 5804
917257da 5805 ret = -EINVAL;
44a9bd18
JA		 5806 if (cpu >= nr_cpu_ids)
5807 goto err;
7889f44d 5808 if (!cpu_online(cpu))
917257da
JA		 5809 goto err;
5810
6c271ce2
JA		 5811 ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
5812 ctx, cpu,
5813 "io_uring-sq");
5814 } else {
5815 ctx->sqo_thread = kthread_create(io_sq_thread, ctx,
5816 "io_uring-sq");
5817 }
5818 if (IS_ERR(ctx->sqo_thread)) {
5819 ret = PTR_ERR(ctx->sqo_thread);
5820 ctx->sqo_thread = NULL;
5821 goto err;
5822 }
5823 wake_up_process(ctx->sqo_thread);
5824 } else if (p->flags & IORING_SETUP_SQ_AFF) {
5825 /* Can't have SQ_AFF without SQPOLL */
5826 ret = -EINVAL;
5827 goto err;
5828 }
5829
24369c2e
PB		 5830 ret = io_init_wq_offload(ctx, p);
5831 if (ret)
2b188cc1 5832 goto err;
2b188cc1
JA		 5833
5834 return 0;
5835err:
54a91f3b 5836 io_finish_async(ctx);
2b188cc1
JA		 5837 mmdrop(ctx->sqo_mm);
5838 ctx->sqo_mm = NULL;
5839 return ret;
5840}
5841
5842static void io_unaccount_mem(struct user_struct *user, unsigned long nr_pages)
5843{
5844 atomic_long_sub(nr_pages, &user->locked_vm);
5845}
5846
5847static int io_account_mem(struct user_struct *user, unsigned long nr_pages)
5848{
5849 unsigned long page_limit, cur_pages, new_pages;
5850
5851 /* Don't allow more pages than we can safely lock */
5852 page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
5853
5854 do {
5855 cur_pages = atomic_long_read(&user->locked_vm);
5856 new_pages = cur_pages + nr_pages;
5857 if (new_pages > page_limit)
5858 return -ENOMEM;
5859 } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
5860 new_pages) != cur_pages);
5861
5862 return 0;
5863}
5864
5865static void io_mem_free(void *ptr)
5866{
52e04ef4
MR		 5867 struct page *page;
5868
5869 if (!ptr)
5870 return;
2b188cc1 5871
52e04ef4 5872 page = virt_to_head_page(ptr);
2b188cc1
JA		 5873 if (put_page_testzero(page))
5874 free_compound_page(page);
5875}
5876
5877static void *io_mem_alloc(size_t size)
5878{
5879 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
5880 __GFP_NORETRY;
5881
5882 return (void *) __get_free_pages(gfp_flags, get_order(size));
5883}
5884
75b28aff
HV		 5885static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
5886 size_t *sq_offset)
5887{
5888 struct io_rings *rings;
5889 size_t off, sq_array_size;
5890
5891 off = struct_size(rings, cqes, cq_entries);
5892 if (off == SIZE_MAX)
5893 return SIZE_MAX;
5894
5895#ifdef CONFIG_SMP
5896 off = ALIGN(off, SMP_CACHE_BYTES);
5897 if (off == 0)
5898 return SIZE_MAX;
5899#endif
5900
5901 sq_array_size = array_size(sizeof(u32), sq_entries);
5902 if (sq_array_size == SIZE_MAX)
5903 return SIZE_MAX;
5904
5905 if (check_add_overflow(off, sq_array_size, &off))
5906 return SIZE_MAX;
5907
5908 if (sq_offset)
5909 *sq_offset = off;
5910
5911 return off;
5912}
5913
2b188cc1
JA		 5914static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
5915{
75b28aff 5916 size_t pages;
2b188cc1 5917
75b28aff
HV		 5918 pages = (size_t)1 << get_order(
5919 rings_size(sq_entries, cq_entries, NULL));
5920 pages += (size_t)1 << get_order(
5921 array_size(sizeof(struct io_uring_sqe), sq_entries));
2b188cc1 5922
75b28aff 5923 return pages;
2b188cc1
JA		 5924}
5925
edafccee
JA		 5926static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
5927{
5928 int i, j;
5929
5930 if (!ctx->user_bufs)
5931 return -ENXIO;
5932
5933 for (i = 0; i < ctx->nr_user_bufs; i++) {
5934 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
5935
5936 for (j = 0; j < imu->nr_bvecs; j++)
27c4d3a3 5937 put_user_page(imu->bvec[j].bv_page);
edafccee
JA		 5938
5939 if (ctx->account_mem)
5940 io_unaccount_mem(ctx->user, imu->nr_bvecs);
d4ef6475 5941 kvfree(imu->bvec);
edafccee
JA		 5942 imu->nr_bvecs = 0;
5943 }
5944
5945 kfree(ctx->user_bufs);
5946 ctx->user_bufs = NULL;
5947 ctx->nr_user_bufs = 0;
5948 return 0;
5949}
5950
5951static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
5952 void __user *arg, unsigned index)
5953{
5954 struct iovec __user *src;
5955
5956#ifdef CONFIG_COMPAT
5957 if (ctx->compat) {
5958 struct compat_iovec __user *ciovs;
5959 struct compat_iovec ciov;
5960
5961 ciovs = (struct compat_iovec __user *) arg;
5962 if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
5963 return -EFAULT;
5964
d55e5f5b 5965 dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
edafccee
JA		 5966 dst->iov_len = ciov.iov_len;
5967 return 0;
5968 }
5969#endif
5970 src = (struct iovec __user *) arg;
5971 if (copy_from_user(dst, &src[index], sizeof(*dst)))
5972 return -EFAULT;
5973 return 0;
5974}
5975
5976static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
5977 unsigned nr_args)
5978{
5979 struct vm_area_struct **vmas = NULL;
5980 struct page **pages = NULL;
5981 int i, j, got_pages = 0;
5982 int ret = -EINVAL;
5983
5984 if (ctx->user_bufs)
5985 return -EBUSY;
5986 if (!nr_args || nr_args > UIO_MAXIOV)
5987 return -EINVAL;
5988
5989 ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf),
5990 GFP_KERNEL);
5991 if (!ctx->user_bufs)
5992 return -ENOMEM;
5993
5994 for (i = 0; i < nr_args; i++) {
5995 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
5996 unsigned long off, start, end, ubuf;
5997 int pret, nr_pages;
5998 struct iovec iov;
5999 size_t size;
6000
6001 ret = io_copy_iov(ctx, &iov, arg, i);
6002 if (ret)
a278682d 6003 goto err;
edafccee
JA		 6004
6005 /*
6006 * Don't impose further limits on the size and buffer
6007 * constraints here, we'll -EINVAL later when IO is
6008 * submitted if they are wrong.
6009 */
6010 ret = -EFAULT;
6011 if (!iov.iov_base || !iov.iov_len)
6012 goto err;
6013
6014 /* arbitrary limit, but we need something */
6015 if (iov.iov_len > SZ_1G)
6016 goto err;
6017
6018 ubuf = (unsigned long) iov.iov_base;
6019 end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
6020 start = ubuf >> PAGE_SHIFT;
6021 nr_pages = end - start;
6022
6023 if (ctx->account_mem) {
6024 ret = io_account_mem(ctx->user, nr_pages);
6025 if (ret)
6026 goto err;
6027 }
6028
6029 ret = 0;
6030 if (!pages || nr_pages > got_pages) {
6031 kfree(vmas);
6032 kfree(pages);
d4ef6475 6033 pages = kvmalloc_array(nr_pages, sizeof(struct page *),
edafccee 6034 GFP_KERNEL);
d4ef6475 6035 vmas = kvmalloc_array(nr_pages,
edafccee
JA		 6036 sizeof(struct vm_area_struct *),
6037 GFP_KERNEL);
6038 if (!pages || !vmas) {
6039 ret = -ENOMEM;
6040 if (ctx->account_mem)
6041 io_unaccount_mem(ctx->user, nr_pages);
6042 goto err;
6043 }
6044 got_pages = nr_pages;
6045 }
6046
d4ef6475 6047 imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec),
edafccee
JA		 6048 GFP_KERNEL);
6049 ret = -ENOMEM;
6050 if (!imu->bvec) {
6051 if (ctx->account_mem)
6052 io_unaccount_mem(ctx->user, nr_pages);
6053 goto err;
6054 }
6055
6056 ret = 0;
6057 down_read(&current->mm->mmap_sem);
932f4a63
IW		 6058 pret = get_user_pages(ubuf, nr_pages,
6059 FOLL_WRITE | FOLL_LONGTERM,
6060 pages, vmas);
edafccee
JA		 6061 if (pret == nr_pages) {
6062 /* don't support file backed memory */
6063 for (j = 0; j < nr_pages; j++) {
6064 struct vm_area_struct *vma = vmas[j];
6065
6066 if (vma->vm_file &&
6067 !is_file_hugepages(vma->vm_file)) {
6068 ret = -EOPNOTSUPP;
6069 break;
6070 }
6071 }
6072 } else {
6073 ret = pret < 0 ? pret : -EFAULT;
6074 }
6075 up_read(&current->mm->mmap_sem);
6076 if (ret) {
6077 /*
6078 * if we did partial map, or found file backed vmas,
6079 * release any pages we did get
6080 */
27c4d3a3
JH		 6081 if (pret > 0)
6082 put_user_pages(pages, pret);
edafccee
JA		 6083 if (ctx->account_mem)
6084 io_unaccount_mem(ctx->user, nr_pages);
d4ef6475 6085 kvfree(imu->bvec);
edafccee
JA		 6086 goto err;
6087 }
6088
6089 off = ubuf & ~PAGE_MASK;
6090 size = iov.iov_len;
6091 for (j = 0; j < nr_pages; j++) {
6092 size_t vec_len;
6093
6094 vec_len = min_t(size_t, size, PAGE_SIZE - off);
6095 imu->bvec[j].bv_page = pages[j];
6096 imu->bvec[j].bv_len = vec_len;
6097 imu->bvec[j].bv_offset = off;
6098 off = 0;
6099 size -= vec_len;
6100 }
6101 /* store original address for later verification */
6102 imu->ubuf = ubuf;
6103 imu->len = iov.iov_len;
6104 imu->nr_bvecs = nr_pages;
6105
6106 ctx->nr_user_bufs++;
6107 }
d4ef6475
MR		 6108 kvfree(pages);
6109 kvfree(vmas);
edafccee
JA		 6110 return 0;
6111err:
d4ef6475
MR		 6112 kvfree(pages);
6113 kvfree(vmas);
edafccee
JA		 6114 io_sqe_buffer_unregister(ctx);
6115 return ret;
6116}
6117
9b402849
JA		 6118static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
6119{
6120 __s32 __user *fds = arg;
6121 int fd;
6122
6123 if (ctx->cq_ev_fd)
6124 return -EBUSY;
6125
6126 if (copy_from_user(&fd, fds, sizeof(*fds)))
6127 return -EFAULT;
6128
6129 ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
6130 if (IS_ERR(ctx->cq_ev_fd)) {
6131 int ret = PTR_ERR(ctx->cq_ev_fd);
6132 ctx->cq_ev_fd = NULL;
6133 return ret;
6134 }
6135
6136 return 0;
6137}
6138
6139static int io_eventfd_unregister(struct io_ring_ctx *ctx)
6140{
6141 if (ctx->cq_ev_fd) {
6142 eventfd_ctx_put(ctx->cq_ev_fd);
6143 ctx->cq_ev_fd = NULL;
6144 return 0;
6145 }
6146
6147 return -ENXIO;
6148}
6149
2b188cc1
JA		 6150static void io_ring_ctx_free(struct io_ring_ctx *ctx)
6151{
6b06314c 6152 io_finish_async(ctx);
2b188cc1
JA		 6153 if (ctx->sqo_mm)
6154 mmdrop(ctx->sqo_mm);
def596e9
JA		 6155
6156 io_iopoll_reap_events(ctx);
edafccee 6157 io_sqe_buffer_unregister(ctx);
6b06314c 6158 io_sqe_files_unregister(ctx);
9b402849 6159 io_eventfd_unregister(ctx);
def596e9 6160
2b188cc1 6161#if defined(CONFIG_UNIX)
355e8d26
EB		 6162 if (ctx->ring_sock) {
6163 ctx->ring_sock->file = NULL; /* so that iput() is called */
2b188cc1 6164 sock_release(ctx->ring_sock);
355e8d26 6165 }
2b188cc1
JA		 6166#endif
6167
75b28aff 6168 io_mem_free(ctx->rings);
2b188cc1 6169 io_mem_free(ctx->sq_sqes);
2b188cc1
JA		 6170
6171 percpu_ref_exit(&ctx->refs);
6172 if (ctx->account_mem)
6173 io_unaccount_mem(ctx->user,
6174 ring_pages(ctx->sq_entries, ctx->cq_entries));
6175 free_uid(ctx->user);
181e448d 6176 put_cred(ctx->creds);
206aefde 6177 kfree(ctx->completions);
78076bb6 6178 kfree(ctx->cancel_hash);
0ddf92e8 6179 kmem_cache_free(req_cachep, ctx->fallback_req);
2b188cc1
JA		 6180 kfree(ctx);
6181}
6182
6183static __poll_t io_uring_poll(struct file *file, poll_table *wait)
6184{
6185 struct io_ring_ctx *ctx = file->private_data;
6186 __poll_t mask = 0;
6187
6188 poll_wait(file, &ctx->cq_wait, wait);
4f7067c3
SB		 6189 /*
6190 * synchronizes with barrier from wq_has_sleeper call in
6191 * io_commit_cqring
6192 */
2b188cc1 6193 smp_rmb();
75b28aff
HV		 6194 if (READ_ONCE(ctx->rings->sq.tail) - ctx->cached_sq_head !=
6195 ctx->rings->sq_ring_entries)
2b188cc1 6196 mask |= EPOLLOUT | EPOLLWRNORM;
daa5de54 6197 if (READ_ONCE(ctx->rings->cq.head) != ctx->cached_cq_tail)
2b188cc1
JA		 6198 mask |= EPOLLIN | EPOLLRDNORM;
6199
6200 return mask;
6201}
6202
6203static int io_uring_fasync(int fd, struct file *file, int on)
6204{
6205 struct io_ring_ctx *ctx = file->private_data;
6206
6207 return fasync_helper(fd, file, on, &ctx->cq_fasync);
6208}
6209
071698e1
JA		 6210static int io_remove_personalities(int id, void *p, void *data)
6211{
6212 struct io_ring_ctx *ctx = data;
6213 const struct cred *cred;
6214
6215 cred = idr_remove(&ctx->personality_idr, id);
6216 if (cred)
6217 put_cred(cred);
6218 return 0;
6219}
6220
2b188cc1
JA		 6221static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
6222{
6223 mutex_lock(&ctx->uring_lock);
6224 percpu_ref_kill(&ctx->refs);
6225 mutex_unlock(&ctx->uring_lock);
6226
5262f567 6227 io_kill_timeouts(ctx);
221c5eb2 6228 io_poll_remove_all(ctx);
561fb04a
JA		 6229
6230 if (ctx->io_wq)
6231 io_wq_cancel_all(ctx->io_wq);
6232
def596e9 6233 io_iopoll_reap_events(ctx);
15dff286
JA		 6234 /* if we failed setting up the ctx, we might not have any rings */
6235 if (ctx->rings)
6236 io_cqring_overflow_flush(ctx, true);
071698e1 6237 idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx);
206aefde 6238 wait_for_completion(&ctx->completions[0]);
2b188cc1
JA		 6239 io_ring_ctx_free(ctx);
6240}
6241
6242static int io_uring_release(struct inode *inode, struct file *file)
6243{
6244 struct io_ring_ctx *ctx = file->private_data;
6245
6246 file->private_data = NULL;
6247 io_ring_ctx_wait_and_kill(ctx);
6248 return 0;
6249}
6250
fcb323cc
JA		 6251static void io_uring_cancel_files(struct io_ring_ctx *ctx,
6252 struct files_struct *files)
6253{
6254 struct io_kiocb *req;
6255 DEFINE_WAIT(wait);
6256
6257 while (!list_empty_careful(&ctx->inflight_list)) {
768134d4 6258 struct io_kiocb *cancel_req = NULL;
fcb323cc
JA		 6259
6260 spin_lock_irq(&ctx->inflight_lock);
6261 list_for_each_entry(req, &ctx->inflight_list, inflight_entry) {
768134d4
JA		 6262 if (req->work.files != files)
6263 continue;
6264 /* req is being completed, ignore */
6265 if (!refcount_inc_not_zero(&req->refs))
6266 continue;
6267 cancel_req = req;
6268 break;
fcb323cc 6269 }
768134d4 6270 if (cancel_req)
fcb323cc 6271 prepare_to_wait(&ctx->inflight_wait, &wait,
768134d4 6272 TASK_UNINTERRUPTIBLE);
fcb323cc
JA		 6273 spin_unlock_irq(&ctx->inflight_lock);
6274
768134d4
JA		 6275 /* We need to keep going until we don't find a matching req */
6276 if (!cancel_req)
fcb323cc 6277 break;
2f6d9b9d
BL		 6278
6279 io_wq_cancel_work(ctx->io_wq, &cancel_req->work);
6280 io_put_req(cancel_req);
fcb323cc
JA		 6281 schedule();
6282 }
768134d4 6283 finish_wait(&ctx->inflight_wait, &wait);
fcb323cc
JA		 6284}
6285
6286static int io_uring_flush(struct file *file, void *data)
6287{
6288 struct io_ring_ctx *ctx = file->private_data;
6289
6290 io_uring_cancel_files(ctx, data);
1d7bb1d5
JA		 6291 if (fatal_signal_pending(current) || (current->flags & PF_EXITING)) {
6292 io_cqring_overflow_flush(ctx, true);
fcb323cc 6293 io_wq_cancel_all(ctx->io_wq);
1d7bb1d5 6294 }
fcb323cc
JA		 6295 return 0;
6296}
6297
6c5c240e
RP		 6298static void *io_uring_validate_mmap_request(struct file *file,
6299 loff_t pgoff, size_t sz)
2b188cc1 6300{
2b188cc1 6301 struct io_ring_ctx *ctx = file->private_data;
6c5c240e 6302 loff_t offset = pgoff << PAGE_SHIFT;
2b188cc1
JA		 6303 struct page *page;
6304 void *ptr;
6305
6306 switch (offset) {
6307 case IORING_OFF_SQ_RING:
75b28aff
HV		 6308 case IORING_OFF_CQ_RING:
6309 ptr = ctx->rings;
2b188cc1
JA		 6310 break;
6311 case IORING_OFF_SQES:
6312 ptr = ctx->sq_sqes;
6313 break;
2b188cc1 6314 default:
6c5c240e 6315 return ERR_PTR(-EINVAL);
2b188cc1
JA		 6316 }
6317
6318 page = virt_to_head_page(ptr);
a50b854e 6319 if (sz > page_size(page))
6c5c240e
RP		 6320 return ERR_PTR(-EINVAL);
6321
6322 return ptr;
6323}
6324
6325#ifdef CONFIG_MMU
6326
6327static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
6328{
6329 size_t sz = vma->vm_end - vma->vm_start;
6330 unsigned long pfn;
6331 void *ptr;
6332
6333 ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
6334 if (IS_ERR(ptr))
6335 return PTR_ERR(ptr);
2b188cc1
JA		 6336
6337 pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
6338 return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
6339}
6340
6c5c240e
RP		 6341#else /* !CONFIG_MMU */
6342
6343static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
6344{
6345 return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL;
6346}
6347
6348static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
6349{
6350 return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
6351}
6352
6353static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
6354 unsigned long addr, unsigned long len,
6355 unsigned long pgoff, unsigned long flags)
6356{
6357 void *ptr;
6358
6359 ptr = io_uring_validate_mmap_request(file, pgoff, len);
6360 if (IS_ERR(ptr))
6361 return PTR_ERR(ptr);
6362
6363 return (unsigned long) ptr;
6364}
6365
6366#endif /* !CONFIG_MMU */
6367
2b188cc1
JA		 6368SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
6369 u32, min_complete, u32, flags, const sigset_t __user *, sig,
6370 size_t, sigsz)
6371{
6372 struct io_ring_ctx *ctx;
6373 long ret = -EBADF;
6374 int submitted = 0;
6375 struct fd f;
6376
6c271ce2 6377 if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
2b188cc1
JA		 6378 return -EINVAL;
6379
6380 f = fdget(fd);
6381 if (!f.file)
6382 return -EBADF;
6383
6384 ret = -EOPNOTSUPP;
6385 if (f.file->f_op != &io_uring_fops)
6386 goto out_fput;
6387
6388 ret = -ENXIO;
6389 ctx = f.file->private_data;
6390 if (!percpu_ref_tryget(&ctx->refs))
6391 goto out_fput;
6392
6c271ce2
JA		 6393 /*
6394 * For SQ polling, the thread will do all submissions and completions.
6395 * Just return the requested submit count, and wake the thread if
6396 * we were asked to.
6397 */
b2a9eada 6398 ret = 0;
6c271ce2 6399 if (ctx->flags & IORING_SETUP_SQPOLL) {
c1edbf5f
JA		 6400 if (!list_empty_careful(&ctx->cq_overflow_list))
6401 io_cqring_overflow_flush(ctx, false);
6c271ce2
JA		 6402 if (flags & IORING_ENTER_SQ_WAKEUP)
6403 wake_up(&ctx->sqo_wait);
6404 submitted = to_submit;
b2a9eada 6405 } else if (to_submit) {
ae9428ca 6406 struct mm_struct *cur_mm;
2b188cc1 6407
44d28279
JA		 6408 if (current->mm != ctx->sqo_mm ||
6409 current_cred() != ctx->creds) {
6410 ret = -EPERM;
6411 goto out;
6412 }
6413
2b188cc1 6414 mutex_lock(&ctx->uring_lock);
ae9428ca
PB		 6415 /* already have mm, so io_submit_sqes() won't try to grab it */
6416 cur_mm = ctx->sqo_mm;
6417 submitted = io_submit_sqes(ctx, to_submit, f.file, fd,
6418 &cur_mm, false);
2b188cc1 6419 mutex_unlock(&ctx->uring_lock);
7c504e65
PB		 6420
6421 if (submitted != to_submit)
6422 goto out;
2b188cc1
JA		 6423 }
6424 if (flags & IORING_ENTER_GETEVENTS) {
def596e9
JA		 6425 unsigned nr_events = 0;
6426
2b188cc1
JA		 6427 min_complete = min(min_complete, ctx->cq_entries);
6428
def596e9 6429 if (ctx->flags & IORING_SETUP_IOPOLL) {
def596e9 6430 ret = io_iopoll_check(ctx, &nr_events, min_complete);
def596e9
JA		 6431 } else {
6432 ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
6433 }
2b188cc1
JA		 6434 }
6435
7c504e65 6436out:
6805b32e 6437 percpu_ref_put(&ctx->refs);
2b188cc1
JA		 6438out_fput:
6439 fdput(f);
6440 return submitted ? submitted : ret;
6441}
6442
6443static const struct file_operations io_uring_fops = {
6444 .release = io_uring_release,
fcb323cc 6445 .flush = io_uring_flush,
2b188cc1 6446 .mmap = io_uring_mmap,
6c5c240e
RP		 6447#ifndef CONFIG_MMU
6448 .get_unmapped_area = io_uring_nommu_get_unmapped_area,
6449 .mmap_capabilities = io_uring_nommu_mmap_capabilities,
6450#endif
2b188cc1
JA		 6451 .poll = io_uring_poll,
6452 .fasync = io_uring_fasync,
6453};
6454
6455static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
6456 struct io_uring_params *p)
6457{
75b28aff
HV		 6458 struct io_rings *rings;
6459 size_t size, sq_array_offset;
2b188cc1 6460
75b28aff
HV		 6461 size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
6462 if (size == SIZE_MAX)
6463 return -EOVERFLOW;
6464
6465 rings = io_mem_alloc(size);
6466 if (!rings)
2b188cc1
JA		 6467 return -ENOMEM;
6468
75b28aff
HV		 6469 ctx->rings = rings;
6470 ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
6471 rings->sq_ring_mask = p->sq_entries - 1;
6472 rings->cq_ring_mask = p->cq_entries - 1;
6473 rings->sq_ring_entries = p->sq_entries;
6474 rings->cq_ring_entries = p->cq_entries;
6475 ctx->sq_mask = rings->sq_ring_mask;
6476 ctx->cq_mask = rings->cq_ring_mask;
6477 ctx->sq_entries = rings->sq_ring_entries;
6478 ctx->cq_entries = rings->cq_ring_entries;
2b188cc1
JA		 6479
6480 size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
eb065d30
JA		 6481 if (size == SIZE_MAX) {
6482 io_mem_free(ctx->rings);
6483 ctx->rings = NULL;
2b188cc1 6484 return -EOVERFLOW;
eb065d30 6485 }
2b188cc1
JA		 6486
6487 ctx->sq_sqes = io_mem_alloc(size);
eb065d30
JA		 6488 if (!ctx->sq_sqes) {
6489 io_mem_free(ctx->rings);
6490 ctx->rings = NULL;
2b188cc1 6491 return -ENOMEM;
eb065d30 6492 }
2b188cc1 6493
2b188cc1
JA		 6494 return 0;
6495}
6496
6497/*
6498 * Allocate an anonymous fd, this is what constitutes the application
6499 * visible backing of an io_uring instance. The application mmaps this
6500 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
6501 * we have to tie this fd to a socket for file garbage collection purposes.
6502 */
6503static int io_uring_get_fd(struct io_ring_ctx *ctx)
6504{
6505 struct file *file;
6506 int ret;
6507
6508#if defined(CONFIG_UNIX)
6509 ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
6510 &ctx->ring_sock);
6511 if (ret)
6512 return ret;
6513#endif
6514
6515 ret = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
6516 if (ret < 0)
6517 goto err;
6518
6519 file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
6520 O_RDWR | O_CLOEXEC);
6521 if (IS_ERR(file)) {
6522 put_unused_fd(ret);
6523 ret = PTR_ERR(file);
6524 goto err;
6525 }
6526
6527#if defined(CONFIG_UNIX)
6528 ctx->ring_sock->file = file;
6529#endif
6530 fd_install(ret, file);
6531 return ret;
6532err:
6533#if defined(CONFIG_UNIX)
6534 sock_release(ctx->ring_sock);
6535 ctx->ring_sock = NULL;
6536#endif
6537 return ret;
6538}
6539
6540static int io_uring_create(unsigned entries, struct io_uring_params *p)
6541{
6542 struct user_struct *user = NULL;
6543 struct io_ring_ctx *ctx;
6544 bool account_mem;
6545 int ret;
6546
8110c1a6 6547 if (!entries)
2b188cc1 6548 return -EINVAL;
8110c1a6
JA		 6549 if (entries > IORING_MAX_ENTRIES) {
6550 if (!(p->flags & IORING_SETUP_CLAMP))
6551 return -EINVAL;
6552 entries = IORING_MAX_ENTRIES;
6553 }
2b188cc1
JA		 6554
6555 /*
6556 * Use twice as many entries for the CQ ring. It's possible for the
6557 * application to drive a higher depth than the size of the SQ ring,
6558 * since the sqes are only used at submission time. This allows for
33a107f0
JA		 6559 * some flexibility in overcommitting a bit. If the application has
6560 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
6561 * of CQ ring entries manually.
2b188cc1
JA		 6562 */
6563 p->sq_entries = roundup_pow_of_two(entries);
33a107f0
JA		 6564 if (p->flags & IORING_SETUP_CQSIZE) {
6565 /*
6566 * If IORING_SETUP_CQSIZE is set, we do the same roundup
6567 * to a power-of-two, if it isn't already. We do NOT impose
6568 * any cq vs sq ring sizing.
6569 */
8110c1a6 6570 if (p->cq_entries < p->sq_entries)
33a107f0 6571 return -EINVAL;
8110c1a6
JA		 6572 if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
6573 if (!(p->flags & IORING_SETUP_CLAMP))
6574 return -EINVAL;
6575 p->cq_entries = IORING_MAX_CQ_ENTRIES;
6576 }
33a107f0
JA		 6577 p->cq_entries = roundup_pow_of_two(p->cq_entries);
6578 } else {
6579 p->cq_entries = 2 * p->sq_entries;
6580 }
2b188cc1
JA		 6581
6582 user = get_uid(current_user());
6583 account_mem = !capable(CAP_IPC_LOCK);
6584
6585 if (account_mem) {
6586 ret = io_account_mem(user,
6587 ring_pages(p->sq_entries, p->cq_entries));
6588 if (ret) {
6589 free_uid(user);
6590 return ret;
6591 }
6592 }
6593
6594 ctx = io_ring_ctx_alloc(p);
6595 if (!ctx) {
6596 if (account_mem)
6597 io_unaccount_mem(user, ring_pages(p->sq_entries,
6598 p->cq_entries));
6599 free_uid(user);
6600 return -ENOMEM;
6601 }
6602 ctx->compat = in_compat_syscall();
6603 ctx->account_mem = account_mem;
6604 ctx->user = user;
0b8c0ec7 6605 ctx->creds = get_current_cred();
2b188cc1
JA		 6606
6607 ret = io_allocate_scq_urings(ctx, p);
6608 if (ret)
6609 goto err;
6610
6c271ce2 6611 ret = io_sq_offload_start(ctx, p);
2b188cc1
JA		 6612 if (ret)
6613 goto err;
6614
2b188cc1 6615 memset(&p->sq_off, 0, sizeof(p->sq_off));
75b28aff
HV		 6616 p->sq_off.head = offsetof(struct io_rings, sq.head);
6617 p->sq_off.tail = offsetof(struct io_rings, sq.tail);
6618 p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
6619 p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
6620 p->sq_off.flags = offsetof(struct io_rings, sq_flags);
6621 p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
6622 p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
2b188cc1
JA		 6623
6624 memset(&p->cq_off, 0, sizeof(p->cq_off));
75b28aff
HV		 6625 p->cq_off.head = offsetof(struct io_rings, cq.head);
6626 p->cq_off.tail = offsetof(struct io_rings, cq.tail);
6627 p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
6628 p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
6629 p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
6630 p->cq_off.cqes = offsetof(struct io_rings, cqes);
ac90f249 6631
044c1ab3
JA		 6632 /*
6633 * Install ring fd as the very last thing, so we don't risk someone
6634 * having closed it before we finish setup
6635 */
6636 ret = io_uring_get_fd(ctx);
6637 if (ret < 0)
6638 goto err;
6639
da8c9690 6640 p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
cccf0ee8
JA		 6641 IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
6642 IORING_FEAT_CUR_PERSONALITY;
c826bd7a 6643 trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
2b188cc1
JA		 6644 return ret;
6645err:
6646 io_ring_ctx_wait_and_kill(ctx);
6647 return ret;
6648}
6649
6650/*
6651 * Sets up an aio uring context, and returns the fd. Applications asks for a
6652 * ring size, we return the actual sq/cq ring sizes (among other things) in the
6653 * params structure passed in.
6654 */
6655static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
6656{
6657 struct io_uring_params p;
6658 long ret;
6659 int i;
6660
6661 if (copy_from_user(&p, params, sizeof(p)))
6662 return -EFAULT;
6663 for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
6664 if (p.resv[i])
6665 return -EINVAL;
6666 }
6667
6c271ce2 6668 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
8110c1a6 6669 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
24369c2e 6670 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ))
2b188cc1
JA		 6671 return -EINVAL;
6672
6673 ret = io_uring_create(entries, &p);
6674 if (ret < 0)
6675 return ret;
6676
6677 if (copy_to_user(params, &p, sizeof(p)))
6678 return -EFAULT;
6679
6680 return ret;
6681}
6682
6683SYSCALL_DEFINE2(io_uring_setup, u32, entries,
6684 struct io_uring_params __user *, params)
6685{
6686 return io_uring_setup(entries, params);
6687}
6688
66f4af93
JA		 6689static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
6690{
6691 struct io_uring_probe *p;
6692 size_t size;
6693 int i, ret;
6694
6695 size = struct_size(p, ops, nr_args);
6696 if (size == SIZE_MAX)
6697 return -EOVERFLOW;
6698 p = kzalloc(size, GFP_KERNEL);
6699 if (!p)
6700 return -ENOMEM;
6701
6702 ret = -EFAULT;
6703 if (copy_from_user(p, arg, size))
6704 goto out;
6705 ret = -EINVAL;
6706 if (memchr_inv(p, 0, size))
6707 goto out;
6708
6709 p->last_op = IORING_OP_LAST - 1;
6710 if (nr_args > IORING_OP_LAST)
6711 nr_args = IORING_OP_LAST;
6712
6713 for (i = 0; i < nr_args; i++) {
6714 p->ops[i].op = i;
6715 if (!io_op_defs[i].not_supported)
6716 p->ops[i].flags = IO_URING_OP_SUPPORTED;
6717 }
6718 p->ops_len = i;
6719
6720 ret = 0;
6721 if (copy_to_user(arg, p, size))
6722 ret = -EFAULT;
6723out:
6724 kfree(p);
6725 return ret;
6726}
6727
071698e1
JA		 6728static int io_register_personality(struct io_ring_ctx *ctx)
6729{
6730 const struct cred *creds = get_current_cred();
6731 int id;
6732
6733 id = idr_alloc_cyclic(&ctx->personality_idr, (void *) creds, 1,
6734 USHRT_MAX, GFP_KERNEL);
6735 if (id < 0)
6736 put_cred(creds);
6737 return id;
6738}
6739
6740static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
6741{
6742 const struct cred *old_creds;
6743
6744 old_creds = idr_remove(&ctx->personality_idr, id);
6745 if (old_creds) {
6746 put_cred(old_creds);
6747 return 0;
6748 }
6749
6750 return -EINVAL;
6751}
6752
6753static bool io_register_op_must_quiesce(int op)
6754{
6755 switch (op) {
6756 case IORING_UNREGISTER_FILES:
6757 case IORING_REGISTER_FILES_UPDATE:
6758 case IORING_REGISTER_PROBE:
6759 case IORING_REGISTER_PERSONALITY:
6760 case IORING_UNREGISTER_PERSONALITY:
6761 return false;
6762 default:
6763 return true;
6764 }
6765}
6766
edafccee
JA		 6767static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
6768 void __user *arg, unsigned nr_args)
b19062a5
JA		 6769 __releases(ctx->uring_lock)
6770 __acquires(ctx->uring_lock)
edafccee
JA		 6771{
6772 int ret;
6773
35fa71a0
JA		 6774 /*
6775 * We're inside the ring mutex, if the ref is already dying, then
6776 * someone else killed the ctx or is already going through
6777 * io_uring_register().
6778 */
6779 if (percpu_ref_is_dying(&ctx->refs))
6780 return -ENXIO;
6781
071698e1 6782 if (io_register_op_must_quiesce(opcode)) {
05f3fb3c 6783 percpu_ref_kill(&ctx->refs);
b19062a5 6784
05f3fb3c
JA		 6785 /*
6786 * Drop uring mutex before waiting for references to exit. If
6787 * another thread is currently inside io_uring_enter() it might
6788 * need to grab the uring_lock to make progress. If we hold it
6789 * here across the drain wait, then we can deadlock. It's safe
6790 * to drop the mutex here, since no new references will come in
6791 * after we've killed the percpu ref.
6792 */
6793 mutex_unlock(&ctx->uring_lock);
c150368b 6794 ret = wait_for_completion_interruptible(&ctx->completions[0]);
05f3fb3c 6795 mutex_lock(&ctx->uring_lock);
c150368b
JA		 6796 if (ret) {
6797 percpu_ref_resurrect(&ctx->refs);
6798 ret = -EINTR;
6799 goto out;
6800 }
05f3fb3c 6801 }
edafccee
JA		 6802
6803 switch (opcode) {
6804 case IORING_REGISTER_BUFFERS:
6805 ret = io_sqe_buffer_register(ctx, arg, nr_args);
6806 break;
6807 case IORING_UNREGISTER_BUFFERS:
6808 ret = -EINVAL;
6809 if (arg || nr_args)
6810 break;
6811 ret = io_sqe_buffer_unregister(ctx);
6812 break;
6b06314c
JA		 6813 case IORING_REGISTER_FILES:
6814 ret = io_sqe_files_register(ctx, arg, nr_args);
6815 break;
6816 case IORING_UNREGISTER_FILES:
6817 ret = -EINVAL;
6818 if (arg || nr_args)
6819 break;
6820 ret = io_sqe_files_unregister(ctx);
6821 break;
c3a31e60
JA		 6822 case IORING_REGISTER_FILES_UPDATE:
6823 ret = io_sqe_files_update(ctx, arg, nr_args);
6824 break;
9b402849 6825 case IORING_REGISTER_EVENTFD:
f2842ab5 6826 case IORING_REGISTER_EVENTFD_ASYNC:
9b402849
JA		 6827 ret = -EINVAL;
6828 if (nr_args != 1)
6829 break;
6830 ret = io_eventfd_register(ctx, arg);
f2842ab5
JA		 6831 if (ret)
6832 break;
6833 if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
6834 ctx->eventfd_async = 1;
6835 else
6836 ctx->eventfd_async = 0;
9b402849
JA		 6837 break;
6838 case IORING_UNREGISTER_EVENTFD:
6839 ret = -EINVAL;
6840 if (arg || nr_args)
6841 break;
6842 ret = io_eventfd_unregister(ctx);
6843 break;
66f4af93
JA		 6844 case IORING_REGISTER_PROBE:
6845 ret = -EINVAL;
6846 if (!arg || nr_args > 256)
6847 break;
6848 ret = io_probe(ctx, arg, nr_args);
6849 break;
071698e1
JA		 6850 case IORING_REGISTER_PERSONALITY:
6851 ret = -EINVAL;
6852 if (arg || nr_args)
6853 break;
6854 ret = io_register_personality(ctx);
6855 break;
6856 case IORING_UNREGISTER_PERSONALITY:
6857 ret = -EINVAL;
6858 if (arg)
6859 break;
6860 ret = io_unregister_personality(ctx, nr_args);
6861 break;
edafccee
JA		 6862 default:
6863 ret = -EINVAL;
6864 break;
6865 }
6866
071698e1 6867 if (io_register_op_must_quiesce(opcode)) {
05f3fb3c 6868 /* bring the ctx back to life */
05f3fb3c 6869 percpu_ref_reinit(&ctx->refs);
c150368b
JA		 6870out:
6871 reinit_completion(&ctx->completions[0]);
05f3fb3c 6872 }
edafccee
JA		 6873 return ret;
6874}
6875
6876SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
6877 void __user *, arg, unsigned int, nr_args)
6878{
6879 struct io_ring_ctx *ctx;
6880 long ret = -EBADF;
6881 struct fd f;
6882
6883 f = fdget(fd);
6884 if (!f.file)
6885 return -EBADF;
6886
6887 ret = -EOPNOTSUPP;
6888 if (f.file->f_op != &io_uring_fops)
6889 goto out_fput;
6890
6891 ctx = f.file->private_data;
6892
6893 mutex_lock(&ctx->uring_lock);
6894 ret = __io_uring_register(ctx, opcode, arg, nr_args);
6895 mutex_unlock(&ctx->uring_lock);
c826bd7a
DD		 6896 trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
6897 ctx->cq_ev_fd != NULL, ret);
edafccee
JA		 6898out_fput:
6899 fdput(f);
6900 return ret;
6901}
6902
2b188cc1
JA		 6903static int __init io_uring_init(void)
6904{
d3656344 6905 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
2b188cc1
JA		 6906 req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
6907 return 0;
6908};
6909__initcall(io_uring_init);
Linux 6.x block layer and io_uring tree(s)