Commit | Line | Data |
---|---|---|
d9b57aa3 JA |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * io_uring opcode handling table | |
4 | */ | |
5 | #include <linux/kernel.h> | |
6 | #include <linux/errno.h> | |
7 | #include <linux/fs.h> | |
8 | #include <linux/file.h> | |
9 | #include <linux/io_uring.h> | |
10 | ||
d9b57aa3 JA |
11 | #include "io_uring.h" |
12 | #include "opdef.h" | |
13 | #include "refs.h" | |
14 | #include "tctx.h" | |
15 | #include "sqpoll.h" | |
16 | #include "fdinfo.h" | |
17 | #include "kbuf.h" | |
18 | #include "rsrc.h" | |
19 | ||
20 | #include "xattr.h" | |
21 | #include "nop.h" | |
22 | #include "fs.h" | |
23 | #include "splice.h" | |
24 | #include "sync.h" | |
25 | #include "advise.h" | |
26 | #include "openclose.h" | |
27 | #include "uring_cmd.h" | |
28 | #include "epoll.h" | |
29 | #include "statx.h" | |
30 | #include "net.h" | |
31 | #include "msg_ring.h" | |
32 | #include "timeout.h" | |
33 | #include "poll.h" | |
34 | #include "cancel.h" | |
35 | #include "rw.h" | |
36 | ||
37 | static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags) | |
38 | { | |
39 | WARN_ON_ONCE(1); | |
40 | return -ECANCELED; | |
41 | } | |
42 | ||
43 | static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb, | |
44 | const struct io_uring_sqe *sqe) | |
45 | { | |
46 | return -EOPNOTSUPP; | |
47 | } | |
48 | ||
49 | const struct io_op_def io_op_defs[] = { | |
50 | [IORING_OP_NOP] = { | |
51 | .audit_skip = 1, | |
52 | .iopoll = 1, | |
53 | .name = "NOP", | |
54 | .prep = io_nop_prep, | |
55 | .issue = io_nop, | |
56 | }, | |
57 | [IORING_OP_READV] = { | |
58 | .needs_file = 1, | |
59 | .unbound_nonreg_file = 1, | |
60 | .pollin = 1, | |
61 | .buffer_select = 1, | |
62 | .plug = 1, | |
63 | .audit_skip = 1, | |
64 | .ioprio = 1, | |
65 | .iopoll = 1, | |
ef0ec1ad | 66 | .iopoll_queue = 1, |
d9b57aa3 JA |
67 | .async_size = sizeof(struct io_async_rw), |
68 | .name = "READV", | |
69 | .prep = io_prep_rw, | |
70 | .issue = io_read, | |
71 | .prep_async = io_readv_prep_async, | |
72 | .cleanup = io_readv_writev_cleanup, | |
47b4c686 | 73 | .fail = io_rw_fail, |
d9b57aa3 JA |
74 | }, |
75 | [IORING_OP_WRITEV] = { | |
76 | .needs_file = 1, | |
77 | .hash_reg_file = 1, | |
78 | .unbound_nonreg_file = 1, | |
79 | .pollout = 1, | |
80 | .plug = 1, | |
81 | .audit_skip = 1, | |
82 | .ioprio = 1, | |
83 | .iopoll = 1, | |
ef0ec1ad | 84 | .iopoll_queue = 1, |
d9b57aa3 JA |
85 | .async_size = sizeof(struct io_async_rw), |
86 | .name = "WRITEV", | |
87 | .prep = io_prep_rw, | |
88 | .issue = io_write, | |
89 | .prep_async = io_writev_prep_async, | |
90 | .cleanup = io_readv_writev_cleanup, | |
47b4c686 | 91 | .fail = io_rw_fail, |
d9b57aa3 JA |
92 | }, |
93 | [IORING_OP_FSYNC] = { | |
94 | .needs_file = 1, | |
95 | .audit_skip = 1, | |
96 | .name = "FSYNC", | |
97 | .prep = io_fsync_prep, | |
98 | .issue = io_fsync, | |
99 | }, | |
100 | [IORING_OP_READ_FIXED] = { | |
101 | .needs_file = 1, | |
102 | .unbound_nonreg_file = 1, | |
103 | .pollin = 1, | |
104 | .plug = 1, | |
105 | .audit_skip = 1, | |
106 | .ioprio = 1, | |
107 | .iopoll = 1, | |
ef0ec1ad | 108 | .iopoll_queue = 1, |
d9b57aa3 JA |
109 | .async_size = sizeof(struct io_async_rw), |
110 | .name = "READ_FIXED", | |
111 | .prep = io_prep_rw, | |
112 | .issue = io_read, | |
47b4c686 | 113 | .fail = io_rw_fail, |
d9b57aa3 JA |
114 | }, |
115 | [IORING_OP_WRITE_FIXED] = { | |
116 | .needs_file = 1, | |
117 | .hash_reg_file = 1, | |
118 | .unbound_nonreg_file = 1, | |
119 | .pollout = 1, | |
120 | .plug = 1, | |
121 | .audit_skip = 1, | |
122 | .ioprio = 1, | |
123 | .iopoll = 1, | |
ef0ec1ad | 124 | .iopoll_queue = 1, |
d9b57aa3 JA |
125 | .async_size = sizeof(struct io_async_rw), |
126 | .name = "WRITE_FIXED", | |
127 | .prep = io_prep_rw, | |
128 | .issue = io_write, | |
47b4c686 | 129 | .fail = io_rw_fail, |
d9b57aa3 JA |
130 | }, |
131 | [IORING_OP_POLL_ADD] = { | |
132 | .needs_file = 1, | |
133 | .unbound_nonreg_file = 1, | |
134 | .audit_skip = 1, | |
135 | .name = "POLL_ADD", | |
136 | .prep = io_poll_add_prep, | |
137 | .issue = io_poll_add, | |
138 | }, | |
139 | [IORING_OP_POLL_REMOVE] = { | |
140 | .audit_skip = 1, | |
141 | .name = "POLL_REMOVE", | |
142 | .prep = io_poll_remove_prep, | |
143 | .issue = io_poll_remove, | |
144 | }, | |
145 | [IORING_OP_SYNC_FILE_RANGE] = { | |
146 | .needs_file = 1, | |
147 | .audit_skip = 1, | |
148 | .name = "SYNC_FILE_RANGE", | |
149 | .prep = io_sfr_prep, | |
150 | .issue = io_sync_file_range, | |
151 | }, | |
152 | [IORING_OP_SENDMSG] = { | |
153 | .needs_file = 1, | |
154 | .unbound_nonreg_file = 1, | |
155 | .pollout = 1, | |
156 | .ioprio = 1, | |
858c293e | 157 | .manual_alloc = 1, |
d9b57aa3 JA |
158 | .name = "SENDMSG", |
159 | #if defined(CONFIG_NET) | |
160 | .async_size = sizeof(struct io_async_msghdr), | |
161 | .prep = io_sendmsg_prep, | |
162 | .issue = io_sendmsg, | |
163 | .prep_async = io_sendmsg_prep_async, | |
164 | .cleanup = io_sendmsg_recvmsg_cleanup, | |
7e6b638e | 165 | .fail = io_sendrecv_fail, |
d9b57aa3 JA |
166 | #else |
167 | .prep = io_eopnotsupp_prep, | |
168 | #endif | |
169 | }, | |
170 | [IORING_OP_RECVMSG] = { | |
171 | .needs_file = 1, | |
172 | .unbound_nonreg_file = 1, | |
173 | .pollin = 1, | |
174 | .buffer_select = 1, | |
175 | .ioprio = 1, | |
858c293e | 176 | .manual_alloc = 1, |
d9b57aa3 JA |
177 | .name = "RECVMSG", |
178 | #if defined(CONFIG_NET) | |
179 | .async_size = sizeof(struct io_async_msghdr), | |
180 | .prep = io_recvmsg_prep, | |
181 | .issue = io_recvmsg, | |
182 | .prep_async = io_recvmsg_prep_async, | |
183 | .cleanup = io_sendmsg_recvmsg_cleanup, | |
7e6b638e | 184 | .fail = io_sendrecv_fail, |
d9b57aa3 JA |
185 | #else |
186 | .prep = io_eopnotsupp_prep, | |
187 | #endif | |
188 | }, | |
189 | [IORING_OP_TIMEOUT] = { | |
190 | .audit_skip = 1, | |
191 | .async_size = sizeof(struct io_timeout_data), | |
192 | .name = "TIMEOUT", | |
193 | .prep = io_timeout_prep, | |
194 | .issue = io_timeout, | |
195 | }, | |
196 | [IORING_OP_TIMEOUT_REMOVE] = { | |
197 | /* used by timeout updates' prep() */ | |
198 | .audit_skip = 1, | |
199 | .name = "TIMEOUT_REMOVE", | |
200 | .prep = io_timeout_remove_prep, | |
201 | .issue = io_timeout_remove, | |
202 | }, | |
203 | [IORING_OP_ACCEPT] = { | |
204 | .needs_file = 1, | |
205 | .unbound_nonreg_file = 1, | |
206 | .pollin = 1, | |
207 | .poll_exclusive = 1, | |
208 | .ioprio = 1, /* used for flags */ | |
209 | .name = "ACCEPT", | |
210 | #if defined(CONFIG_NET) | |
211 | .prep = io_accept_prep, | |
212 | .issue = io_accept, | |
213 | #else | |
214 | .prep = io_eopnotsupp_prep, | |
215 | #endif | |
216 | }, | |
217 | [IORING_OP_ASYNC_CANCEL] = { | |
218 | .audit_skip = 1, | |
219 | .name = "ASYNC_CANCEL", | |
220 | .prep = io_async_cancel_prep, | |
221 | .issue = io_async_cancel, | |
222 | }, | |
223 | [IORING_OP_LINK_TIMEOUT] = { | |
224 | .audit_skip = 1, | |
225 | .async_size = sizeof(struct io_timeout_data), | |
226 | .name = "LINK_TIMEOUT", | |
227 | .prep = io_link_timeout_prep, | |
228 | .issue = io_no_issue, | |
229 | }, | |
230 | [IORING_OP_CONNECT] = { | |
231 | .needs_file = 1, | |
232 | .unbound_nonreg_file = 1, | |
233 | .pollout = 1, | |
234 | .name = "CONNECT", | |
235 | #if defined(CONFIG_NET) | |
236 | .async_size = sizeof(struct io_async_connect), | |
237 | .prep = io_connect_prep, | |
238 | .issue = io_connect, | |
239 | .prep_async = io_connect_prep_async, | |
240 | #else | |
241 | .prep = io_eopnotsupp_prep, | |
242 | #endif | |
243 | }, | |
244 | [IORING_OP_FALLOCATE] = { | |
245 | .needs_file = 1, | |
246 | .name = "FALLOCATE", | |
247 | .prep = io_fallocate_prep, | |
248 | .issue = io_fallocate, | |
249 | }, | |
250 | [IORING_OP_OPENAT] = { | |
251 | .name = "OPENAT", | |
252 | .prep = io_openat_prep, | |
253 | .issue = io_openat, | |
254 | .cleanup = io_open_cleanup, | |
255 | }, | |
256 | [IORING_OP_CLOSE] = { | |
257 | .name = "CLOSE", | |
258 | .prep = io_close_prep, | |
259 | .issue = io_close, | |
260 | }, | |
d9808ceb | 261 | [IORING_OP_FILES_UPDATE] = { |
d9b57aa3 JA |
262 | .audit_skip = 1, |
263 | .iopoll = 1, | |
d9808ceb PB |
264 | .name = "FILES_UPDATE", |
265 | .prep = io_files_update_prep, | |
266 | .issue = io_files_update, | |
d9b57aa3 JA |
267 | }, |
268 | [IORING_OP_STATX] = { | |
269 | .audit_skip = 1, | |
270 | .name = "STATX", | |
271 | .prep = io_statx_prep, | |
272 | .issue = io_statx, | |
273 | .cleanup = io_statx_cleanup, | |
274 | }, | |
275 | [IORING_OP_READ] = { | |
276 | .needs_file = 1, | |
277 | .unbound_nonreg_file = 1, | |
278 | .pollin = 1, | |
279 | .buffer_select = 1, | |
280 | .plug = 1, | |
281 | .audit_skip = 1, | |
282 | .ioprio = 1, | |
283 | .iopoll = 1, | |
ef0ec1ad | 284 | .iopoll_queue = 1, |
d9b57aa3 JA |
285 | .async_size = sizeof(struct io_async_rw), |
286 | .name = "READ", | |
287 | .prep = io_prep_rw, | |
288 | .issue = io_read, | |
47b4c686 | 289 | .fail = io_rw_fail, |
d9b57aa3 JA |
290 | }, |
291 | [IORING_OP_WRITE] = { | |
292 | .needs_file = 1, | |
293 | .hash_reg_file = 1, | |
294 | .unbound_nonreg_file = 1, | |
295 | .pollout = 1, | |
296 | .plug = 1, | |
297 | .audit_skip = 1, | |
298 | .ioprio = 1, | |
299 | .iopoll = 1, | |
ef0ec1ad | 300 | .iopoll_queue = 1, |
d9b57aa3 JA |
301 | .async_size = sizeof(struct io_async_rw), |
302 | .name = "WRITE", | |
303 | .prep = io_prep_rw, | |
304 | .issue = io_write, | |
47b4c686 | 305 | .fail = io_rw_fail, |
d9b57aa3 JA |
306 | }, |
307 | [IORING_OP_FADVISE] = { | |
308 | .needs_file = 1, | |
309 | .audit_skip = 1, | |
310 | .name = "FADVISE", | |
311 | .prep = io_fadvise_prep, | |
312 | .issue = io_fadvise, | |
313 | }, | |
314 | [IORING_OP_MADVISE] = { | |
315 | .name = "MADVISE", | |
316 | .prep = io_madvise_prep, | |
317 | .issue = io_madvise, | |
318 | }, | |
319 | [IORING_OP_SEND] = { | |
320 | .needs_file = 1, | |
321 | .unbound_nonreg_file = 1, | |
322 | .pollout = 1, | |
323 | .audit_skip = 1, | |
324 | .ioprio = 1, | |
516e82f0 | 325 | .manual_alloc = 1, |
d9b57aa3 JA |
326 | .name = "SEND", |
327 | #if defined(CONFIG_NET) | |
516e82f0 | 328 | .async_size = sizeof(struct io_async_msghdr), |
d9b57aa3 JA |
329 | .prep = io_sendmsg_prep, |
330 | .issue = io_send, | |
7e6b638e | 331 | .fail = io_sendrecv_fail, |
516e82f0 | 332 | .prep_async = io_send_prep_async, |
d9b57aa3 JA |
333 | #else |
334 | .prep = io_eopnotsupp_prep, | |
335 | #endif | |
336 | }, | |
337 | [IORING_OP_RECV] = { | |
338 | .needs_file = 1, | |
339 | .unbound_nonreg_file = 1, | |
340 | .pollin = 1, | |
341 | .buffer_select = 1, | |
342 | .audit_skip = 1, | |
343 | .ioprio = 1, | |
344 | .name = "RECV", | |
345 | #if defined(CONFIG_NET) | |
346 | .prep = io_recvmsg_prep, | |
347 | .issue = io_recv, | |
7e6b638e | 348 | .fail = io_sendrecv_fail, |
d9b57aa3 JA |
349 | #else |
350 | .prep = io_eopnotsupp_prep, | |
351 | #endif | |
352 | }, | |
353 | [IORING_OP_OPENAT2] = { | |
354 | .name = "OPENAT2", | |
355 | .prep = io_openat2_prep, | |
356 | .issue = io_openat2, | |
357 | .cleanup = io_open_cleanup, | |
358 | }, | |
359 | [IORING_OP_EPOLL_CTL] = { | |
360 | .unbound_nonreg_file = 1, | |
361 | .audit_skip = 1, | |
362 | .name = "EPOLL", | |
363 | #if defined(CONFIG_EPOLL) | |
364 | .prep = io_epoll_ctl_prep, | |
365 | .issue = io_epoll_ctl, | |
366 | #else | |
367 | .prep = io_eopnotsupp_prep, | |
368 | #endif | |
369 | }, | |
370 | [IORING_OP_SPLICE] = { | |
371 | .needs_file = 1, | |
372 | .hash_reg_file = 1, | |
373 | .unbound_nonreg_file = 1, | |
374 | .audit_skip = 1, | |
375 | .name = "SPLICE", | |
376 | .prep = io_splice_prep, | |
377 | .issue = io_splice, | |
378 | }, | |
379 | [IORING_OP_PROVIDE_BUFFERS] = { | |
380 | .audit_skip = 1, | |
381 | .iopoll = 1, | |
382 | .name = "PROVIDE_BUFFERS", | |
383 | .prep = io_provide_buffers_prep, | |
384 | .issue = io_provide_buffers, | |
385 | }, | |
386 | [IORING_OP_REMOVE_BUFFERS] = { | |
387 | .audit_skip = 1, | |
388 | .iopoll = 1, | |
389 | .name = "REMOVE_BUFFERS", | |
390 | .prep = io_remove_buffers_prep, | |
391 | .issue = io_remove_buffers, | |
392 | }, | |
393 | [IORING_OP_TEE] = { | |
394 | .needs_file = 1, | |
395 | .hash_reg_file = 1, | |
396 | .unbound_nonreg_file = 1, | |
397 | .audit_skip = 1, | |
398 | .name = "TEE", | |
399 | .prep = io_tee_prep, | |
400 | .issue = io_tee, | |
401 | }, | |
402 | [IORING_OP_SHUTDOWN] = { | |
403 | .needs_file = 1, | |
404 | .name = "SHUTDOWN", | |
405 | #if defined(CONFIG_NET) | |
406 | .prep = io_shutdown_prep, | |
407 | .issue = io_shutdown, | |
408 | #else | |
409 | .prep = io_eopnotsupp_prep, | |
410 | #endif | |
411 | }, | |
412 | [IORING_OP_RENAMEAT] = { | |
413 | .name = "RENAMEAT", | |
414 | .prep = io_renameat_prep, | |
415 | .issue = io_renameat, | |
416 | .cleanup = io_renameat_cleanup, | |
417 | }, | |
418 | [IORING_OP_UNLINKAT] = { | |
419 | .name = "UNLINKAT", | |
420 | .prep = io_unlinkat_prep, | |
421 | .issue = io_unlinkat, | |
422 | .cleanup = io_unlinkat_cleanup, | |
423 | }, | |
424 | [IORING_OP_MKDIRAT] = { | |
425 | .name = "MKDIRAT", | |
426 | .prep = io_mkdirat_prep, | |
427 | .issue = io_mkdirat, | |
428 | .cleanup = io_mkdirat_cleanup, | |
429 | }, | |
430 | [IORING_OP_SYMLINKAT] = { | |
431 | .name = "SYMLINKAT", | |
432 | .prep = io_symlinkat_prep, | |
433 | .issue = io_symlinkat, | |
434 | .cleanup = io_link_cleanup, | |
435 | }, | |
436 | [IORING_OP_LINKAT] = { | |
437 | .name = "LINKAT", | |
438 | .prep = io_linkat_prep, | |
439 | .issue = io_linkat, | |
440 | .cleanup = io_link_cleanup, | |
441 | }, | |
442 | [IORING_OP_MSG_RING] = { | |
443 | .needs_file = 1, | |
444 | .iopoll = 1, | |
445 | .name = "MSG_RING", | |
446 | .prep = io_msg_ring_prep, | |
447 | .issue = io_msg_ring, | |
11373026 | 448 | .cleanup = io_msg_ring_cleanup, |
d9b57aa3 JA |
449 | }, |
450 | [IORING_OP_FSETXATTR] = { | |
451 | .needs_file = 1, | |
452 | .name = "FSETXATTR", | |
453 | .prep = io_fsetxattr_prep, | |
454 | .issue = io_fsetxattr, | |
455 | .cleanup = io_xattr_cleanup, | |
456 | }, | |
457 | [IORING_OP_SETXATTR] = { | |
458 | .name = "SETXATTR", | |
459 | .prep = io_setxattr_prep, | |
460 | .issue = io_setxattr, | |
461 | .cleanup = io_xattr_cleanup, | |
462 | }, | |
463 | [IORING_OP_FGETXATTR] = { | |
464 | .needs_file = 1, | |
465 | .name = "FGETXATTR", | |
466 | .prep = io_fgetxattr_prep, | |
467 | .issue = io_fgetxattr, | |
468 | .cleanup = io_xattr_cleanup, | |
469 | }, | |
470 | [IORING_OP_GETXATTR] = { | |
471 | .name = "GETXATTR", | |
472 | .prep = io_getxattr_prep, | |
473 | .issue = io_getxattr, | |
474 | .cleanup = io_xattr_cleanup, | |
475 | }, | |
476 | [IORING_OP_SOCKET] = { | |
477 | .audit_skip = 1, | |
478 | .name = "SOCKET", | |
479 | #if defined(CONFIG_NET) | |
480 | .prep = io_socket_prep, | |
481 | .issue = io_socket, | |
482 | #else | |
483 | .prep = io_eopnotsupp_prep, | |
484 | #endif | |
485 | }, | |
486 | [IORING_OP_URING_CMD] = { | |
487 | .needs_file = 1, | |
488 | .plug = 1, | |
489 | .name = "URING_CMD", | |
5756a3a7 | 490 | .iopoll = 1, |
ef0ec1ad | 491 | .iopoll_queue = 1, |
d9b57aa3 JA |
492 | .async_size = uring_cmd_pdu_size(1), |
493 | .prep = io_uring_cmd_prep, | |
494 | .issue = io_uring_cmd, | |
495 | .prep_async = io_uring_cmd_prep_async, | |
496 | }, | |
b48c312b | 497 | [IORING_OP_SEND_ZC] = { |
9bd3f728 | 498 | .name = "SEND_ZC", |
06a5464b PB |
499 | .needs_file = 1, |
500 | .unbound_nonreg_file = 1, | |
501 | .pollout = 1, | |
502 | .audit_skip = 1, | |
503 | .ioprio = 1, | |
581711c4 | 504 | .manual_alloc = 1, |
06a5464b | 505 | #if defined(CONFIG_NET) |
581711c4 | 506 | .async_size = sizeof(struct io_async_msghdr), |
b0e9b551 PB |
507 | .prep = io_send_zc_prep, |
508 | .issue = io_send_zc, | |
516e82f0 | 509 | .prep_async = io_send_prep_async, |
b0e9b551 | 510 | .cleanup = io_send_zc_cleanup, |
c4c0009e | 511 | .fail = io_sendrecv_fail, |
06a5464b PB |
512 | #else |
513 | .prep = io_eopnotsupp_prep, | |
493108d9 PB |
514 | #endif |
515 | }, | |
516 | [IORING_OP_SENDMSG_ZC] = { | |
517 | .name = "SENDMSG_ZC", | |
518 | .needs_file = 1, | |
519 | .unbound_nonreg_file = 1, | |
520 | .pollout = 1, | |
493108d9 PB |
521 | .ioprio = 1, |
522 | .manual_alloc = 1, | |
523 | #if defined(CONFIG_NET) | |
524 | .async_size = sizeof(struct io_async_msghdr), | |
525 | .prep = io_send_zc_prep, | |
526 | .issue = io_sendmsg_zc, | |
527 | .prep_async = io_sendmsg_prep_async, | |
528 | .cleanup = io_send_zc_cleanup, | |
529 | .fail = io_sendrecv_fail, | |
530 | #else | |
531 | .prep = io_eopnotsupp_prep, | |
06a5464b | 532 | #endif |
06a5464b | 533 | }, |
d9b57aa3 JA |
534 | }; |
535 | ||
536 | const char *io_uring_get_opcode(u8 opcode) | |
537 | { | |
538 | if (opcode < IORING_OP_LAST) | |
539 | return io_op_defs[opcode].name; | |
540 | return "INVALID"; | |
541 | } | |
542 | ||
543 | void __init io_uring_optable_init(void) | |
544 | { | |
545 | int i; | |
546 | ||
547 | BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST); | |
548 | ||
549 | for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) { | |
550 | BUG_ON(!io_op_defs[i].prep); | |
551 | if (io_op_defs[i].prep != io_eopnotsupp_prep) | |
552 | BUG_ON(!io_op_defs[i].issue); | |
553 | WARN_ON_ONCE(!io_op_defs[i].name); | |
554 | } | |
555 | } |