1 // SPDX-License-Identifier: GPL-2.0
3 * io_uring opcode handling table
5 #include <linux/kernel.h>
6 #include <linux/errno.h>
8 #include <linux/file.h>
9 #include <linux/io_uring.h>
26 #include "openclose.h"
27 #include "uring_cmd.h"
40 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
46 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
47 const struct io_uring_sqe *sqe)
52 const struct io_issue_def io_issue_defs[] = {
61 .unbound_nonreg_file = 1,
70 .async_size = sizeof(struct io_async_rw),
71 .prep = io_prep_readv,
74 [IORING_OP_WRITEV] = {
77 .unbound_nonreg_file = 1,
85 .async_size = sizeof(struct io_async_rw),
86 .prep = io_prep_writev,
92 .prep = io_fsync_prep,
95 [IORING_OP_READ_FIXED] = {
97 .unbound_nonreg_file = 1,
104 .async_size = sizeof(struct io_async_rw),
105 .prep = io_prep_read_fixed,
108 [IORING_OP_WRITE_FIXED] = {
111 .unbound_nonreg_file = 1,
118 .async_size = sizeof(struct io_async_rw),
119 .prep = io_prep_write_fixed,
122 [IORING_OP_POLL_ADD] = {
124 .unbound_nonreg_file = 1,
126 .prep = io_poll_add_prep,
127 .issue = io_poll_add,
129 [IORING_OP_POLL_REMOVE] = {
131 .prep = io_poll_remove_prep,
132 .issue = io_poll_remove,
134 [IORING_OP_SYNC_FILE_RANGE] = {
138 .issue = io_sync_file_range,
140 [IORING_OP_SENDMSG] = {
142 .unbound_nonreg_file = 1,
145 #if defined(CONFIG_NET)
146 .async_size = sizeof(struct io_async_msghdr),
147 .prep = io_sendmsg_prep,
150 .prep = io_eopnotsupp_prep,
153 [IORING_OP_RECVMSG] = {
155 .unbound_nonreg_file = 1,
159 #if defined(CONFIG_NET)
160 .async_size = sizeof(struct io_async_msghdr),
161 .prep = io_recvmsg_prep,
164 .prep = io_eopnotsupp_prep,
167 [IORING_OP_TIMEOUT] = {
169 .async_size = sizeof(struct io_timeout_data),
170 .prep = io_timeout_prep,
173 [IORING_OP_TIMEOUT_REMOVE] = {
174 /* used by timeout updates' prep() */
176 .prep = io_timeout_remove_prep,
177 .issue = io_timeout_remove,
179 [IORING_OP_ACCEPT] = {
181 .unbound_nonreg_file = 1,
184 .ioprio = 1, /* used for flags */
185 #if defined(CONFIG_NET)
186 .prep = io_accept_prep,
189 .prep = io_eopnotsupp_prep,
192 [IORING_OP_ASYNC_CANCEL] = {
194 .prep = io_async_cancel_prep,
195 .issue = io_async_cancel,
197 [IORING_OP_LINK_TIMEOUT] = {
199 .async_size = sizeof(struct io_timeout_data),
200 .prep = io_link_timeout_prep,
201 .issue = io_no_issue,
203 [IORING_OP_CONNECT] = {
205 .unbound_nonreg_file = 1,
207 #if defined(CONFIG_NET)
208 .async_size = sizeof(struct io_async_msghdr),
209 .prep = io_connect_prep,
212 .prep = io_eopnotsupp_prep,
215 [IORING_OP_FALLOCATE] = {
217 .prep = io_fallocate_prep,
218 .issue = io_fallocate,
220 [IORING_OP_OPENAT] = {
221 .prep = io_openat_prep,
224 [IORING_OP_CLOSE] = {
225 .prep = io_close_prep,
228 [IORING_OP_FILES_UPDATE] = {
231 .prep = io_files_update_prep,
232 .issue = io_files_update,
234 [IORING_OP_STATX] = {
236 .prep = io_statx_prep,
241 .unbound_nonreg_file = 1,
249 .async_size = sizeof(struct io_async_rw),
250 .prep = io_prep_read,
253 [IORING_OP_WRITE] = {
256 .unbound_nonreg_file = 1,
263 .async_size = sizeof(struct io_async_rw),
264 .prep = io_prep_write,
267 [IORING_OP_FADVISE] = {
270 .prep = io_fadvise_prep,
273 [IORING_OP_MADVISE] = {
275 .prep = io_madvise_prep,
280 .unbound_nonreg_file = 1,
285 #if defined(CONFIG_NET)
286 .async_size = sizeof(struct io_async_msghdr),
287 .prep = io_sendmsg_prep,
290 .prep = io_eopnotsupp_prep,
295 .unbound_nonreg_file = 1,
300 #if defined(CONFIG_NET)
301 .async_size = sizeof(struct io_async_msghdr),
302 .prep = io_recvmsg_prep,
305 .prep = io_eopnotsupp_prep,
308 [IORING_OP_OPENAT2] = {
309 .prep = io_openat2_prep,
312 [IORING_OP_EPOLL_CTL] = {
313 .unbound_nonreg_file = 1,
315 #if defined(CONFIG_EPOLL)
316 .prep = io_epoll_ctl_prep,
317 .issue = io_epoll_ctl,
319 .prep = io_eopnotsupp_prep,
322 [IORING_OP_SPLICE] = {
325 .unbound_nonreg_file = 1,
327 .prep = io_splice_prep,
330 [IORING_OP_PROVIDE_BUFFERS] = {
333 .prep = io_provide_buffers_prep,
334 .issue = io_provide_buffers,
336 [IORING_OP_REMOVE_BUFFERS] = {
339 .prep = io_remove_buffers_prep,
340 .issue = io_remove_buffers,
345 .unbound_nonreg_file = 1,
350 [IORING_OP_SHUTDOWN] = {
352 #if defined(CONFIG_NET)
353 .prep = io_shutdown_prep,
354 .issue = io_shutdown,
356 .prep = io_eopnotsupp_prep,
359 [IORING_OP_RENAMEAT] = {
360 .prep = io_renameat_prep,
361 .issue = io_renameat,
363 [IORING_OP_UNLINKAT] = {
364 .prep = io_unlinkat_prep,
365 .issue = io_unlinkat,
367 [IORING_OP_MKDIRAT] = {
368 .prep = io_mkdirat_prep,
371 [IORING_OP_SYMLINKAT] = {
372 .prep = io_symlinkat_prep,
373 .issue = io_symlinkat,
375 [IORING_OP_LINKAT] = {
376 .prep = io_linkat_prep,
379 [IORING_OP_MSG_RING] = {
382 .prep = io_msg_ring_prep,
383 .issue = io_msg_ring,
385 [IORING_OP_FSETXATTR] = {
387 .prep = io_fsetxattr_prep,
388 .issue = io_fsetxattr,
390 [IORING_OP_SETXATTR] = {
391 .prep = io_setxattr_prep,
392 .issue = io_setxattr,
394 [IORING_OP_FGETXATTR] = {
396 .prep = io_fgetxattr_prep,
397 .issue = io_fgetxattr,
399 [IORING_OP_GETXATTR] = {
400 .prep = io_getxattr_prep,
401 .issue = io_getxattr,
403 [IORING_OP_SOCKET] = {
405 #if defined(CONFIG_NET)
406 .prep = io_socket_prep,
409 .prep = io_eopnotsupp_prep,
412 [IORING_OP_URING_CMD] = {
417 .async_size = 2 * sizeof(struct io_uring_sqe),
418 .prep = io_uring_cmd_prep,
419 .issue = io_uring_cmd,
421 [IORING_OP_SEND_ZC] = {
423 .unbound_nonreg_file = 1,
427 #if defined(CONFIG_NET)
428 .async_size = sizeof(struct io_async_msghdr),
429 .prep = io_send_zc_prep,
432 .prep = io_eopnotsupp_prep,
435 [IORING_OP_SENDMSG_ZC] = {
437 .unbound_nonreg_file = 1,
440 #if defined(CONFIG_NET)
441 .async_size = sizeof(struct io_async_msghdr),
442 .prep = io_send_zc_prep,
443 .issue = io_sendmsg_zc,
445 .prep = io_eopnotsupp_prep,
448 [IORING_OP_READ_MULTISHOT] = {
450 .unbound_nonreg_file = 1,
454 .async_size = sizeof(struct io_async_rw),
455 .prep = io_read_mshot_prep,
456 .issue = io_read_mshot,
458 [IORING_OP_WAITID] = {
459 .async_size = sizeof(struct io_waitid_async),
460 .prep = io_waitid_prep,
463 [IORING_OP_FUTEX_WAIT] = {
464 #if defined(CONFIG_FUTEX)
465 .prep = io_futex_prep,
466 .issue = io_futex_wait,
468 .prep = io_eopnotsupp_prep,
471 [IORING_OP_FUTEX_WAKE] = {
472 #if defined(CONFIG_FUTEX)
473 .prep = io_futex_prep,
474 .issue = io_futex_wake,
476 .prep = io_eopnotsupp_prep,
479 [IORING_OP_FUTEX_WAITV] = {
480 #if defined(CONFIG_FUTEX)
481 .prep = io_futexv_prep,
482 .issue = io_futexv_wait,
484 .prep = io_eopnotsupp_prep,
487 [IORING_OP_FIXED_FD_INSTALL] = {
489 .prep = io_install_fixed_fd_prep,
490 .issue = io_install_fixed_fd,
492 [IORING_OP_FTRUNCATE] = {
495 .prep = io_ftruncate_prep,
496 .issue = io_ftruncate,
500 const struct io_cold_def io_cold_defs[] = {
504 [IORING_OP_READV] = {
506 .cleanup = io_readv_writev_cleanup,
509 [IORING_OP_WRITEV] = {
511 .cleanup = io_readv_writev_cleanup,
514 [IORING_OP_FSYNC] = {
517 [IORING_OP_READ_FIXED] = {
518 .name = "READ_FIXED",
519 .cleanup = io_readv_writev_cleanup,
522 [IORING_OP_WRITE_FIXED] = {
523 .name = "WRITE_FIXED",
524 .cleanup = io_readv_writev_cleanup,
527 [IORING_OP_POLL_ADD] = {
530 [IORING_OP_POLL_REMOVE] = {
531 .name = "POLL_REMOVE",
533 [IORING_OP_SYNC_FILE_RANGE] = {
534 .name = "SYNC_FILE_RANGE",
536 [IORING_OP_SENDMSG] = {
538 #if defined(CONFIG_NET)
539 .cleanup = io_sendmsg_recvmsg_cleanup,
540 .fail = io_sendrecv_fail,
543 [IORING_OP_RECVMSG] = {
545 #if defined(CONFIG_NET)
546 .cleanup = io_sendmsg_recvmsg_cleanup,
547 .fail = io_sendrecv_fail,
550 [IORING_OP_TIMEOUT] = {
553 [IORING_OP_TIMEOUT_REMOVE] = {
554 .name = "TIMEOUT_REMOVE",
556 [IORING_OP_ACCEPT] = {
559 [IORING_OP_ASYNC_CANCEL] = {
560 .name = "ASYNC_CANCEL",
562 [IORING_OP_LINK_TIMEOUT] = {
563 .name = "LINK_TIMEOUT",
565 [IORING_OP_CONNECT] = {
568 [IORING_OP_FALLOCATE] = {
571 [IORING_OP_OPENAT] = {
573 .cleanup = io_open_cleanup,
575 [IORING_OP_CLOSE] = {
578 [IORING_OP_FILES_UPDATE] = {
579 .name = "FILES_UPDATE",
581 [IORING_OP_STATX] = {
583 .cleanup = io_statx_cleanup,
587 .cleanup = io_readv_writev_cleanup,
590 [IORING_OP_WRITE] = {
592 .cleanup = io_readv_writev_cleanup,
595 [IORING_OP_FADVISE] = {
598 [IORING_OP_MADVISE] = {
603 #if defined(CONFIG_NET)
604 .cleanup = io_sendmsg_recvmsg_cleanup,
605 .fail = io_sendrecv_fail,
610 #if defined(CONFIG_NET)
611 .cleanup = io_sendmsg_recvmsg_cleanup,
612 .fail = io_sendrecv_fail,
615 [IORING_OP_OPENAT2] = {
617 .cleanup = io_open_cleanup,
619 [IORING_OP_EPOLL_CTL] = {
622 [IORING_OP_SPLICE] = {
625 [IORING_OP_PROVIDE_BUFFERS] = {
626 .name = "PROVIDE_BUFFERS",
628 [IORING_OP_REMOVE_BUFFERS] = {
629 .name = "REMOVE_BUFFERS",
634 [IORING_OP_SHUTDOWN] = {
637 [IORING_OP_RENAMEAT] = {
639 .cleanup = io_renameat_cleanup,
641 [IORING_OP_UNLINKAT] = {
643 .cleanup = io_unlinkat_cleanup,
645 [IORING_OP_MKDIRAT] = {
647 .cleanup = io_mkdirat_cleanup,
649 [IORING_OP_SYMLINKAT] = {
651 .cleanup = io_link_cleanup,
653 [IORING_OP_LINKAT] = {
655 .cleanup = io_link_cleanup,
657 [IORING_OP_MSG_RING] = {
659 .cleanup = io_msg_ring_cleanup,
661 [IORING_OP_FSETXATTR] = {
663 .cleanup = io_xattr_cleanup,
665 [IORING_OP_SETXATTR] = {
667 .cleanup = io_xattr_cleanup,
669 [IORING_OP_FGETXATTR] = {
671 .cleanup = io_xattr_cleanup,
673 [IORING_OP_GETXATTR] = {
675 .cleanup = io_xattr_cleanup,
677 [IORING_OP_SOCKET] = {
680 [IORING_OP_URING_CMD] = {
683 [IORING_OP_SEND_ZC] = {
685 #if defined(CONFIG_NET)
686 .cleanup = io_send_zc_cleanup,
687 .fail = io_sendrecv_fail,
690 [IORING_OP_SENDMSG_ZC] = {
691 .name = "SENDMSG_ZC",
692 #if defined(CONFIG_NET)
693 .cleanup = io_send_zc_cleanup,
694 .fail = io_sendrecv_fail,
697 [IORING_OP_READ_MULTISHOT] = {
698 .name = "READ_MULTISHOT",
699 .cleanup = io_readv_writev_cleanup,
701 [IORING_OP_WAITID] = {
704 [IORING_OP_FUTEX_WAIT] = {
705 .name = "FUTEX_WAIT",
707 [IORING_OP_FUTEX_WAKE] = {
708 .name = "FUTEX_WAKE",
710 [IORING_OP_FUTEX_WAITV] = {
711 .name = "FUTEX_WAITV",
713 [IORING_OP_FIXED_FD_INSTALL] = {
714 .name = "FIXED_FD_INSTALL",
716 [IORING_OP_FTRUNCATE] = {
721 const char *io_uring_get_opcode(u8 opcode)
723 if (opcode < IORING_OP_LAST)
724 return io_cold_defs[opcode].name;
728 void __init io_uring_optable_init(void)
732 BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
733 BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
735 for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
736 BUG_ON(!io_issue_defs[i].prep);
737 if (io_issue_defs[i].prep != io_eopnotsupp_prep)
738 BUG_ON(!io_issue_defs[i].issue);
739 WARN_ON_ONCE(!io_cold_defs[i].name);