1 // SPDX-License-Identifier: GPL-2.0
3 * io_uring opcode handling table
5 #include <linux/kernel.h>
6 #include <linux/errno.h>
8 #include <linux/file.h>
9 #include <linux/io_uring.h>
26 #include "openclose.h"
27 #include "uring_cmd.h"
40 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
46 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
47 const struct io_uring_sqe *sqe)
52 const struct io_issue_def io_issue_defs[] = {
61 .unbound_nonreg_file = 1,
73 [IORING_OP_WRITEV] = {
76 .unbound_nonreg_file = 1,
90 .prep = io_fsync_prep,
93 [IORING_OP_READ_FIXED] = {
95 .unbound_nonreg_file = 1,
102 .prep = io_prep_rw_fixed,
105 [IORING_OP_WRITE_FIXED] = {
108 .unbound_nonreg_file = 1,
115 .prep = io_prep_rw_fixed,
118 [IORING_OP_POLL_ADD] = {
120 .unbound_nonreg_file = 1,
122 .prep = io_poll_add_prep,
123 .issue = io_poll_add,
125 [IORING_OP_POLL_REMOVE] = {
127 .prep = io_poll_remove_prep,
128 .issue = io_poll_remove,
130 [IORING_OP_SYNC_FILE_RANGE] = {
134 .issue = io_sync_file_range,
136 [IORING_OP_SENDMSG] = {
138 .unbound_nonreg_file = 1,
142 #if defined(CONFIG_NET)
143 .prep = io_sendmsg_prep,
146 .prep = io_eopnotsupp_prep,
149 [IORING_OP_RECVMSG] = {
151 .unbound_nonreg_file = 1,
156 #if defined(CONFIG_NET)
157 .prep = io_recvmsg_prep,
160 .prep = io_eopnotsupp_prep,
163 [IORING_OP_TIMEOUT] = {
165 .prep = io_timeout_prep,
168 [IORING_OP_TIMEOUT_REMOVE] = {
169 /* used by timeout updates' prep() */
171 .prep = io_timeout_remove_prep,
172 .issue = io_timeout_remove,
174 [IORING_OP_ACCEPT] = {
176 .unbound_nonreg_file = 1,
179 .ioprio = 1, /* used for flags */
180 #if defined(CONFIG_NET)
181 .prep = io_accept_prep,
184 .prep = io_eopnotsupp_prep,
187 [IORING_OP_ASYNC_CANCEL] = {
189 .prep = io_async_cancel_prep,
190 .issue = io_async_cancel,
192 [IORING_OP_LINK_TIMEOUT] = {
194 .prep = io_link_timeout_prep,
195 .issue = io_no_issue,
197 [IORING_OP_CONNECT] = {
199 .unbound_nonreg_file = 1,
201 #if defined(CONFIG_NET)
202 .prep = io_connect_prep,
205 .prep = io_eopnotsupp_prep,
208 [IORING_OP_FALLOCATE] = {
210 .prep = io_fallocate_prep,
211 .issue = io_fallocate,
213 [IORING_OP_OPENAT] = {
214 .prep = io_openat_prep,
217 [IORING_OP_CLOSE] = {
218 .prep = io_close_prep,
221 [IORING_OP_FILES_UPDATE] = {
224 .prep = io_files_update_prep,
225 .issue = io_files_update,
227 [IORING_OP_STATX] = {
229 .prep = io_statx_prep,
234 .unbound_nonreg_file = 1,
245 [IORING_OP_WRITE] = {
248 .unbound_nonreg_file = 1,
258 [IORING_OP_FADVISE] = {
261 .prep = io_fadvise_prep,
264 [IORING_OP_MADVISE] = {
266 .prep = io_madvise_prep,
271 .unbound_nonreg_file = 1,
276 #if defined(CONFIG_NET)
277 .prep = io_sendmsg_prep,
280 .prep = io_eopnotsupp_prep,
285 .unbound_nonreg_file = 1,
290 #if defined(CONFIG_NET)
291 .prep = io_recvmsg_prep,
294 .prep = io_eopnotsupp_prep,
297 [IORING_OP_OPENAT2] = {
298 .prep = io_openat2_prep,
301 [IORING_OP_EPOLL_CTL] = {
302 .unbound_nonreg_file = 1,
304 #if defined(CONFIG_EPOLL)
305 .prep = io_epoll_ctl_prep,
306 .issue = io_epoll_ctl,
308 .prep = io_eopnotsupp_prep,
311 [IORING_OP_SPLICE] = {
314 .unbound_nonreg_file = 1,
316 .prep = io_splice_prep,
319 [IORING_OP_PROVIDE_BUFFERS] = {
322 .prep = io_provide_buffers_prep,
323 .issue = io_provide_buffers,
325 [IORING_OP_REMOVE_BUFFERS] = {
328 .prep = io_remove_buffers_prep,
329 .issue = io_remove_buffers,
334 .unbound_nonreg_file = 1,
339 [IORING_OP_SHUTDOWN] = {
341 #if defined(CONFIG_NET)
342 .prep = io_shutdown_prep,
343 .issue = io_shutdown,
345 .prep = io_eopnotsupp_prep,
348 [IORING_OP_RENAMEAT] = {
349 .prep = io_renameat_prep,
350 .issue = io_renameat,
352 [IORING_OP_UNLINKAT] = {
353 .prep = io_unlinkat_prep,
354 .issue = io_unlinkat,
356 [IORING_OP_MKDIRAT] = {
357 .prep = io_mkdirat_prep,
360 [IORING_OP_SYMLINKAT] = {
361 .prep = io_symlinkat_prep,
362 .issue = io_symlinkat,
364 [IORING_OP_LINKAT] = {
365 .prep = io_linkat_prep,
368 [IORING_OP_MSG_RING] = {
371 .prep = io_msg_ring_prep,
372 .issue = io_msg_ring,
374 [IORING_OP_FSETXATTR] = {
376 .prep = io_fsetxattr_prep,
377 .issue = io_fsetxattr,
379 [IORING_OP_SETXATTR] = {
380 .prep = io_setxattr_prep,
381 .issue = io_setxattr,
383 [IORING_OP_FGETXATTR] = {
385 .prep = io_fgetxattr_prep,
386 .issue = io_fgetxattr,
388 [IORING_OP_GETXATTR] = {
389 .prep = io_getxattr_prep,
390 .issue = io_getxattr,
392 [IORING_OP_SOCKET] = {
394 #if defined(CONFIG_NET)
395 .prep = io_socket_prep,
398 .prep = io_eopnotsupp_prep,
401 [IORING_OP_URING_CMD] = {
406 .prep = io_uring_cmd_prep,
407 .issue = io_uring_cmd,
409 [IORING_OP_SEND_ZC] = {
411 .unbound_nonreg_file = 1,
416 #if defined(CONFIG_NET)
417 .prep = io_send_zc_prep,
420 .prep = io_eopnotsupp_prep,
423 [IORING_OP_SENDMSG_ZC] = {
425 .unbound_nonreg_file = 1,
429 #if defined(CONFIG_NET)
430 .prep = io_send_zc_prep,
431 .issue = io_sendmsg_zc,
433 .prep = io_eopnotsupp_prep,
436 [IORING_OP_READ_MULTISHOT] = {
438 .unbound_nonreg_file = 1,
442 .prep = io_read_mshot_prep,
443 .issue = io_read_mshot,
445 [IORING_OP_WAITID] = {
446 .prep = io_waitid_prep,
449 [IORING_OP_FUTEX_WAIT] = {
450 #if defined(CONFIG_FUTEX)
451 .prep = io_futex_prep,
452 .issue = io_futex_wait,
454 .prep = io_eopnotsupp_prep,
457 [IORING_OP_FUTEX_WAKE] = {
458 #if defined(CONFIG_FUTEX)
459 .prep = io_futex_prep,
460 .issue = io_futex_wake,
462 .prep = io_eopnotsupp_prep,
465 [IORING_OP_FUTEX_WAITV] = {
466 #if defined(CONFIG_FUTEX)
467 .prep = io_futexv_prep,
468 .issue = io_futexv_wait,
470 .prep = io_eopnotsupp_prep,
473 [IORING_OP_FIXED_FD_INSTALL] = {
475 .prep = io_install_fixed_fd_prep,
476 .issue = io_install_fixed_fd,
478 [IORING_OP_FTRUNCATE] = {
481 .prep = io_ftruncate_prep,
482 .issue = io_ftruncate,
486 const struct io_cold_def io_cold_defs[] = {
490 [IORING_OP_READV] = {
491 .async_size = sizeof(struct io_async_rw),
493 .prep_async = io_readv_prep_async,
494 .cleanup = io_readv_writev_cleanup,
497 [IORING_OP_WRITEV] = {
498 .async_size = sizeof(struct io_async_rw),
500 .prep_async = io_writev_prep_async,
501 .cleanup = io_readv_writev_cleanup,
504 [IORING_OP_FSYNC] = {
507 [IORING_OP_READ_FIXED] = {
508 .async_size = sizeof(struct io_async_rw),
509 .name = "READ_FIXED",
512 [IORING_OP_WRITE_FIXED] = {
513 .async_size = sizeof(struct io_async_rw),
514 .name = "WRITE_FIXED",
517 [IORING_OP_POLL_ADD] = {
520 [IORING_OP_POLL_REMOVE] = {
521 .name = "POLL_REMOVE",
523 [IORING_OP_SYNC_FILE_RANGE] = {
524 .name = "SYNC_FILE_RANGE",
526 [IORING_OP_SENDMSG] = {
528 #if defined(CONFIG_NET)
529 .async_size = sizeof(struct io_async_msghdr),
530 .prep_async = io_sendmsg_prep_async,
531 .cleanup = io_sendmsg_recvmsg_cleanup,
532 .fail = io_sendrecv_fail,
535 [IORING_OP_RECVMSG] = {
537 #if defined(CONFIG_NET)
538 .async_size = sizeof(struct io_async_msghdr),
539 .prep_async = io_recvmsg_prep_async,
540 .cleanup = io_sendmsg_recvmsg_cleanup,
541 .fail = io_sendrecv_fail,
544 [IORING_OP_TIMEOUT] = {
545 .async_size = sizeof(struct io_timeout_data),
548 [IORING_OP_TIMEOUT_REMOVE] = {
549 .name = "TIMEOUT_REMOVE",
551 [IORING_OP_ACCEPT] = {
554 [IORING_OP_ASYNC_CANCEL] = {
555 .name = "ASYNC_CANCEL",
557 [IORING_OP_LINK_TIMEOUT] = {
558 .async_size = sizeof(struct io_timeout_data),
559 .name = "LINK_TIMEOUT",
561 [IORING_OP_CONNECT] = {
563 #if defined(CONFIG_NET)
564 .async_size = sizeof(struct io_async_connect),
565 .prep_async = io_connect_prep_async,
568 [IORING_OP_FALLOCATE] = {
571 [IORING_OP_OPENAT] = {
573 .cleanup = io_open_cleanup,
575 [IORING_OP_CLOSE] = {
578 [IORING_OP_FILES_UPDATE] = {
579 .name = "FILES_UPDATE",
581 [IORING_OP_STATX] = {
583 .cleanup = io_statx_cleanup,
586 .async_size = sizeof(struct io_async_rw),
590 [IORING_OP_WRITE] = {
591 .async_size = sizeof(struct io_async_rw),
595 [IORING_OP_FADVISE] = {
598 [IORING_OP_MADVISE] = {
603 #if defined(CONFIG_NET)
604 .async_size = sizeof(struct io_async_msghdr),
605 .fail = io_sendrecv_fail,
606 .prep_async = io_send_prep_async,
611 #if defined(CONFIG_NET)
612 .fail = io_sendrecv_fail,
615 [IORING_OP_OPENAT2] = {
617 .cleanup = io_open_cleanup,
619 [IORING_OP_EPOLL_CTL] = {
622 [IORING_OP_SPLICE] = {
625 [IORING_OP_PROVIDE_BUFFERS] = {
626 .name = "PROVIDE_BUFFERS",
628 [IORING_OP_REMOVE_BUFFERS] = {
629 .name = "REMOVE_BUFFERS",
634 [IORING_OP_SHUTDOWN] = {
637 [IORING_OP_RENAMEAT] = {
639 .cleanup = io_renameat_cleanup,
641 [IORING_OP_UNLINKAT] = {
643 .cleanup = io_unlinkat_cleanup,
645 [IORING_OP_MKDIRAT] = {
647 .cleanup = io_mkdirat_cleanup,
649 [IORING_OP_SYMLINKAT] = {
651 .cleanup = io_link_cleanup,
653 [IORING_OP_LINKAT] = {
655 .cleanup = io_link_cleanup,
657 [IORING_OP_MSG_RING] = {
659 .cleanup = io_msg_ring_cleanup,
661 [IORING_OP_FSETXATTR] = {
663 .cleanup = io_xattr_cleanup,
665 [IORING_OP_SETXATTR] = {
667 .cleanup = io_xattr_cleanup,
669 [IORING_OP_FGETXATTR] = {
671 .cleanup = io_xattr_cleanup,
673 [IORING_OP_GETXATTR] = {
675 .cleanup = io_xattr_cleanup,
677 [IORING_OP_SOCKET] = {
680 [IORING_OP_URING_CMD] = {
682 .async_size = 2 * sizeof(struct io_uring_sqe),
683 .prep_async = io_uring_cmd_prep_async,
685 [IORING_OP_SEND_ZC] = {
687 #if defined(CONFIG_NET)
688 .async_size = sizeof(struct io_async_msghdr),
689 .prep_async = io_send_prep_async,
690 .cleanup = io_send_zc_cleanup,
691 .fail = io_sendrecv_fail,
694 [IORING_OP_SENDMSG_ZC] = {
695 .name = "SENDMSG_ZC",
696 #if defined(CONFIG_NET)
697 .async_size = sizeof(struct io_async_msghdr),
698 .prep_async = io_sendmsg_prep_async,
699 .cleanup = io_send_zc_cleanup,
700 .fail = io_sendrecv_fail,
703 [IORING_OP_READ_MULTISHOT] = {
704 .name = "READ_MULTISHOT",
706 [IORING_OP_WAITID] = {
708 .async_size = sizeof(struct io_waitid_async),
710 [IORING_OP_FUTEX_WAIT] = {
711 .name = "FUTEX_WAIT",
713 [IORING_OP_FUTEX_WAKE] = {
714 .name = "FUTEX_WAKE",
716 [IORING_OP_FUTEX_WAITV] = {
717 .name = "FUTEX_WAITV",
719 [IORING_OP_FIXED_FD_INSTALL] = {
720 .name = "FIXED_FD_INSTALL",
722 [IORING_OP_FTRUNCATE] = {
727 const char *io_uring_get_opcode(u8 opcode)
729 if (opcode < IORING_OP_LAST)
730 return io_cold_defs[opcode].name;
734 void __init io_uring_optable_init(void)
738 BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
739 BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
741 for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
742 BUG_ON(!io_issue_defs[i].prep);
743 if (io_issue_defs[i].prep != io_eopnotsupp_prep)
744 BUG_ON(!io_issue_defs[i].issue);
745 WARN_ON_ONCE(!io_cold_defs[i].name);