io_uring/opdef.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * io_uring opcode handling table
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/errno.h>
   7 #include <linux/fs.h>
   8 #include <linux/file.h>
   9 #include <linux/io_uring.h>
  10 #include <linux/io_uring/cmd.h>
  11
  12 #include "io_uring.h"
  13 #include "opdef.h"
  14 #include "refs.h"
  15 #include "tctx.h"
  16 #include "sqpoll.h"
  17 #include "fdinfo.h"
  18 #include "kbuf.h"
  19 #include "rsrc.h"
  20
  21 #include "xattr.h"
  22 #include "nop.h"
  23 #include "fs.h"
  24 #include "splice.h"
  25 #include "sync.h"
  26 #include "advise.h"
  27 #include "openclose.h"
  28 #include "uring_cmd.h"
  29 #include "epoll.h"
  30 #include "statx.h"
  31 #include "net.h"
  32 #include "msg_ring.h"
  33 #include "timeout.h"
  34 #include "poll.h"
  35 #include "cancel.h"
  36 #include "rw.h"
  37 #include "waitid.h"
  38 #include "futex.h"
  39 #include "truncate.h"
  40 #include "zcrx.h"
  41
  42 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
  43 {
  44         WARN_ON_ONCE(1);
  45         return -ECANCELED;
  46 }
  47
  48 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
  49                                              const struct io_uring_sqe *sqe)
  50 {
  51         return -EOPNOTSUPP;
  52 }
  53
  54 const struct io_issue_def io_issue_defs[] = {
  55         [IORING_OP_NOP] = {
  56                 .audit_skip             = 1,
  57                 .iopoll                 = 1,
  58                 .prep                   = io_nop_prep,
  59                 .issue                  = io_nop,
  60         },
  61         [IORING_OP_READV] = {
  62                 .needs_file             = 1,
  63                 .unbound_nonreg_file    = 1,
  64                 .pollin                 = 1,
  65                 .buffer_select          = 1,
  66                 .plug                   = 1,
  67                 .audit_skip             = 1,
  68                 .ioprio                 = 1,
  69                 .iopoll                 = 1,
  70                 .iopoll_queue           = 1,
  71                 .vectored               = 1,
  72                 .async_size             = sizeof(struct io_async_rw),
  73                 .prep                   = io_prep_readv,
  74                 .issue                  = io_read,
  75         },
  76         [IORING_OP_WRITEV] = {
  77                 .needs_file             = 1,
  78                 .hash_reg_file          = 1,
  79                 .unbound_nonreg_file    = 1,
  80                 .pollout                = 1,
  81                 .plug                   = 1,
  82                 .audit_skip             = 1,
  83                 .ioprio                 = 1,
  84                 .iopoll                 = 1,
  85                 .iopoll_queue           = 1,
  86                 .vectored               = 1,
  87                 .async_size             = sizeof(struct io_async_rw),
  88                 .prep                   = io_prep_writev,
  89                 .issue                  = io_write,
  90         },
  91         [IORING_OP_FSYNC] = {
  92                 .needs_file             = 1,
  93                 .audit_skip             = 1,
  94                 .prep                   = io_fsync_prep,
  95                 .issue                  = io_fsync,
  96         },
  97         [IORING_OP_READ_FIXED] = {
  98                 .needs_file             = 1,
  99                 .unbound_nonreg_file    = 1,
 100                 .pollin                 = 1,
 101                 .plug                   = 1,
 102                 .audit_skip             = 1,
 103                 .ioprio                 = 1,
 104                 .iopoll                 = 1,
 105                 .iopoll_queue           = 1,
 106                 .async_size             = sizeof(struct io_async_rw),
 107                 .prep                   = io_prep_read_fixed,
 108                 .issue                  = io_read_fixed,
 109         },
 110         [IORING_OP_WRITE_FIXED] = {
 111                 .needs_file             = 1,
 112                 .hash_reg_file          = 1,
 113                 .unbound_nonreg_file    = 1,
 114                 .pollout                = 1,
 115                 .plug                   = 1,
 116                 .audit_skip             = 1,
 117                 .ioprio                 = 1,
 118                 .iopoll                 = 1,
 119                 .iopoll_queue           = 1,
 120                 .async_size             = sizeof(struct io_async_rw),
 121                 .prep                   = io_prep_write_fixed,
 122                 .issue                  = io_write_fixed,
 123         },
 124         [IORING_OP_POLL_ADD] = {
 125                 .needs_file             = 1,
 126                 .unbound_nonreg_file    = 1,
 127                 .audit_skip             = 1,
 128                 .prep                   = io_poll_add_prep,
 129                 .issue                  = io_poll_add,
 130         },
 131         [IORING_OP_POLL_REMOVE] = {
 132                 .audit_skip             = 1,
 133                 .prep                   = io_poll_remove_prep,
 134                 .issue                  = io_poll_remove,
 135         },
 136         [IORING_OP_SYNC_FILE_RANGE] = {
 137                 .needs_file             = 1,
 138                 .audit_skip             = 1,
 139                 .prep                   = io_sfr_prep,
 140                 .issue                  = io_sync_file_range,
 141         },
 142         [IORING_OP_SENDMSG] = {
 143                 .needs_file             = 1,
 144                 .unbound_nonreg_file    = 1,
 145                 .pollout                = 1,
 146                 .ioprio                 = 1,
 147 #if defined(CONFIG_NET)
 148                 .async_size             = sizeof(struct io_async_msghdr),
 149                 .prep                   = io_sendmsg_prep,
 150                 .issue                  = io_sendmsg,
 151 #else
 152                 .prep                   = io_eopnotsupp_prep,
 153 #endif
 154         },
 155         [IORING_OP_RECVMSG] = {
 156                 .needs_file             = 1,
 157                 .unbound_nonreg_file    = 1,
 158                 .pollin                 = 1,
 159                 .buffer_select          = 1,
 160                 .ioprio                 = 1,
 161 #if defined(CONFIG_NET)
 162                 .async_size             = sizeof(struct io_async_msghdr),
 163                 .prep                   = io_recvmsg_prep,
 164                 .issue                  = io_recvmsg,
 165 #else
 166                 .prep                   = io_eopnotsupp_prep,
 167 #endif
 168         },
 169         [IORING_OP_TIMEOUT] = {
 170                 .audit_skip             = 1,
 171                 .async_size             = sizeof(struct io_timeout_data),
 172                 .prep                   = io_timeout_prep,
 173                 .issue                  = io_timeout,
 174         },
 175         [IORING_OP_TIMEOUT_REMOVE] = {
 176                 /* used by timeout updates' prep() */
 177                 .audit_skip             = 1,
 178                 .prep                   = io_timeout_remove_prep,
 179                 .issue                  = io_timeout_remove,
 180         },
 181         [IORING_OP_ACCEPT] = {
 182                 .needs_file             = 1,
 183                 .unbound_nonreg_file    = 1,
 184                 .pollin                 = 1,
 185                 .poll_exclusive         = 1,
 186                 .ioprio                 = 1,    /* used for flags */
 187 #if defined(CONFIG_NET)
 188                 .prep                   = io_accept_prep,
 189                 .issue                  = io_accept,
 190 #else
 191                 .prep                   = io_eopnotsupp_prep,
 192 #endif
 193         },
 194         [IORING_OP_ASYNC_CANCEL] = {
 195                 .audit_skip             = 1,
 196                 .prep                   = io_async_cancel_prep,
 197                 .issue                  = io_async_cancel,
 198         },
 199         [IORING_OP_LINK_TIMEOUT] = {
 200                 .audit_skip             = 1,
 201                 .async_size             = sizeof(struct io_timeout_data),
 202                 .prep                   = io_link_timeout_prep,
 203                 .issue                  = io_no_issue,
 204         },
 205         [IORING_OP_CONNECT] = {
 206                 .needs_file             = 1,
 207                 .unbound_nonreg_file    = 1,
 208                 .pollout                = 1,
 209 #if defined(CONFIG_NET)
 210                 .async_size             = sizeof(struct io_async_msghdr),
 211                 .prep                   = io_connect_prep,
 212                 .issue                  = io_connect,
 213 #else
 214                 .prep                   = io_eopnotsupp_prep,
 215 #endif
 216         },
 217         [IORING_OP_FALLOCATE] = {
 218                 .needs_file             = 1,
 219                 .hash_reg_file          = 1,
 220                 .prep                   = io_fallocate_prep,
 221                 .issue                  = io_fallocate,
 222         },
 223         [IORING_OP_OPENAT] = {
 224                 .prep                   = io_openat_prep,
 225                 .issue                  = io_openat,
 226         },
 227         [IORING_OP_CLOSE] = {
 228                 .prep                   = io_close_prep,
 229                 .issue                  = io_close,
 230         },
 231         [IORING_OP_FILES_UPDATE] = {
 232                 .audit_skip             = 1,
 233                 .iopoll                 = 1,
 234                 .prep                   = io_files_update_prep,
 235                 .issue                  = io_files_update,
 236         },
 237         [IORING_OP_STATX] = {
 238                 .audit_skip             = 1,
 239                 .prep                   = io_statx_prep,
 240                 .issue                  = io_statx,
 241         },
 242         [IORING_OP_READ] = {
 243                 .needs_file             = 1,
 244                 .unbound_nonreg_file    = 1,
 245                 .pollin                 = 1,
 246                 .buffer_select          = 1,
 247                 .plug                   = 1,
 248                 .audit_skip             = 1,
 249                 .ioprio                 = 1,
 250                 .iopoll                 = 1,
 251                 .iopoll_queue           = 1,
 252                 .async_size             = sizeof(struct io_async_rw),
 253                 .prep                   = io_prep_read,
 254                 .issue                  = io_read,
 255         },
 256         [IORING_OP_WRITE] = {
 257                 .needs_file             = 1,
 258                 .hash_reg_file          = 1,
 259                 .unbound_nonreg_file    = 1,
 260                 .pollout                = 1,
 261                 .plug                   = 1,
 262                 .audit_skip             = 1,
 263                 .ioprio                 = 1,
 264                 .iopoll                 = 1,
 265                 .iopoll_queue           = 1,
 266                 .async_size             = sizeof(struct io_async_rw),
 267                 .prep                   = io_prep_write,
 268                 .issue                  = io_write,
 269         },
 270         [IORING_OP_FADVISE] = {
 271                 .needs_file             = 1,
 272                 .audit_skip             = 1,
 273                 .prep                   = io_fadvise_prep,
 274                 .issue                  = io_fadvise,
 275         },
 276         [IORING_OP_MADVISE] = {
 277                 .audit_skip             = 1,
 278                 .prep                   = io_madvise_prep,
 279                 .issue                  = io_madvise,
 280         },
 281         [IORING_OP_SEND] = {
 282                 .needs_file             = 1,
 283                 .unbound_nonreg_file    = 1,
 284                 .pollout                = 1,
 285                 .audit_skip             = 1,
 286                 .ioprio                 = 1,
 287                 .buffer_select          = 1,
 288 #if defined(CONFIG_NET)
 289                 .async_size             = sizeof(struct io_async_msghdr),
 290                 .prep                   = io_sendmsg_prep,
 291                 .issue                  = io_send,
 292 #else
 293                 .prep                   = io_eopnotsupp_prep,
 294 #endif
 295         },
 296         [IORING_OP_RECV] = {
 297                 .needs_file             = 1,
 298                 .unbound_nonreg_file    = 1,
 299                 .pollin                 = 1,
 300                 .buffer_select          = 1,
 301                 .audit_skip             = 1,
 302                 .ioprio                 = 1,
 303 #if defined(CONFIG_NET)
 304                 .async_size             = sizeof(struct io_async_msghdr),
 305                 .prep                   = io_recvmsg_prep,
 306                 .issue                  = io_recv,
 307 #else
 308                 .prep                   = io_eopnotsupp_prep,
 309 #endif
 310         },
 311         [IORING_OP_OPENAT2] = {
 312                 .prep                   = io_openat2_prep,
 313                 .issue                  = io_openat2,
 314         },
 315         [IORING_OP_EPOLL_CTL] = {
 316                 .unbound_nonreg_file    = 1,
 317                 .audit_skip             = 1,
 318 #if defined(CONFIG_EPOLL)
 319                 .prep                   = io_epoll_ctl_prep,
 320                 .issue                  = io_epoll_ctl,
 321 #else
 322                 .prep                   = io_eopnotsupp_prep,
 323 #endif
 324         },
 325         [IORING_OP_SPLICE] = {
 326                 .needs_file             = 1,
 327                 .hash_reg_file          = 1,
 328                 .unbound_nonreg_file    = 1,
 329                 .audit_skip             = 1,
 330                 .prep                   = io_splice_prep,
 331                 .issue                  = io_splice,
 332         },
 333         [IORING_OP_PROVIDE_BUFFERS] = {
 334                 .audit_skip             = 1,
 335                 .iopoll                 = 1,
 336                 .prep                   = io_provide_buffers_prep,
 337                 .issue                  = io_manage_buffers_legacy,
 338         },
 339         [IORING_OP_REMOVE_BUFFERS] = {
 340                 .audit_skip             = 1,
 341                 .iopoll                 = 1,
 342                 .prep                   = io_remove_buffers_prep,
 343                 .issue                  = io_manage_buffers_legacy,
 344         },
 345         [IORING_OP_TEE] = {
 346                 .needs_file             = 1,
 347                 .hash_reg_file          = 1,
 348                 .unbound_nonreg_file    = 1,
 349                 .audit_skip             = 1,
 350                 .prep                   = io_tee_prep,
 351                 .issue                  = io_tee,
 352         },
 353         [IORING_OP_SHUTDOWN] = {
 354                 .needs_file             = 1,
 355 #if defined(CONFIG_NET)
 356                 .prep                   = io_shutdown_prep,
 357                 .issue                  = io_shutdown,
 358 #else
 359                 .prep                   = io_eopnotsupp_prep,
 360 #endif
 361         },
 362         [IORING_OP_RENAMEAT] = {
 363                 .prep                   = io_renameat_prep,
 364                 .issue                  = io_renameat,
 365         },
 366         [IORING_OP_UNLINKAT] = {
 367                 .prep                   = io_unlinkat_prep,
 368                 .issue                  = io_unlinkat,
 369         },
 370         [IORING_OP_MKDIRAT] = {
 371                 .prep                   = io_mkdirat_prep,
 372                 .issue                  = io_mkdirat,
 373         },
 374         [IORING_OP_SYMLINKAT] = {
 375                 .prep                   = io_symlinkat_prep,
 376                 .issue                  = io_symlinkat,
 377         },
 378         [IORING_OP_LINKAT] = {
 379                 .prep                   = io_linkat_prep,
 380                 .issue                  = io_linkat,
 381         },
 382         [IORING_OP_MSG_RING] = {
 383                 .needs_file             = 1,
 384                 .iopoll                 = 1,
 385                 .prep                   = io_msg_ring_prep,
 386                 .issue                  = io_msg_ring,
 387         },
 388         [IORING_OP_FSETXATTR] = {
 389                 .needs_file = 1,
 390                 .prep                   = io_fsetxattr_prep,
 391                 .issue                  = io_fsetxattr,
 392         },
 393         [IORING_OP_SETXATTR] = {
 394                 .prep                   = io_setxattr_prep,
 395                 .issue                  = io_setxattr,
 396         },
 397         [IORING_OP_FGETXATTR] = {
 398                 .needs_file = 1,
 399                 .prep                   = io_fgetxattr_prep,
 400                 .issue                  = io_fgetxattr,
 401         },
 402         [IORING_OP_GETXATTR] = {
 403                 .prep                   = io_getxattr_prep,
 404                 .issue                  = io_getxattr,
 405         },
 406         [IORING_OP_SOCKET] = {
 407                 .audit_skip             = 1,
 408 #if defined(CONFIG_NET)
 409                 .prep                   = io_socket_prep,
 410                 .issue                  = io_socket,
 411 #else
 412                 .prep                   = io_eopnotsupp_prep,
 413 #endif
 414         },
 415         [IORING_OP_URING_CMD] = {
 416                 .needs_file             = 1,
 417                 .plug                   = 1,
 418                 .iopoll                 = 1,
 419                 .iopoll_queue           = 1,
 420                 .async_size             = sizeof(struct io_async_cmd),
 421                 .prep                   = io_uring_cmd_prep,
 422                 .issue                  = io_uring_cmd,
 423         },
 424         [IORING_OP_SEND_ZC] = {
 425                 .needs_file             = 1,
 426                 .unbound_nonreg_file    = 1,
 427                 .pollout                = 1,
 428                 .audit_skip             = 1,
 429                 .ioprio                 = 1,
 430 #if defined(CONFIG_NET)
 431                 .async_size             = sizeof(struct io_async_msghdr),
 432                 .prep                   = io_send_zc_prep,
 433                 .issue                  = io_send_zc,
 434 #else
 435                 .prep                   = io_eopnotsupp_prep,
 436 #endif
 437         },
 438         [IORING_OP_SENDMSG_ZC] = {
 439                 .needs_file             = 1,
 440                 .unbound_nonreg_file    = 1,
 441                 .pollout                = 1,
 442                 .ioprio                 = 1,
 443 #if defined(CONFIG_NET)
 444                 .async_size             = sizeof(struct io_async_msghdr),
 445                 .prep                   = io_send_zc_prep,
 446                 .issue                  = io_sendmsg_zc,
 447 #else
 448                 .prep                   = io_eopnotsupp_prep,
 449 #endif
 450         },
 451         [IORING_OP_READ_MULTISHOT] = {
 452                 .needs_file             = 1,
 453                 .unbound_nonreg_file    = 1,
 454                 .pollin                 = 1,
 455                 .buffer_select          = 1,
 456                 .audit_skip             = 1,
 457                 .async_size             = sizeof(struct io_async_rw),
 458                 .prep                   = io_read_mshot_prep,
 459                 .issue                  = io_read_mshot,
 460         },
 461         [IORING_OP_WAITID] = {
 462                 .async_size             = sizeof(struct io_waitid_async),
 463                 .prep                   = io_waitid_prep,
 464                 .issue                  = io_waitid,
 465         },
 466         [IORING_OP_FUTEX_WAIT] = {
 467 #if defined(CONFIG_FUTEX)
 468                 .prep                   = io_futex_prep,
 469                 .issue                  = io_futex_wait,
 470 #else
 471                 .prep                   = io_eopnotsupp_prep,
 472 #endif
 473         },
 474         [IORING_OP_FUTEX_WAKE] = {
 475 #if defined(CONFIG_FUTEX)
 476                 .prep                   = io_futex_prep,
 477                 .issue                  = io_futex_wake,
 478 #else
 479                 .prep                   = io_eopnotsupp_prep,
 480 #endif
 481         },
 482         [IORING_OP_FUTEX_WAITV] = {
 483 #if defined(CONFIG_FUTEX)
 484                 .prep                   = io_futexv_prep,
 485                 .issue                  = io_futexv_wait,
 486 #else
 487                 .prep                   = io_eopnotsupp_prep,
 488 #endif
 489         },
 490         [IORING_OP_FIXED_FD_INSTALL] = {
 491                 .needs_file             = 1,
 492                 .prep                   = io_install_fixed_fd_prep,
 493                 .issue                  = io_install_fixed_fd,
 494         },
 495         [IORING_OP_FTRUNCATE] = {
 496                 .needs_file             = 1,
 497                 .hash_reg_file          = 1,
 498                 .prep                   = io_ftruncate_prep,
 499                 .issue                  = io_ftruncate,
 500         },
 501         [IORING_OP_BIND] = {
 502 #if defined(CONFIG_NET)
 503                 .needs_file             = 1,
 504                 .prep                   = io_bind_prep,
 505                 .issue                  = io_bind,
 506                 .async_size             = sizeof(struct io_async_msghdr),
 507 #else
 508                 .prep                   = io_eopnotsupp_prep,
 509 #endif
 510         },
 511         [IORING_OP_LISTEN] = {
 512 #if defined(CONFIG_NET)
 513                 .needs_file             = 1,
 514                 .prep                   = io_listen_prep,
 515                 .issue                  = io_listen,
 516                 .async_size             = sizeof(struct io_async_msghdr),
 517 #else
 518                 .prep                   = io_eopnotsupp_prep,
 519 #endif
 520         },
 521         [IORING_OP_RECV_ZC] = {
 522                 .needs_file             = 1,
 523                 .unbound_nonreg_file    = 1,
 524                 .pollin                 = 1,
 525                 .ioprio                 = 1,
 526 #if defined(CONFIG_NET)
 527                 .prep                   = io_recvzc_prep,
 528                 .issue                  = io_recvzc,
 529 #else
 530                 .prep                   = io_eopnotsupp_prep,
 531 #endif
 532         },
 533         [IORING_OP_EPOLL_WAIT] = {
 534                 .needs_file             = 1,
 535                 .audit_skip             = 1,
 536                 .pollin                 = 1,
 537 #if defined(CONFIG_EPOLL)
 538                 .prep                   = io_epoll_wait_prep,
 539                 .issue                  = io_epoll_wait,
 540 #else
 541                 .prep                   = io_eopnotsupp_prep,
 542 #endif
 543         },
 544         [IORING_OP_READV_FIXED] = {
 545                 .needs_file             = 1,
 546                 .unbound_nonreg_file    = 1,
 547                 .pollin                 = 1,
 548                 .plug                   = 1,
 549                 .audit_skip             = 1,
 550                 .ioprio                 = 1,
 551                 .iopoll                 = 1,
 552                 .iopoll_queue           = 1,
 553                 .vectored               = 1,
 554                 .async_size             = sizeof(struct io_async_rw),
 555                 .prep                   = io_prep_readv_fixed,
 556                 .issue                  = io_read,
 557         },
 558         [IORING_OP_WRITEV_FIXED] = {
 559                 .needs_file             = 1,
 560                 .hash_reg_file          = 1,
 561                 .unbound_nonreg_file    = 1,
 562                 .pollout                = 1,
 563                 .plug                   = 1,
 564                 .audit_skip             = 1,
 565                 .ioprio                 = 1,
 566                 .iopoll                 = 1,
 567                 .iopoll_queue           = 1,
 568                 .vectored               = 1,
 569                 .async_size             = sizeof(struct io_async_rw),
 570                 .prep                   = io_prep_writev_fixed,
 571                 .issue                  = io_write,
 572         },
 573         [IORING_OP_PIPE] = {
 574                 .prep                   = io_pipe_prep,
 575                 .issue                  = io_pipe,
 576         },
 577 };
 578
 579 const struct io_cold_def io_cold_defs[] = {
 580         [IORING_OP_NOP] = {
 581                 .name                   = "NOP",
 582         },
 583         [IORING_OP_READV] = {
 584                 .name                   = "READV",
 585                 .cleanup                = io_readv_writev_cleanup,
 586                 .fail                   = io_rw_fail,
 587         },
 588         [IORING_OP_WRITEV] = {
 589                 .name                   = "WRITEV",
 590                 .cleanup                = io_readv_writev_cleanup,
 591                 .fail                   = io_rw_fail,
 592         },
 593         [IORING_OP_FSYNC] = {
 594                 .name                   = "FSYNC",
 595         },
 596         [IORING_OP_READ_FIXED] = {
 597                 .name                   = "READ_FIXED",
 598                 .cleanup                = io_readv_writev_cleanup,
 599                 .fail                   = io_rw_fail,
 600         },
 601         [IORING_OP_WRITE_FIXED] = {
 602                 .name                   = "WRITE_FIXED",
 603                 .cleanup                = io_readv_writev_cleanup,
 604                 .fail                   = io_rw_fail,
 605         },
 606         [IORING_OP_POLL_ADD] = {
 607                 .name                   = "POLL_ADD",
 608         },
 609         [IORING_OP_POLL_REMOVE] = {
 610                 .name                   = "POLL_REMOVE",
 611         },
 612         [IORING_OP_SYNC_FILE_RANGE] = {
 613                 .name                   = "SYNC_FILE_RANGE",
 614         },
 615         [IORING_OP_SENDMSG] = {
 616                 .name                   = "SENDMSG",
 617 #if defined(CONFIG_NET)
 618                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 619                 .fail                   = io_sendrecv_fail,
 620 #endif
 621         },
 622         [IORING_OP_RECVMSG] = {
 623                 .name                   = "RECVMSG",
 624 #if defined(CONFIG_NET)
 625                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 626                 .fail                   = io_sendrecv_fail,
 627 #endif
 628         },
 629         [IORING_OP_TIMEOUT] = {
 630                 .name                   = "TIMEOUT",
 631         },
 632         [IORING_OP_TIMEOUT_REMOVE] = {
 633                 .name                   = "TIMEOUT_REMOVE",
 634         },
 635         [IORING_OP_ACCEPT] = {
 636                 .name                   = "ACCEPT",
 637         },
 638         [IORING_OP_ASYNC_CANCEL] = {
 639                 .name                   = "ASYNC_CANCEL",
 640         },
 641         [IORING_OP_LINK_TIMEOUT] = {
 642                 .name                   = "LINK_TIMEOUT",
 643         },
 644         [IORING_OP_CONNECT] = {
 645                 .name                   = "CONNECT",
 646         },
 647         [IORING_OP_FALLOCATE] = {
 648                 .name                   = "FALLOCATE",
 649         },
 650         [IORING_OP_OPENAT] = {
 651                 .name                   = "OPENAT",
 652                 .cleanup                = io_open_cleanup,
 653         },
 654         [IORING_OP_CLOSE] = {
 655                 .name                   = "CLOSE",
 656         },
 657         [IORING_OP_FILES_UPDATE] = {
 658                 .name                   = "FILES_UPDATE",
 659         },
 660         [IORING_OP_STATX] = {
 661                 .name                   = "STATX",
 662                 .cleanup                = io_statx_cleanup,
 663         },
 664         [IORING_OP_READ] = {
 665                 .name                   = "READ",
 666                 .cleanup                = io_readv_writev_cleanup,
 667                 .fail                   = io_rw_fail,
 668         },
 669         [IORING_OP_WRITE] = {
 670                 .name                   = "WRITE",
 671                 .cleanup                = io_readv_writev_cleanup,
 672                 .fail                   = io_rw_fail,
 673         },
 674         [IORING_OP_FADVISE] = {
 675                 .name                   = "FADVISE",
 676         },
 677         [IORING_OP_MADVISE] = {
 678                 .name                   = "MADVISE",
 679         },
 680         [IORING_OP_SEND] = {
 681                 .name                   = "SEND",
 682 #if defined(CONFIG_NET)
 683                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 684                 .fail                   = io_sendrecv_fail,
 685 #endif
 686         },
 687         [IORING_OP_RECV] = {
 688                 .name                   = "RECV",
 689 #if defined(CONFIG_NET)
 690                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 691                 .fail                   = io_sendrecv_fail,
 692 #endif
 693         },
 694         [IORING_OP_OPENAT2] = {
 695                 .name                   = "OPENAT2",
 696                 .cleanup                = io_open_cleanup,
 697         },
 698         [IORING_OP_EPOLL_CTL] = {
 699                 .name                   = "EPOLL",
 700         },
 701         [IORING_OP_SPLICE] = {
 702                 .name                   = "SPLICE",
 703                 .cleanup                = io_splice_cleanup,
 704         },
 705         [IORING_OP_PROVIDE_BUFFERS] = {
 706                 .name                   = "PROVIDE_BUFFERS",
 707         },
 708         [IORING_OP_REMOVE_BUFFERS] = {
 709                 .name                   = "REMOVE_BUFFERS",
 710         },
 711         [IORING_OP_TEE] = {
 712                 .name                   = "TEE",
 713                 .cleanup                = io_splice_cleanup,
 714         },
 715         [IORING_OP_SHUTDOWN] = {
 716                 .name                   = "SHUTDOWN",
 717         },
 718         [IORING_OP_RENAMEAT] = {
 719                 .name                   = "RENAMEAT",
 720                 .cleanup                = io_renameat_cleanup,
 721         },
 722         [IORING_OP_UNLINKAT] = {
 723                 .name                   = "UNLINKAT",
 724                 .cleanup                = io_unlinkat_cleanup,
 725         },
 726         [IORING_OP_MKDIRAT] = {
 727                 .name                   = "MKDIRAT",
 728                 .cleanup                = io_mkdirat_cleanup,
 729         },
 730         [IORING_OP_SYMLINKAT] = {
 731                 .name                   = "SYMLINKAT",
 732                 .cleanup                = io_link_cleanup,
 733         },
 734         [IORING_OP_LINKAT] = {
 735                 .name                   = "LINKAT",
 736                 .cleanup                = io_link_cleanup,
 737         },
 738         [IORING_OP_MSG_RING] = {
 739                 .name                   = "MSG_RING",
 740                 .cleanup                = io_msg_ring_cleanup,
 741         },
 742         [IORING_OP_FSETXATTR] = {
 743                 .name                   = "FSETXATTR",
 744                 .cleanup                = io_xattr_cleanup,
 745         },
 746         [IORING_OP_SETXATTR] = {
 747                 .name                   = "SETXATTR",
 748                 .cleanup                = io_xattr_cleanup,
 749         },
 750         [IORING_OP_FGETXATTR] = {
 751                 .name                   = "FGETXATTR",
 752                 .cleanup                = io_xattr_cleanup,
 753         },
 754         [IORING_OP_GETXATTR] = {
 755                 .name                   = "GETXATTR",
 756                 .cleanup                = io_xattr_cleanup,
 757         },
 758         [IORING_OP_SOCKET] = {
 759                 .name                   = "SOCKET",
 760         },
 761         [IORING_OP_URING_CMD] = {
 762                 .name                   = "URING_CMD",
 763                 .cleanup                = io_uring_cmd_cleanup,
 764         },
 765         [IORING_OP_SEND_ZC] = {
 766                 .name                   = "SEND_ZC",
 767 #if defined(CONFIG_NET)
 768                 .cleanup                = io_send_zc_cleanup,
 769                 .fail                   = io_sendrecv_fail,
 770 #endif
 771         },
 772         [IORING_OP_SENDMSG_ZC] = {
 773                 .name                   = "SENDMSG_ZC",
 774 #if defined(CONFIG_NET)
 775                 .cleanup                = io_send_zc_cleanup,
 776                 .fail                   = io_sendrecv_fail,
 777 #endif
 778         },
 779         [IORING_OP_READ_MULTISHOT] = {
 780                 .name                   = "READ_MULTISHOT",
 781                 .cleanup                = io_readv_writev_cleanup,
 782         },
 783         [IORING_OP_WAITID] = {
 784                 .name                   = "WAITID",
 785         },
 786         [IORING_OP_FUTEX_WAIT] = {
 787                 .name                   = "FUTEX_WAIT",
 788         },
 789         [IORING_OP_FUTEX_WAKE] = {
 790                 .name                   = "FUTEX_WAKE",
 791         },
 792         [IORING_OP_FUTEX_WAITV] = {
 793                 .name                   = "FUTEX_WAITV",
 794         },
 795         [IORING_OP_FIXED_FD_INSTALL] = {
 796                 .name                   = "FIXED_FD_INSTALL",
 797         },
 798         [IORING_OP_FTRUNCATE] = {
 799                 .name                   = "FTRUNCATE",
 800         },
 801         [IORING_OP_BIND] = {
 802                 .name                   = "BIND",
 803         },
 804         [IORING_OP_LISTEN] = {
 805                 .name                   = "LISTEN",
 806         },
 807         [IORING_OP_RECV_ZC] = {
 808                 .name                   = "RECV_ZC",
 809         },
 810         [IORING_OP_EPOLL_WAIT] = {
 811                 .name                   = "EPOLL_WAIT",
 812         },
 813         [IORING_OP_READV_FIXED] = {
 814                 .name                   = "READV_FIXED",
 815                 .cleanup                = io_readv_writev_cleanup,
 816                 .fail                   = io_rw_fail,
 817         },
 818         [IORING_OP_WRITEV_FIXED] = {
 819                 .name                   = "WRITEV_FIXED",
 820                 .cleanup                = io_readv_writev_cleanup,
 821                 .fail                   = io_rw_fail,
 822         },
 823         [IORING_OP_PIPE] = {
 824                 .name                   = "PIPE",
 825         },
 826 };
 827
 828 const char *io_uring_get_opcode(u8 opcode)
 829 {
 830         if (opcode < IORING_OP_LAST)
 831                 return io_cold_defs[opcode].name;
 832         return "INVALID";
 833 }
 834
 835 bool io_uring_op_supported(u8 opcode)
 836 {
 837         if (opcode < IORING_OP_LAST &&
 838             io_issue_defs[opcode].prep != io_eopnotsupp_prep)
 839                 return true;
 840         return false;
 841 }
 842
 843 void __init io_uring_optable_init(void)
 844 {
 845         int i;
 846
 847         BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
 848         BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
 849
 850         for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
 851                 BUG_ON(!io_issue_defs[i].prep);
 852                 if (io_issue_defs[i].prep != io_eopnotsupp_prep)
 853                         BUG_ON(!io_issue_defs[i].issue);
 854                 WARN_ON_ONCE(!io_cold_defs[i].name);
 855         }
 856 }