io_uring/opdef.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * io_uring opcode handling table
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/errno.h>
   7 #include <linux/fs.h>
   8 #include <linux/file.h>
   9 #include <linux/io_uring.h>
  10
  11 #include "io_uring.h"
  12 #include "opdef.h"
  13 #include "refs.h"
  14 #include "tctx.h"
  15 #include "sqpoll.h"
  16 #include "fdinfo.h"
  17 #include "kbuf.h"
  18 #include "rsrc.h"
  19
  20 #include "xattr.h"
  21 #include "nop.h"
  22 #include "fs.h"
  23 #include "splice.h"
  24 #include "sync.h"
  25 #include "advise.h"
  26 #include "openclose.h"
  27 #include "uring_cmd.h"
  28 #include "epoll.h"
  29 #include "statx.h"
  30 #include "net.h"
  31 #include "msg_ring.h"
  32 #include "timeout.h"
  33 #include "poll.h"
  34 #include "cancel.h"
  35 #include "rw.h"
  36 #include "waitid.h"
  37
  38 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
  39 {
  40         WARN_ON_ONCE(1);
  41         return -ECANCELED;
  42 }
  43
  44 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
  45                                              const struct io_uring_sqe *sqe)
  46 {
  47         return -EOPNOTSUPP;
  48 }
  49
  50 const struct io_issue_def io_issue_defs[] = {
  51         [IORING_OP_NOP] = {
  52                 .audit_skip             = 1,
  53                 .iopoll                 = 1,
  54                 .prep                   = io_nop_prep,
  55                 .issue                  = io_nop,
  56         },
  57         [IORING_OP_READV] = {
  58                 .needs_file             = 1,
  59                 .unbound_nonreg_file    = 1,
  60                 .pollin                 = 1,
  61                 .buffer_select          = 1,
  62                 .plug                   = 1,
  63                 .audit_skip             = 1,
  64                 .ioprio                 = 1,
  65                 .iopoll                 = 1,
  66                 .iopoll_queue           = 1,
  67                 .vectored               = 1,
  68                 .prep                   = io_prep_rw,
  69                 .issue                  = io_read,
  70         },
  71         [IORING_OP_WRITEV] = {
  72                 .needs_file             = 1,
  73                 .hash_reg_file          = 1,
  74                 .unbound_nonreg_file    = 1,
  75                 .pollout                = 1,
  76                 .plug                   = 1,
  77                 .audit_skip             = 1,
  78                 .ioprio                 = 1,
  79                 .iopoll                 = 1,
  80                 .iopoll_queue           = 1,
  81                 .vectored               = 1,
  82                 .prep                   = io_prep_rw,
  83                 .issue                  = io_write,
  84         },
  85         [IORING_OP_FSYNC] = {
  86                 .needs_file             = 1,
  87                 .audit_skip             = 1,
  88                 .prep                   = io_fsync_prep,
  89                 .issue                  = io_fsync,
  90         },
  91         [IORING_OP_READ_FIXED] = {
  92                 .needs_file             = 1,
  93                 .unbound_nonreg_file    = 1,
  94                 .pollin                 = 1,
  95                 .plug                   = 1,
  96                 .audit_skip             = 1,
  97                 .ioprio                 = 1,
  98                 .iopoll                 = 1,
  99                 .iopoll_queue           = 1,
 100                 .prep                   = io_prep_rw,
 101                 .issue                  = io_read,
 102         },
 103         [IORING_OP_WRITE_FIXED] = {
 104                 .needs_file             = 1,
 105                 .hash_reg_file          = 1,
 106                 .unbound_nonreg_file    = 1,
 107                 .pollout                = 1,
 108                 .plug                   = 1,
 109                 .audit_skip             = 1,
 110                 .ioprio                 = 1,
 111                 .iopoll                 = 1,
 112                 .iopoll_queue           = 1,
 113                 .prep                   = io_prep_rw,
 114                 .issue                  = io_write,
 115         },
 116         [IORING_OP_POLL_ADD] = {
 117                 .needs_file             = 1,
 118                 .unbound_nonreg_file    = 1,
 119                 .audit_skip             = 1,
 120                 .prep                   = io_poll_add_prep,
 121                 .issue                  = io_poll_add,
 122         },
 123         [IORING_OP_POLL_REMOVE] = {
 124                 .audit_skip             = 1,
 125                 .prep                   = io_poll_remove_prep,
 126                 .issue                  = io_poll_remove,
 127         },
 128         [IORING_OP_SYNC_FILE_RANGE] = {
 129                 .needs_file             = 1,
 130                 .audit_skip             = 1,
 131                 .prep                   = io_sfr_prep,
 132                 .issue                  = io_sync_file_range,
 133         },
 134         [IORING_OP_SENDMSG] = {
 135                 .needs_file             = 1,
 136                 .unbound_nonreg_file    = 1,
 137                 .pollout                = 1,
 138                 .ioprio                 = 1,
 139                 .manual_alloc           = 1,
 140 #if defined(CONFIG_NET)
 141                 .prep                   = io_sendmsg_prep,
 142                 .issue                  = io_sendmsg,
 143 #else
 144                 .prep                   = io_eopnotsupp_prep,
 145 #endif
 146         },
 147         [IORING_OP_RECVMSG] = {
 148                 .needs_file             = 1,
 149                 .unbound_nonreg_file    = 1,
 150                 .pollin                 = 1,
 151                 .buffer_select          = 1,
 152                 .ioprio                 = 1,
 153                 .manual_alloc           = 1,
 154 #if defined(CONFIG_NET)
 155                 .prep                   = io_recvmsg_prep,
 156                 .issue                  = io_recvmsg,
 157 #else
 158                 .prep                   = io_eopnotsupp_prep,
 159 #endif
 160         },
 161         [IORING_OP_TIMEOUT] = {
 162                 .audit_skip             = 1,
 163                 .prep                   = io_timeout_prep,
 164                 .issue                  = io_timeout,
 165         },
 166         [IORING_OP_TIMEOUT_REMOVE] = {
 167                 /* used by timeout updates' prep() */
 168                 .audit_skip             = 1,
 169                 .prep                   = io_timeout_remove_prep,
 170                 .issue                  = io_timeout_remove,
 171         },
 172         [IORING_OP_ACCEPT] = {
 173                 .needs_file             = 1,
 174                 .unbound_nonreg_file    = 1,
 175                 .pollin                 = 1,
 176                 .poll_exclusive         = 1,
 177                 .ioprio                 = 1,    /* used for flags */
 178 #if defined(CONFIG_NET)
 179                 .prep                   = io_accept_prep,
 180                 .issue                  = io_accept,
 181 #else
 182                 .prep                   = io_eopnotsupp_prep,
 183 #endif
 184         },
 185         [IORING_OP_ASYNC_CANCEL] = {
 186                 .audit_skip             = 1,
 187                 .prep                   = io_async_cancel_prep,
 188                 .issue                  = io_async_cancel,
 189         },
 190         [IORING_OP_LINK_TIMEOUT] = {
 191                 .audit_skip             = 1,
 192                 .prep                   = io_link_timeout_prep,
 193                 .issue                  = io_no_issue,
 194         },
 195         [IORING_OP_CONNECT] = {
 196                 .needs_file             = 1,
 197                 .unbound_nonreg_file    = 1,
 198                 .pollout                = 1,
 199 #if defined(CONFIG_NET)
 200                 .prep                   = io_connect_prep,
 201                 .issue                  = io_connect,
 202 #else
 203                 .prep                   = io_eopnotsupp_prep,
 204 #endif
 205         },
 206         [IORING_OP_FALLOCATE] = {
 207                 .needs_file             = 1,
 208                 .prep                   = io_fallocate_prep,
 209                 .issue                  = io_fallocate,
 210         },
 211         [IORING_OP_OPENAT] = {
 212                 .prep                   = io_openat_prep,
 213                 .issue                  = io_openat,
 214         },
 215         [IORING_OP_CLOSE] = {
 216                 .prep                   = io_close_prep,
 217                 .issue                  = io_close,
 218         },
 219         [IORING_OP_FILES_UPDATE] = {
 220                 .audit_skip             = 1,
 221                 .iopoll                 = 1,
 222                 .prep                   = io_files_update_prep,
 223                 .issue                  = io_files_update,
 224         },
 225         [IORING_OP_STATX] = {
 226                 .audit_skip             = 1,
 227                 .prep                   = io_statx_prep,
 228                 .issue                  = io_statx,
 229         },
 230         [IORING_OP_READ] = {
 231                 .needs_file             = 1,
 232                 .unbound_nonreg_file    = 1,
 233                 .pollin                 = 1,
 234                 .buffer_select          = 1,
 235                 .plug                   = 1,
 236                 .audit_skip             = 1,
 237                 .ioprio                 = 1,
 238                 .iopoll                 = 1,
 239                 .iopoll_queue           = 1,
 240                 .prep                   = io_prep_rw,
 241                 .issue                  = io_read,
 242         },
 243         [IORING_OP_WRITE] = {
 244                 .needs_file             = 1,
 245                 .hash_reg_file          = 1,
 246                 .unbound_nonreg_file    = 1,
 247                 .pollout                = 1,
 248                 .plug                   = 1,
 249                 .audit_skip             = 1,
 250                 .ioprio                 = 1,
 251                 .iopoll                 = 1,
 252                 .iopoll_queue           = 1,
 253                 .prep                   = io_prep_rw,
 254                 .issue                  = io_write,
 255         },
 256         [IORING_OP_FADVISE] = {
 257                 .needs_file             = 1,
 258                 .audit_skip             = 1,
 259                 .prep                   = io_fadvise_prep,
 260                 .issue                  = io_fadvise,
 261         },
 262         [IORING_OP_MADVISE] = {
 263                 .audit_skip             = 1,
 264                 .prep                   = io_madvise_prep,
 265                 .issue                  = io_madvise,
 266         },
 267         [IORING_OP_SEND] = {
 268                 .needs_file             = 1,
 269                 .unbound_nonreg_file    = 1,
 270                 .pollout                = 1,
 271                 .audit_skip             = 1,
 272                 .ioprio                 = 1,
 273                 .manual_alloc           = 1,
 274 #if defined(CONFIG_NET)
 275                 .prep                   = io_sendmsg_prep,
 276                 .issue                  = io_send,
 277 #else
 278                 .prep                   = io_eopnotsupp_prep,
 279 #endif
 280         },
 281         [IORING_OP_RECV] = {
 282                 .needs_file             = 1,
 283                 .unbound_nonreg_file    = 1,
 284                 .pollin                 = 1,
 285                 .buffer_select          = 1,
 286                 .audit_skip             = 1,
 287                 .ioprio                 = 1,
 288 #if defined(CONFIG_NET)
 289                 .prep                   = io_recvmsg_prep,
 290                 .issue                  = io_recv,
 291 #else
 292                 .prep                   = io_eopnotsupp_prep,
 293 #endif
 294         },
 295         [IORING_OP_OPENAT2] = {
 296                 .prep                   = io_openat2_prep,
 297                 .issue                  = io_openat2,
 298         },
 299         [IORING_OP_EPOLL_CTL] = {
 300                 .unbound_nonreg_file    = 1,
 301                 .audit_skip             = 1,
 302 #if defined(CONFIG_EPOLL)
 303                 .prep                   = io_epoll_ctl_prep,
 304                 .issue                  = io_epoll_ctl,
 305 #else
 306                 .prep                   = io_eopnotsupp_prep,
 307 #endif
 308         },
 309         [IORING_OP_SPLICE] = {
 310                 .needs_file             = 1,
 311                 .hash_reg_file          = 1,
 312                 .unbound_nonreg_file    = 1,
 313                 .audit_skip             = 1,
 314                 .prep                   = io_splice_prep,
 315                 .issue                  = io_splice,
 316         },
 317         [IORING_OP_PROVIDE_BUFFERS] = {
 318                 .audit_skip             = 1,
 319                 .iopoll                 = 1,
 320                 .prep                   = io_provide_buffers_prep,
 321                 .issue                  = io_provide_buffers,
 322         },
 323         [IORING_OP_REMOVE_BUFFERS] = {
 324                 .audit_skip             = 1,
 325                 .iopoll                 = 1,
 326                 .prep                   = io_remove_buffers_prep,
 327                 .issue                  = io_remove_buffers,
 328         },
 329         [IORING_OP_TEE] = {
 330                 .needs_file             = 1,
 331                 .hash_reg_file          = 1,
 332                 .unbound_nonreg_file    = 1,
 333                 .audit_skip             = 1,
 334                 .prep                   = io_tee_prep,
 335                 .issue                  = io_tee,
 336         },
 337         [IORING_OP_SHUTDOWN] = {
 338                 .needs_file             = 1,
 339 #if defined(CONFIG_NET)
 340                 .prep                   = io_shutdown_prep,
 341                 .issue                  = io_shutdown,
 342 #else
 343                 .prep                   = io_eopnotsupp_prep,
 344 #endif
 345         },
 346         [IORING_OP_RENAMEAT] = {
 347                 .prep                   = io_renameat_prep,
 348                 .issue                  = io_renameat,
 349         },
 350         [IORING_OP_UNLINKAT] = {
 351                 .prep                   = io_unlinkat_prep,
 352                 .issue                  = io_unlinkat,
 353         },
 354         [IORING_OP_MKDIRAT] = {
 355                 .prep                   = io_mkdirat_prep,
 356                 .issue                  = io_mkdirat,
 357         },
 358         [IORING_OP_SYMLINKAT] = {
 359                 .prep                   = io_symlinkat_prep,
 360                 .issue                  = io_symlinkat,
 361         },
 362         [IORING_OP_LINKAT] = {
 363                 .prep                   = io_linkat_prep,
 364                 .issue                  = io_linkat,
 365         },
 366         [IORING_OP_MSG_RING] = {
 367                 .needs_file             = 1,
 368                 .iopoll                 = 1,
 369                 .prep                   = io_msg_ring_prep,
 370                 .issue                  = io_msg_ring,
 371         },
 372         [IORING_OP_FSETXATTR] = {
 373                 .needs_file = 1,
 374                 .prep                   = io_fsetxattr_prep,
 375                 .issue                  = io_fsetxattr,
 376         },
 377         [IORING_OP_SETXATTR] = {
 378                 .prep                   = io_setxattr_prep,
 379                 .issue                  = io_setxattr,
 380         },
 381         [IORING_OP_FGETXATTR] = {
 382                 .needs_file = 1,
 383                 .prep                   = io_fgetxattr_prep,
 384                 .issue                  = io_fgetxattr,
 385         },
 386         [IORING_OP_GETXATTR] = {
 387                 .prep                   = io_getxattr_prep,
 388                 .issue                  = io_getxattr,
 389         },
 390         [IORING_OP_SOCKET] = {
 391                 .audit_skip             = 1,
 392 #if defined(CONFIG_NET)
 393                 .prep                   = io_socket_prep,
 394                 .issue                  = io_socket,
 395 #else
 396                 .prep                   = io_eopnotsupp_prep,
 397 #endif
 398         },
 399         [IORING_OP_URING_CMD] = {
 400                 .needs_file             = 1,
 401                 .plug                   = 1,
 402                 .iopoll                 = 1,
 403                 .iopoll_queue           = 1,
 404                 .prep                   = io_uring_cmd_prep,
 405                 .issue                  = io_uring_cmd,
 406         },
 407         [IORING_OP_SEND_ZC] = {
 408                 .needs_file             = 1,
 409                 .unbound_nonreg_file    = 1,
 410                 .pollout                = 1,
 411                 .audit_skip             = 1,
 412                 .ioprio                 = 1,
 413                 .manual_alloc           = 1,
 414 #if defined(CONFIG_NET)
 415                 .prep                   = io_send_zc_prep,
 416                 .issue                  = io_send_zc,
 417 #else
 418                 .prep                   = io_eopnotsupp_prep,
 419 #endif
 420         },
 421         [IORING_OP_SENDMSG_ZC] = {
 422                 .needs_file             = 1,
 423                 .unbound_nonreg_file    = 1,
 424                 .pollout                = 1,
 425                 .ioprio                 = 1,
 426                 .manual_alloc           = 1,
 427 #if defined(CONFIG_NET)
 428                 .prep                   = io_send_zc_prep,
 429                 .issue                  = io_sendmsg_zc,
 430 #else
 431                 .prep                   = io_eopnotsupp_prep,
 432 #endif
 433         },
 434         [IORING_OP_READ_MULTISHOT] = {
 435                 .needs_file             = 1,
 436                 .unbound_nonreg_file    = 1,
 437                 .pollin                 = 1,
 438                 .buffer_select          = 1,
 439                 .audit_skip             = 1,
 440                 .prep                   = io_read_mshot_prep,
 441                 .issue                  = io_read_mshot,
 442         },
 443         [IORING_OP_WAITID] = {
 444                 .prep                   = io_waitid_prep,
 445                 .issue                  = io_waitid,
 446         },
 447 };
 448
 449 const struct io_cold_def io_cold_defs[] = {
 450         [IORING_OP_NOP] = {
 451                 .name                   = "NOP",
 452         },
 453         [IORING_OP_READV] = {
 454                 .async_size             = sizeof(struct io_async_rw),
 455                 .name                   = "READV",
 456                 .prep_async             = io_readv_prep_async,
 457                 .cleanup                = io_readv_writev_cleanup,
 458                 .fail                   = io_rw_fail,
 459         },
 460         [IORING_OP_WRITEV] = {
 461                 .async_size             = sizeof(struct io_async_rw),
 462                 .name                   = "WRITEV",
 463                 .prep_async             = io_writev_prep_async,
 464                 .cleanup                = io_readv_writev_cleanup,
 465                 .fail                   = io_rw_fail,
 466         },
 467         [IORING_OP_FSYNC] = {
 468                 .name                   = "FSYNC",
 469         },
 470         [IORING_OP_READ_FIXED] = {
 471                 .async_size             = sizeof(struct io_async_rw),
 472                 .name                   = "READ_FIXED",
 473                 .fail                   = io_rw_fail,
 474         },
 475         [IORING_OP_WRITE_FIXED] = {
 476                 .async_size             = sizeof(struct io_async_rw),
 477                 .name                   = "WRITE_FIXED",
 478                 .fail                   = io_rw_fail,
 479         },
 480         [IORING_OP_POLL_ADD] = {
 481                 .name                   = "POLL_ADD",
 482         },
 483         [IORING_OP_POLL_REMOVE] = {
 484                 .name                   = "POLL_REMOVE",
 485         },
 486         [IORING_OP_SYNC_FILE_RANGE] = {
 487                 .name                   = "SYNC_FILE_RANGE",
 488         },
 489         [IORING_OP_SENDMSG] = {
 490                 .name                   = "SENDMSG",
 491 #if defined(CONFIG_NET)
 492                 .async_size             = sizeof(struct io_async_msghdr),
 493                 .prep_async             = io_sendmsg_prep_async,
 494                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 495                 .fail                   = io_sendrecv_fail,
 496 #endif
 497         },
 498         [IORING_OP_RECVMSG] = {
 499                 .name                   = "RECVMSG",
 500 #if defined(CONFIG_NET)
 501                 .async_size             = sizeof(struct io_async_msghdr),
 502                 .prep_async             = io_recvmsg_prep_async,
 503                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 504                 .fail                   = io_sendrecv_fail,
 505 #endif
 506         },
 507         [IORING_OP_TIMEOUT] = {
 508                 .async_size             = sizeof(struct io_timeout_data),
 509                 .name                   = "TIMEOUT",
 510         },
 511         [IORING_OP_TIMEOUT_REMOVE] = {
 512                 .name                   = "TIMEOUT_REMOVE",
 513         },
 514         [IORING_OP_ACCEPT] = {
 515                 .name                   = "ACCEPT",
 516         },
 517         [IORING_OP_ASYNC_CANCEL] = {
 518                 .name                   = "ASYNC_CANCEL",
 519         },
 520         [IORING_OP_LINK_TIMEOUT] = {
 521                 .async_size             = sizeof(struct io_timeout_data),
 522                 .name                   = "LINK_TIMEOUT",
 523         },
 524         [IORING_OP_CONNECT] = {
 525                 .name                   = "CONNECT",
 526 #if defined(CONFIG_NET)
 527                 .async_size             = sizeof(struct io_async_connect),
 528                 .prep_async             = io_connect_prep_async,
 529 #endif
 530         },
 531         [IORING_OP_FALLOCATE] = {
 532                 .name                   = "FALLOCATE",
 533         },
 534         [IORING_OP_OPENAT] = {
 535                 .name                   = "OPENAT",
 536                 .cleanup                = io_open_cleanup,
 537         },
 538         [IORING_OP_CLOSE] = {
 539                 .name                   = "CLOSE",
 540         },
 541         [IORING_OP_FILES_UPDATE] = {
 542                 .name                   = "FILES_UPDATE",
 543         },
 544         [IORING_OP_STATX] = {
 545                 .name                   = "STATX",
 546                 .cleanup                = io_statx_cleanup,
 547         },
 548         [IORING_OP_READ] = {
 549                 .async_size             = sizeof(struct io_async_rw),
 550                 .name                   = "READ",
 551                 .fail                   = io_rw_fail,
 552         },
 553         [IORING_OP_WRITE] = {
 554                 .async_size             = sizeof(struct io_async_rw),
 555                 .name                   = "WRITE",
 556                 .fail                   = io_rw_fail,
 557         },
 558         [IORING_OP_FADVISE] = {
 559                 .name                   = "FADVISE",
 560         },
 561         [IORING_OP_MADVISE] = {
 562                 .name                   = "MADVISE",
 563         },
 564         [IORING_OP_SEND] = {
 565                 .name                   = "SEND",
 566 #if defined(CONFIG_NET)
 567                 .async_size             = sizeof(struct io_async_msghdr),
 568                 .fail                   = io_sendrecv_fail,
 569                 .prep_async             = io_send_prep_async,
 570 #endif
 571         },
 572         [IORING_OP_RECV] = {
 573                 .name                   = "RECV",
 574 #if defined(CONFIG_NET)
 575                 .fail                   = io_sendrecv_fail,
 576 #endif
 577         },
 578         [IORING_OP_OPENAT2] = {
 579                 .name                   = "OPENAT2",
 580                 .cleanup                = io_open_cleanup,
 581         },
 582         [IORING_OP_EPOLL_CTL] = {
 583                 .name                   = "EPOLL",
 584         },
 585         [IORING_OP_SPLICE] = {
 586                 .name                   = "SPLICE",
 587         },
 588         [IORING_OP_PROVIDE_BUFFERS] = {
 589                 .name                   = "PROVIDE_BUFFERS",
 590         },
 591         [IORING_OP_REMOVE_BUFFERS] = {
 592                 .name                   = "REMOVE_BUFFERS",
 593         },
 594         [IORING_OP_TEE] = {
 595                 .name                   = "TEE",
 596         },
 597         [IORING_OP_SHUTDOWN] = {
 598                 .name                   = "SHUTDOWN",
 599         },
 600         [IORING_OP_RENAMEAT] = {
 601                 .name                   = "RENAMEAT",
 602                 .cleanup                = io_renameat_cleanup,
 603         },
 604         [IORING_OP_UNLINKAT] = {
 605                 .name                   = "UNLINKAT",
 606                 .cleanup                = io_unlinkat_cleanup,
 607         },
 608         [IORING_OP_MKDIRAT] = {
 609                 .name                   = "MKDIRAT",
 610                 .cleanup                = io_mkdirat_cleanup,
 611         },
 612         [IORING_OP_SYMLINKAT] = {
 613                 .name                   = "SYMLINKAT",
 614                 .cleanup                = io_link_cleanup,
 615         },
 616         [IORING_OP_LINKAT] = {
 617                 .name                   = "LINKAT",
 618                 .cleanup                = io_link_cleanup,
 619         },
 620         [IORING_OP_MSG_RING] = {
 621                 .name                   = "MSG_RING",
 622                 .cleanup                = io_msg_ring_cleanup,
 623         },
 624         [IORING_OP_FSETXATTR] = {
 625                 .name                   = "FSETXATTR",
 626                 .cleanup                = io_xattr_cleanup,
 627         },
 628         [IORING_OP_SETXATTR] = {
 629                 .name                   = "SETXATTR",
 630                 .cleanup                = io_xattr_cleanup,
 631         },
 632         [IORING_OP_FGETXATTR] = {
 633                 .name                   = "FGETXATTR",
 634                 .cleanup                = io_xattr_cleanup,
 635         },
 636         [IORING_OP_GETXATTR] = {
 637                 .name                   = "GETXATTR",
 638                 .cleanup                = io_xattr_cleanup,
 639         },
 640         [IORING_OP_SOCKET] = {
 641                 .name                   = "SOCKET",
 642         },
 643         [IORING_OP_URING_CMD] = {
 644                 .name                   = "URING_CMD",
 645                 .async_size             = 2 * sizeof(struct io_uring_sqe),
 646                 .prep_async             = io_uring_cmd_prep_async,
 647         },
 648         [IORING_OP_SEND_ZC] = {
 649                 .name                   = "SEND_ZC",
 650 #if defined(CONFIG_NET)
 651                 .async_size             = sizeof(struct io_async_msghdr),
 652                 .prep_async             = io_send_prep_async,
 653                 .cleanup                = io_send_zc_cleanup,
 654                 .fail                   = io_sendrecv_fail,
 655 #endif
 656         },
 657         [IORING_OP_SENDMSG_ZC] = {
 658                 .name                   = "SENDMSG_ZC",
 659 #if defined(CONFIG_NET)
 660                 .async_size             = sizeof(struct io_async_msghdr),
 661                 .prep_async             = io_sendmsg_prep_async,
 662                 .cleanup                = io_send_zc_cleanup,
 663                 .fail                   = io_sendrecv_fail,
 664 #endif
 665         },
 666         [IORING_OP_READ_MULTISHOT] = {
 667                 .name                   = "READ_MULTISHOT",
 668         },
 669         [IORING_OP_WAITID] = {
 670                 .name                   = "WAITID",
 671                 .async_size             = sizeof(struct io_waitid_async),
 672         },
 673 };
 674
 675 const char *io_uring_get_opcode(u8 opcode)
 676 {
 677         if (opcode < IORING_OP_LAST)
 678                 return io_cold_defs[opcode].name;
 679         return "INVALID";
 680 }
 681
 682 void __init io_uring_optable_init(void)
 683 {
 684         int i;
 685
 686         BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
 687         BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
 688
 689         for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
 690                 BUG_ON(!io_issue_defs[i].prep);
 691                 if (io_issue_defs[i].prep != io_eopnotsupp_prep)
 692                         BUG_ON(!io_issue_defs[i].issue);
 693                 WARN_ON_ONCE(!io_cold_defs[i].name);
 694         }
 695 }