Commit | Line | Data |
---|---|---|
d9b57aa3 JA |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * io_uring opcode handling table | |
4 | */ | |
5 | #include <linux/kernel.h> | |
6 | #include <linux/errno.h> | |
7 | #include <linux/fs.h> | |
8 | #include <linux/file.h> | |
9 | #include <linux/io_uring.h> | |
10 | ||
d9b57aa3 JA |
11 | #include "io_uring.h" |
12 | #include "opdef.h" | |
13 | #include "refs.h" | |
14 | #include "tctx.h" | |
15 | #include "sqpoll.h" | |
16 | #include "fdinfo.h" | |
17 | #include "kbuf.h" | |
18 | #include "rsrc.h" | |
19 | ||
20 | #include "xattr.h" | |
21 | #include "nop.h" | |
22 | #include "fs.h" | |
23 | #include "splice.h" | |
24 | #include "sync.h" | |
25 | #include "advise.h" | |
26 | #include "openclose.h" | |
27 | #include "uring_cmd.h" | |
28 | #include "epoll.h" | |
29 | #include "statx.h" | |
30 | #include "net.h" | |
31 | #include "msg_ring.h" | |
32 | #include "timeout.h" | |
33 | #include "poll.h" | |
34 | #include "cancel.h" | |
35 | #include "rw.h" | |
f31ecf67 | 36 | #include "waitid.h" |
194bb58c | 37 | #include "futex.h" |
d9b57aa3 JA |
38 | |
39 | static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags) | |
40 | { | |
41 | WARN_ON_ONCE(1); | |
42 | return -ECANCELED; | |
43 | } | |
44 | ||
45 | static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb, | |
46 | const struct io_uring_sqe *sqe) | |
47 | { | |
48 | return -EOPNOTSUPP; | |
49 | } | |
50 | ||
a7dd2782 | 51 | const struct io_issue_def io_issue_defs[] = { |
d9b57aa3 JA |
52 | [IORING_OP_NOP] = { |
53 | .audit_skip = 1, | |
54 | .iopoll = 1, | |
d9b57aa3 JA |
55 | .prep = io_nop_prep, |
56 | .issue = io_nop, | |
57 | }, | |
58 | [IORING_OP_READV] = { | |
59 | .needs_file = 1, | |
60 | .unbound_nonreg_file = 1, | |
61 | .pollin = 1, | |
62 | .buffer_select = 1, | |
63 | .plug = 1, | |
64 | .audit_skip = 1, | |
65 | .ioprio = 1, | |
66 | .iopoll = 1, | |
ef0ec1ad | 67 | .iopoll_queue = 1, |
d2d778fb | 68 | .vectored = 1, |
0e984ec8 | 69 | .prep = io_prep_rwv, |
d9b57aa3 | 70 | .issue = io_read, |
d9b57aa3 JA |
71 | }, |
72 | [IORING_OP_WRITEV] = { | |
73 | .needs_file = 1, | |
74 | .hash_reg_file = 1, | |
75 | .unbound_nonreg_file = 1, | |
76 | .pollout = 1, | |
77 | .plug = 1, | |
78 | .audit_skip = 1, | |
79 | .ioprio = 1, | |
80 | .iopoll = 1, | |
ef0ec1ad | 81 | .iopoll_queue = 1, |
d2d778fb | 82 | .vectored = 1, |
0e984ec8 | 83 | .prep = io_prep_rwv, |
d9b57aa3 | 84 | .issue = io_write, |
d9b57aa3 JA |
85 | }, |
86 | [IORING_OP_FSYNC] = { | |
87 | .needs_file = 1, | |
88 | .audit_skip = 1, | |
d9b57aa3 JA |
89 | .prep = io_fsync_prep, |
90 | .issue = io_fsync, | |
91 | }, | |
92 | [IORING_OP_READ_FIXED] = { | |
93 | .needs_file = 1, | |
94 | .unbound_nonreg_file = 1, | |
95 | .pollin = 1, | |
96 | .plug = 1, | |
97 | .audit_skip = 1, | |
98 | .ioprio = 1, | |
99 | .iopoll = 1, | |
ef0ec1ad | 100 | .iopoll_queue = 1, |
f688944c | 101 | .prep = io_prep_rw_fixed, |
d9b57aa3 JA |
102 | .issue = io_read, |
103 | }, | |
104 | [IORING_OP_WRITE_FIXED] = { | |
105 | .needs_file = 1, | |
106 | .hash_reg_file = 1, | |
107 | .unbound_nonreg_file = 1, | |
108 | .pollout = 1, | |
109 | .plug = 1, | |
110 | .audit_skip = 1, | |
111 | .ioprio = 1, | |
112 | .iopoll = 1, | |
ef0ec1ad | 113 | .iopoll_queue = 1, |
f688944c | 114 | .prep = io_prep_rw_fixed, |
d9b57aa3 JA |
115 | .issue = io_write, |
116 | }, | |
117 | [IORING_OP_POLL_ADD] = { | |
118 | .needs_file = 1, | |
119 | .unbound_nonreg_file = 1, | |
120 | .audit_skip = 1, | |
d9b57aa3 JA |
121 | .prep = io_poll_add_prep, |
122 | .issue = io_poll_add, | |
123 | }, | |
124 | [IORING_OP_POLL_REMOVE] = { | |
125 | .audit_skip = 1, | |
d9b57aa3 JA |
126 | .prep = io_poll_remove_prep, |
127 | .issue = io_poll_remove, | |
128 | }, | |
129 | [IORING_OP_SYNC_FILE_RANGE] = { | |
130 | .needs_file = 1, | |
131 | .audit_skip = 1, | |
d9b57aa3 JA |
132 | .prep = io_sfr_prep, |
133 | .issue = io_sync_file_range, | |
134 | }, | |
135 | [IORING_OP_SENDMSG] = { | |
136 | .needs_file = 1, | |
137 | .unbound_nonreg_file = 1, | |
138 | .pollout = 1, | |
139 | .ioprio = 1, | |
858c293e | 140 | .manual_alloc = 1, |
d9b57aa3 | 141 | #if defined(CONFIG_NET) |
d9b57aa3 JA |
142 | .prep = io_sendmsg_prep, |
143 | .issue = io_sendmsg, | |
d9b57aa3 JA |
144 | #else |
145 | .prep = io_eopnotsupp_prep, | |
146 | #endif | |
147 | }, | |
148 | [IORING_OP_RECVMSG] = { | |
149 | .needs_file = 1, | |
150 | .unbound_nonreg_file = 1, | |
151 | .pollin = 1, | |
152 | .buffer_select = 1, | |
153 | .ioprio = 1, | |
858c293e | 154 | .manual_alloc = 1, |
d9b57aa3 | 155 | #if defined(CONFIG_NET) |
d9b57aa3 JA |
156 | .prep = io_recvmsg_prep, |
157 | .issue = io_recvmsg, | |
d9b57aa3 JA |
158 | #else |
159 | .prep = io_eopnotsupp_prep, | |
160 | #endif | |
161 | }, | |
162 | [IORING_OP_TIMEOUT] = { | |
163 | .audit_skip = 1, | |
d9b57aa3 JA |
164 | .prep = io_timeout_prep, |
165 | .issue = io_timeout, | |
166 | }, | |
167 | [IORING_OP_TIMEOUT_REMOVE] = { | |
168 | /* used by timeout updates' prep() */ | |
169 | .audit_skip = 1, | |
d9b57aa3 JA |
170 | .prep = io_timeout_remove_prep, |
171 | .issue = io_timeout_remove, | |
172 | }, | |
173 | [IORING_OP_ACCEPT] = { | |
174 | .needs_file = 1, | |
175 | .unbound_nonreg_file = 1, | |
176 | .pollin = 1, | |
177 | .poll_exclusive = 1, | |
178 | .ioprio = 1, /* used for flags */ | |
d9b57aa3 JA |
179 | #if defined(CONFIG_NET) |
180 | .prep = io_accept_prep, | |
181 | .issue = io_accept, | |
182 | #else | |
183 | .prep = io_eopnotsupp_prep, | |
184 | #endif | |
185 | }, | |
186 | [IORING_OP_ASYNC_CANCEL] = { | |
187 | .audit_skip = 1, | |
d9b57aa3 JA |
188 | .prep = io_async_cancel_prep, |
189 | .issue = io_async_cancel, | |
190 | }, | |
191 | [IORING_OP_LINK_TIMEOUT] = { | |
192 | .audit_skip = 1, | |
d9b57aa3 JA |
193 | .prep = io_link_timeout_prep, |
194 | .issue = io_no_issue, | |
195 | }, | |
196 | [IORING_OP_CONNECT] = { | |
197 | .needs_file = 1, | |
198 | .unbound_nonreg_file = 1, | |
199 | .pollout = 1, | |
d9b57aa3 | 200 | #if defined(CONFIG_NET) |
d9b57aa3 JA |
201 | .prep = io_connect_prep, |
202 | .issue = io_connect, | |
d9b57aa3 JA |
203 | #else |
204 | .prep = io_eopnotsupp_prep, | |
205 | #endif | |
206 | }, | |
207 | [IORING_OP_FALLOCATE] = { | |
208 | .needs_file = 1, | |
d9b57aa3 JA |
209 | .prep = io_fallocate_prep, |
210 | .issue = io_fallocate, | |
211 | }, | |
212 | [IORING_OP_OPENAT] = { | |
d9b57aa3 JA |
213 | .prep = io_openat_prep, |
214 | .issue = io_openat, | |
d9b57aa3 JA |
215 | }, |
216 | [IORING_OP_CLOSE] = { | |
d9b57aa3 JA |
217 | .prep = io_close_prep, |
218 | .issue = io_close, | |
219 | }, | |
d9808ceb | 220 | [IORING_OP_FILES_UPDATE] = { |
d9b57aa3 JA |
221 | .audit_skip = 1, |
222 | .iopoll = 1, | |
d9808ceb PB |
223 | .prep = io_files_update_prep, |
224 | .issue = io_files_update, | |
d9b57aa3 JA |
225 | }, |
226 | [IORING_OP_STATX] = { | |
227 | .audit_skip = 1, | |
d9b57aa3 JA |
228 | .prep = io_statx_prep, |
229 | .issue = io_statx, | |
d9b57aa3 JA |
230 | }, |
231 | [IORING_OP_READ] = { | |
232 | .needs_file = 1, | |
233 | .unbound_nonreg_file = 1, | |
234 | .pollin = 1, | |
235 | .buffer_select = 1, | |
236 | .plug = 1, | |
237 | .audit_skip = 1, | |
238 | .ioprio = 1, | |
239 | .iopoll = 1, | |
ef0ec1ad | 240 | .iopoll_queue = 1, |
d9b57aa3 JA |
241 | .prep = io_prep_rw, |
242 | .issue = io_read, | |
243 | }, | |
244 | [IORING_OP_WRITE] = { | |
245 | .needs_file = 1, | |
246 | .hash_reg_file = 1, | |
247 | .unbound_nonreg_file = 1, | |
248 | .pollout = 1, | |
249 | .plug = 1, | |
250 | .audit_skip = 1, | |
251 | .ioprio = 1, | |
252 | .iopoll = 1, | |
ef0ec1ad | 253 | .iopoll_queue = 1, |
d9b57aa3 JA |
254 | .prep = io_prep_rw, |
255 | .issue = io_write, | |
256 | }, | |
257 | [IORING_OP_FADVISE] = { | |
258 | .needs_file = 1, | |
259 | .audit_skip = 1, | |
d9b57aa3 JA |
260 | .prep = io_fadvise_prep, |
261 | .issue = io_fadvise, | |
262 | }, | |
263 | [IORING_OP_MADVISE] = { | |
fbe870a7 | 264 | .audit_skip = 1, |
d9b57aa3 JA |
265 | .prep = io_madvise_prep, |
266 | .issue = io_madvise, | |
267 | }, | |
268 | [IORING_OP_SEND] = { | |
269 | .needs_file = 1, | |
270 | .unbound_nonreg_file = 1, | |
271 | .pollout = 1, | |
272 | .audit_skip = 1, | |
273 | .ioprio = 1, | |
516e82f0 | 274 | .manual_alloc = 1, |
d9b57aa3 JA |
275 | #if defined(CONFIG_NET) |
276 | .prep = io_sendmsg_prep, | |
277 | .issue = io_send, | |
278 | #else | |
279 | .prep = io_eopnotsupp_prep, | |
280 | #endif | |
281 | }, | |
282 | [IORING_OP_RECV] = { | |
283 | .needs_file = 1, | |
284 | .unbound_nonreg_file = 1, | |
285 | .pollin = 1, | |
286 | .buffer_select = 1, | |
287 | .audit_skip = 1, | |
288 | .ioprio = 1, | |
d9b57aa3 JA |
289 | #if defined(CONFIG_NET) |
290 | .prep = io_recvmsg_prep, | |
291 | .issue = io_recv, | |
292 | #else | |
293 | .prep = io_eopnotsupp_prep, | |
294 | #endif | |
295 | }, | |
296 | [IORING_OP_OPENAT2] = { | |
d9b57aa3 JA |
297 | .prep = io_openat2_prep, |
298 | .issue = io_openat2, | |
d9b57aa3 JA |
299 | }, |
300 | [IORING_OP_EPOLL_CTL] = { | |
301 | .unbound_nonreg_file = 1, | |
302 | .audit_skip = 1, | |
d9b57aa3 JA |
303 | #if defined(CONFIG_EPOLL) |
304 | .prep = io_epoll_ctl_prep, | |
305 | .issue = io_epoll_ctl, | |
306 | #else | |
307 | .prep = io_eopnotsupp_prep, | |
308 | #endif | |
309 | }, | |
310 | [IORING_OP_SPLICE] = { | |
311 | .needs_file = 1, | |
312 | .hash_reg_file = 1, | |
313 | .unbound_nonreg_file = 1, | |
314 | .audit_skip = 1, | |
d9b57aa3 JA |
315 | .prep = io_splice_prep, |
316 | .issue = io_splice, | |
317 | }, | |
318 | [IORING_OP_PROVIDE_BUFFERS] = { | |
319 | .audit_skip = 1, | |
320 | .iopoll = 1, | |
d9b57aa3 JA |
321 | .prep = io_provide_buffers_prep, |
322 | .issue = io_provide_buffers, | |
323 | }, | |
324 | [IORING_OP_REMOVE_BUFFERS] = { | |
325 | .audit_skip = 1, | |
326 | .iopoll = 1, | |
d9b57aa3 JA |
327 | .prep = io_remove_buffers_prep, |
328 | .issue = io_remove_buffers, | |
329 | }, | |
330 | [IORING_OP_TEE] = { | |
331 | .needs_file = 1, | |
332 | .hash_reg_file = 1, | |
333 | .unbound_nonreg_file = 1, | |
334 | .audit_skip = 1, | |
d9b57aa3 JA |
335 | .prep = io_tee_prep, |
336 | .issue = io_tee, | |
337 | }, | |
338 | [IORING_OP_SHUTDOWN] = { | |
339 | .needs_file = 1, | |
d9b57aa3 JA |
340 | #if defined(CONFIG_NET) |
341 | .prep = io_shutdown_prep, | |
342 | .issue = io_shutdown, | |
343 | #else | |
344 | .prep = io_eopnotsupp_prep, | |
345 | #endif | |
346 | }, | |
347 | [IORING_OP_RENAMEAT] = { | |
d9b57aa3 JA |
348 | .prep = io_renameat_prep, |
349 | .issue = io_renameat, | |
d9b57aa3 JA |
350 | }, |
351 | [IORING_OP_UNLINKAT] = { | |
d9b57aa3 JA |
352 | .prep = io_unlinkat_prep, |
353 | .issue = io_unlinkat, | |
d9b57aa3 JA |
354 | }, |
355 | [IORING_OP_MKDIRAT] = { | |
d9b57aa3 JA |
356 | .prep = io_mkdirat_prep, |
357 | .issue = io_mkdirat, | |
d9b57aa3 JA |
358 | }, |
359 | [IORING_OP_SYMLINKAT] = { | |
d9b57aa3 JA |
360 | .prep = io_symlinkat_prep, |
361 | .issue = io_symlinkat, | |
d9b57aa3 JA |
362 | }, |
363 | [IORING_OP_LINKAT] = { | |
d9b57aa3 JA |
364 | .prep = io_linkat_prep, |
365 | .issue = io_linkat, | |
d9b57aa3 JA |
366 | }, |
367 | [IORING_OP_MSG_RING] = { | |
368 | .needs_file = 1, | |
369 | .iopoll = 1, | |
d9b57aa3 JA |
370 | .prep = io_msg_ring_prep, |
371 | .issue = io_msg_ring, | |
372 | }, | |
373 | [IORING_OP_FSETXATTR] = { | |
374 | .needs_file = 1, | |
d9b57aa3 JA |
375 | .prep = io_fsetxattr_prep, |
376 | .issue = io_fsetxattr, | |
d9b57aa3 JA |
377 | }, |
378 | [IORING_OP_SETXATTR] = { | |
d9b57aa3 JA |
379 | .prep = io_setxattr_prep, |
380 | .issue = io_setxattr, | |
d9b57aa3 JA |
381 | }, |
382 | [IORING_OP_FGETXATTR] = { | |
383 | .needs_file = 1, | |
d9b57aa3 JA |
384 | .prep = io_fgetxattr_prep, |
385 | .issue = io_fgetxattr, | |
d9b57aa3 JA |
386 | }, |
387 | [IORING_OP_GETXATTR] = { | |
d9b57aa3 JA |
388 | .prep = io_getxattr_prep, |
389 | .issue = io_getxattr, | |
d9b57aa3 JA |
390 | }, |
391 | [IORING_OP_SOCKET] = { | |
392 | .audit_skip = 1, | |
d9b57aa3 JA |
393 | #if defined(CONFIG_NET) |
394 | .prep = io_socket_prep, | |
395 | .issue = io_socket, | |
396 | #else | |
397 | .prep = io_eopnotsupp_prep, | |
398 | #endif | |
399 | }, | |
400 | [IORING_OP_URING_CMD] = { | |
401 | .needs_file = 1, | |
402 | .plug = 1, | |
5756a3a7 | 403 | .iopoll = 1, |
ef0ec1ad | 404 | .iopoll_queue = 1, |
d9b57aa3 JA |
405 | .prep = io_uring_cmd_prep, |
406 | .issue = io_uring_cmd, | |
d9b57aa3 | 407 | }, |
b48c312b | 408 | [IORING_OP_SEND_ZC] = { |
06a5464b PB |
409 | .needs_file = 1, |
410 | .unbound_nonreg_file = 1, | |
411 | .pollout = 1, | |
412 | .audit_skip = 1, | |
413 | .ioprio = 1, | |
581711c4 | 414 | .manual_alloc = 1, |
06a5464b | 415 | #if defined(CONFIG_NET) |
b0e9b551 PB |
416 | .prep = io_send_zc_prep, |
417 | .issue = io_send_zc, | |
06a5464b PB |
418 | #else |
419 | .prep = io_eopnotsupp_prep, | |
493108d9 PB |
420 | #endif |
421 | }, | |
422 | [IORING_OP_SENDMSG_ZC] = { | |
493108d9 PB |
423 | .needs_file = 1, |
424 | .unbound_nonreg_file = 1, | |
425 | .pollout = 1, | |
493108d9 PB |
426 | .ioprio = 1, |
427 | .manual_alloc = 1, | |
428 | #if defined(CONFIG_NET) | |
493108d9 PB |
429 | .prep = io_send_zc_prep, |
430 | .issue = io_sendmsg_zc, | |
f30bd4d0 BL |
431 | #else |
432 | .prep = io_eopnotsupp_prep, | |
433 | #endif | |
434 | }, | |
fc68fcda JA |
435 | [IORING_OP_READ_MULTISHOT] = { |
436 | .needs_file = 1, | |
437 | .unbound_nonreg_file = 1, | |
438 | .pollin = 1, | |
439 | .buffer_select = 1, | |
440 | .audit_skip = 1, | |
441 | .prep = io_read_mshot_prep, | |
442 | .issue = io_read_mshot, | |
443 | }, | |
f31ecf67 JA |
444 | [IORING_OP_WAITID] = { |
445 | .prep = io_waitid_prep, | |
446 | .issue = io_waitid, | |
447 | }, | |
194bb58c JA |
448 | [IORING_OP_FUTEX_WAIT] = { |
449 | #if defined(CONFIG_FUTEX) | |
450 | .prep = io_futex_prep, | |
451 | .issue = io_futex_wait, | |
452 | #else | |
453 | .prep = io_eopnotsupp_prep, | |
454 | #endif | |
455 | }, | |
456 | [IORING_OP_FUTEX_WAKE] = { | |
457 | #if defined(CONFIG_FUTEX) | |
458 | .prep = io_futex_prep, | |
459 | .issue = io_futex_wake, | |
460 | #else | |
461 | .prep = io_eopnotsupp_prep, | |
8f350194 JA |
462 | #endif |
463 | }, | |
464 | [IORING_OP_FUTEX_WAITV] = { | |
465 | #if defined(CONFIG_FUTEX) | |
466 | .prep = io_futexv_prep, | |
467 | .issue = io_futexv_wait, | |
468 | #else | |
469 | .prep = io_eopnotsupp_prep, | |
194bb58c JA |
470 | #endif |
471 | }, | |
f30bd4d0 BL |
472 | }; |
473 | ||
f30bd4d0 BL |
474 | const struct io_cold_def io_cold_defs[] = { |
475 | [IORING_OP_NOP] = { | |
476 | .name = "NOP", | |
477 | }, | |
478 | [IORING_OP_READV] = { | |
479 | .async_size = sizeof(struct io_async_rw), | |
480 | .name = "READV", | |
481 | .prep_async = io_readv_prep_async, | |
482 | .cleanup = io_readv_writev_cleanup, | |
483 | .fail = io_rw_fail, | |
484 | }, | |
485 | [IORING_OP_WRITEV] = { | |
486 | .async_size = sizeof(struct io_async_rw), | |
487 | .name = "WRITEV", | |
488 | .prep_async = io_writev_prep_async, | |
489 | .cleanup = io_readv_writev_cleanup, | |
490 | .fail = io_rw_fail, | |
491 | }, | |
492 | [IORING_OP_FSYNC] = { | |
493 | .name = "FSYNC", | |
494 | }, | |
495 | [IORING_OP_READ_FIXED] = { | |
496 | .async_size = sizeof(struct io_async_rw), | |
497 | .name = "READ_FIXED", | |
498 | .fail = io_rw_fail, | |
499 | }, | |
500 | [IORING_OP_WRITE_FIXED] = { | |
501 | .async_size = sizeof(struct io_async_rw), | |
502 | .name = "WRITE_FIXED", | |
503 | .fail = io_rw_fail, | |
504 | }, | |
505 | [IORING_OP_POLL_ADD] = { | |
506 | .name = "POLL_ADD", | |
507 | }, | |
508 | [IORING_OP_POLL_REMOVE] = { | |
509 | .name = "POLL_REMOVE", | |
510 | }, | |
511 | [IORING_OP_SYNC_FILE_RANGE] = { | |
512 | .name = "SYNC_FILE_RANGE", | |
513 | }, | |
514 | [IORING_OP_SENDMSG] = { | |
515 | .name = "SENDMSG", | |
516 | #if defined(CONFIG_NET) | |
517 | .async_size = sizeof(struct io_async_msghdr), | |
518 | .prep_async = io_sendmsg_prep_async, | |
519 | .cleanup = io_sendmsg_recvmsg_cleanup, | |
520 | .fail = io_sendrecv_fail, | |
521 | #endif | |
522 | }, | |
523 | [IORING_OP_RECVMSG] = { | |
524 | .name = "RECVMSG", | |
525 | #if defined(CONFIG_NET) | |
526 | .async_size = sizeof(struct io_async_msghdr), | |
527 | .prep_async = io_recvmsg_prep_async, | |
528 | .cleanup = io_sendmsg_recvmsg_cleanup, | |
529 | .fail = io_sendrecv_fail, | |
530 | #endif | |
531 | }, | |
532 | [IORING_OP_TIMEOUT] = { | |
533 | .async_size = sizeof(struct io_timeout_data), | |
534 | .name = "TIMEOUT", | |
535 | }, | |
536 | [IORING_OP_TIMEOUT_REMOVE] = { | |
537 | .name = "TIMEOUT_REMOVE", | |
538 | }, | |
539 | [IORING_OP_ACCEPT] = { | |
540 | .name = "ACCEPT", | |
541 | }, | |
542 | [IORING_OP_ASYNC_CANCEL] = { | |
543 | .name = "ASYNC_CANCEL", | |
544 | }, | |
545 | [IORING_OP_LINK_TIMEOUT] = { | |
546 | .async_size = sizeof(struct io_timeout_data), | |
547 | .name = "LINK_TIMEOUT", | |
548 | }, | |
549 | [IORING_OP_CONNECT] = { | |
550 | .name = "CONNECT", | |
551 | #if defined(CONFIG_NET) | |
552 | .async_size = sizeof(struct io_async_connect), | |
553 | .prep_async = io_connect_prep_async, | |
554 | #endif | |
555 | }, | |
556 | [IORING_OP_FALLOCATE] = { | |
557 | .name = "FALLOCATE", | |
558 | }, | |
559 | [IORING_OP_OPENAT] = { | |
560 | .name = "OPENAT", | |
561 | .cleanup = io_open_cleanup, | |
562 | }, | |
563 | [IORING_OP_CLOSE] = { | |
564 | .name = "CLOSE", | |
565 | }, | |
566 | [IORING_OP_FILES_UPDATE] = { | |
567 | .name = "FILES_UPDATE", | |
568 | }, | |
569 | [IORING_OP_STATX] = { | |
570 | .name = "STATX", | |
571 | .cleanup = io_statx_cleanup, | |
572 | }, | |
573 | [IORING_OP_READ] = { | |
574 | .async_size = sizeof(struct io_async_rw), | |
575 | .name = "READ", | |
576 | .fail = io_rw_fail, | |
577 | }, | |
578 | [IORING_OP_WRITE] = { | |
579 | .async_size = sizeof(struct io_async_rw), | |
580 | .name = "WRITE", | |
581 | .fail = io_rw_fail, | |
582 | }, | |
583 | [IORING_OP_FADVISE] = { | |
584 | .name = "FADVISE", | |
585 | }, | |
586 | [IORING_OP_MADVISE] = { | |
587 | .name = "MADVISE", | |
588 | }, | |
589 | [IORING_OP_SEND] = { | |
590 | .name = "SEND", | |
591 | #if defined(CONFIG_NET) | |
592 | .async_size = sizeof(struct io_async_msghdr), | |
593 | .fail = io_sendrecv_fail, | |
594 | .prep_async = io_send_prep_async, | |
595 | #endif | |
596 | }, | |
597 | [IORING_OP_RECV] = { | |
598 | .name = "RECV", | |
599 | #if defined(CONFIG_NET) | |
600 | .fail = io_sendrecv_fail, | |
601 | #endif | |
602 | }, | |
603 | [IORING_OP_OPENAT2] = { | |
604 | .name = "OPENAT2", | |
605 | .cleanup = io_open_cleanup, | |
606 | }, | |
607 | [IORING_OP_EPOLL_CTL] = { | |
608 | .name = "EPOLL", | |
609 | }, | |
610 | [IORING_OP_SPLICE] = { | |
611 | .name = "SPLICE", | |
612 | }, | |
613 | [IORING_OP_PROVIDE_BUFFERS] = { | |
614 | .name = "PROVIDE_BUFFERS", | |
615 | }, | |
616 | [IORING_OP_REMOVE_BUFFERS] = { | |
617 | .name = "REMOVE_BUFFERS", | |
618 | }, | |
619 | [IORING_OP_TEE] = { | |
620 | .name = "TEE", | |
621 | }, | |
622 | [IORING_OP_SHUTDOWN] = { | |
623 | .name = "SHUTDOWN", | |
624 | }, | |
625 | [IORING_OP_RENAMEAT] = { | |
626 | .name = "RENAMEAT", | |
627 | .cleanup = io_renameat_cleanup, | |
628 | }, | |
629 | [IORING_OP_UNLINKAT] = { | |
630 | .name = "UNLINKAT", | |
631 | .cleanup = io_unlinkat_cleanup, | |
632 | }, | |
633 | [IORING_OP_MKDIRAT] = { | |
634 | .name = "MKDIRAT", | |
635 | .cleanup = io_mkdirat_cleanup, | |
636 | }, | |
637 | [IORING_OP_SYMLINKAT] = { | |
638 | .name = "SYMLINKAT", | |
639 | .cleanup = io_link_cleanup, | |
640 | }, | |
641 | [IORING_OP_LINKAT] = { | |
642 | .name = "LINKAT", | |
643 | .cleanup = io_link_cleanup, | |
644 | }, | |
645 | [IORING_OP_MSG_RING] = { | |
646 | .name = "MSG_RING", | |
647 | .cleanup = io_msg_ring_cleanup, | |
648 | }, | |
649 | [IORING_OP_FSETXATTR] = { | |
650 | .name = "FSETXATTR", | |
651 | .cleanup = io_xattr_cleanup, | |
652 | }, | |
653 | [IORING_OP_SETXATTR] = { | |
654 | .name = "SETXATTR", | |
655 | .cleanup = io_xattr_cleanup, | |
656 | }, | |
657 | [IORING_OP_FGETXATTR] = { | |
658 | .name = "FGETXATTR", | |
659 | .cleanup = io_xattr_cleanup, | |
660 | }, | |
661 | [IORING_OP_GETXATTR] = { | |
662 | .name = "GETXATTR", | |
663 | .cleanup = io_xattr_cleanup, | |
664 | }, | |
665 | [IORING_OP_SOCKET] = { | |
666 | .name = "SOCKET", | |
667 | }, | |
668 | [IORING_OP_URING_CMD] = { | |
669 | .name = "URING_CMD", | |
fd9b8547 | 670 | .async_size = 2 * sizeof(struct io_uring_sqe), |
f30bd4d0 BL |
671 | .prep_async = io_uring_cmd_prep_async, |
672 | }, | |
673 | [IORING_OP_SEND_ZC] = { | |
674 | .name = "SEND_ZC", | |
675 | #if defined(CONFIG_NET) | |
676 | .async_size = sizeof(struct io_async_msghdr), | |
677 | .prep_async = io_send_prep_async, | |
678 | .cleanup = io_send_zc_cleanup, | |
679 | .fail = io_sendrecv_fail, | |
680 | #endif | |
681 | }, | |
682 | [IORING_OP_SENDMSG_ZC] = { | |
683 | .name = "SENDMSG_ZC", | |
684 | #if defined(CONFIG_NET) | |
685 | .async_size = sizeof(struct io_async_msghdr), | |
493108d9 PB |
686 | .prep_async = io_sendmsg_prep_async, |
687 | .cleanup = io_send_zc_cleanup, | |
688 | .fail = io_sendrecv_fail, | |
06a5464b | 689 | #endif |
06a5464b | 690 | }, |
fc68fcda JA |
691 | [IORING_OP_READ_MULTISHOT] = { |
692 | .name = "READ_MULTISHOT", | |
693 | }, | |
f31ecf67 JA |
694 | [IORING_OP_WAITID] = { |
695 | .name = "WAITID", | |
696 | .async_size = sizeof(struct io_waitid_async), | |
697 | }, | |
194bb58c JA |
698 | [IORING_OP_FUTEX_WAIT] = { |
699 | .name = "FUTEX_WAIT", | |
700 | }, | |
701 | [IORING_OP_FUTEX_WAKE] = { | |
702 | .name = "FUTEX_WAKE", | |
703 | }, | |
8f350194 JA |
704 | [IORING_OP_FUTEX_WAITV] = { |
705 | .name = "FUTEX_WAITV", | |
706 | }, | |
d9b57aa3 JA |
707 | }; |
708 | ||
709 | const char *io_uring_get_opcode(u8 opcode) | |
710 | { | |
711 | if (opcode < IORING_OP_LAST) | |
f30bd4d0 | 712 | return io_cold_defs[opcode].name; |
d9b57aa3 JA |
713 | return "INVALID"; |
714 | } | |
715 | ||
716 | void __init io_uring_optable_init(void) | |
717 | { | |
718 | int i; | |
719 | ||
f30bd4d0 | 720 | BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST); |
a7dd2782 | 721 | BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST); |
d9b57aa3 | 722 | |
a7dd2782 BL |
723 | for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) { |
724 | BUG_ON(!io_issue_defs[i].prep); | |
725 | if (io_issue_defs[i].prep != io_eopnotsupp_prep) | |
726 | BUG_ON(!io_issue_defs[i].issue); | |
f30bd4d0 | 727 | WARN_ON_ONCE(!io_cold_defs[i].name); |
d9b57aa3 JA |
728 | } |
729 | } |