projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
io_uring: fix multishot ending when not polled
[linux-block.git] / io_uring / net.c
CommitLineData
f9ead18c
JA		 1// SPDX-License-Identifier: GPL-2.0
2#include <linux/kernel.h>
3#include <linux/errno.h>
4#include <linux/file.h>
5#include <linux/slab.h>
6#include <linux/net.h>
7#include <linux/compat.h>
8#include <net/compat.h>
9#include <linux/io_uring.h>
10
11#include <uapi/linux/io_uring.h>
12
f9ead18c 13#include "io_uring.h"
3b77495a 14#include "kbuf.h"
43e0bbbd 15#include "alloc_cache.h"
f9ead18c
JA		 16#include "net.h"
17
18#if defined(CONFIG_NET)
19struct io_shutdown {
20 struct file *file;
21 int how;
22};
23
24struct io_accept {
25 struct file *file;
26 struct sockaddr __user *addr;
27 int __user *addr_len;
28 int flags;
29 u32 file_slot;
30 unsigned long nofile;
31};
32
33struct io_socket {
34 struct file *file;
35 int domain;
36 int type;
37 int protocol;
38 int flags;
39 u32 file_slot;
40 unsigned long nofile;
41};
42
43struct io_connect {
44 struct file *file;
45 struct sockaddr __user *addr;
46 int addr_len;
47};
48
49struct io_sr_msg {
50 struct file *file;
51 union {
52 struct compat_msghdr __user *umsg_compat;
53 struct user_msghdr __user *umsg;
54 void __user *buf;
55 };
56 int msg_flags;
57 size_t len;
58 size_t done_io;
59 unsigned int flags;
60};
61
62#define IO_APOLL_MULTI_POLLED (REQ_F_APOLL_MULTISHOT | REQ_F_POLLED)
63
64int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
65{
66 struct io_shutdown *shutdown = io_kiocb_to_cmd(req);
67
68 if (unlikely(sqe->off || sqe->addr || sqe->rw_flags ||
69 sqe->buf_index || sqe->splice_fd_in))
70 return -EINVAL;
71
72 shutdown->how = READ_ONCE(sqe->len);
73 return 0;
74}
75
76int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
77{
78 struct io_shutdown *shutdown = io_kiocb_to_cmd(req);
79 struct socket *sock;
80 int ret;
81
82 if (issue_flags & IO_URING_F_NONBLOCK)
83 return -EAGAIN;
84
85 sock = sock_from_file(req->file);
86 if (unlikely(!sock))
87 return -ENOTSOCK;
88
89 ret = __sys_shutdown_sock(sock, shutdown->how);
90 io_req_set_res(req, ret, 0);
91 return IOU_OK;
92}
93
94static bool io_net_retry(struct socket *sock, int flags)
95{
96 if (!(flags & MSG_WAITALL))
97 return false;
98 return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET;
99}
100
43e0bbbd
JA		 101static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags)
102{
103 struct io_async_msghdr *hdr = req->async_data;
104
105 if (!hdr || issue_flags & IO_URING_F_UNLOCKED)
106 return;
107
108 /* Let normal cleanup path reap it if we fail adding to the cache */
109 if (io_alloc_cache_put(&req->ctx->netmsg_cache, &hdr->cache)) {
110 req->async_data = NULL;
111 req->flags &= ~REQ_F_ASYNC_DATA;
112 }
113}
114
115static struct io_async_msghdr *io_recvmsg_alloc_async(struct io_kiocb *req,
116 unsigned int issue_flags)
117{
118 struct io_ring_ctx *ctx = req->ctx;
119 struct io_cache_entry *entry;
120
121 if (!(issue_flags & IO_URING_F_UNLOCKED) &&
122 (entry = io_alloc_cache_get(&ctx->netmsg_cache)) != NULL) {
123 struct io_async_msghdr *hdr;
124
125 hdr = container_of(entry, struct io_async_msghdr, cache);
126 req->flags |= REQ_F_ASYNC_DATA;
127 req->async_data = hdr;
128 return hdr;
129 }
130
131 if (!io_alloc_async_data(req))
132 return req->async_data;
133
134 return NULL;
135}
136
f9ead18c 137static int io_setup_async_msg(struct io_kiocb *req,
43e0bbbd
JA		 138 struct io_async_msghdr *kmsg,
139 unsigned int issue_flags)
f9ead18c
JA		 140{
141 struct io_async_msghdr *async_msg = req->async_data;
142
143 if (async_msg)
144 return -EAGAIN;
43e0bbbd
JA		 145 async_msg = io_recvmsg_alloc_async(req, issue_flags);
146 if (!async_msg) {
f9ead18c
JA		 147 kfree(kmsg->free_iov);
148 return -ENOMEM;
149 }
f9ead18c
JA		 150 req->flags |= REQ_F_NEED_CLEANUP;
151 memcpy(async_msg, kmsg, sizeof(*kmsg));
152 async_msg->msg.msg_name = &async_msg->addr;
153 /* if were using fast_iov, set it to the new one */
154 if (!async_msg->free_iov)
155 async_msg->msg.msg_iter.iov = async_msg->fast_iov;
156
157 return -EAGAIN;
158}
159
160static int io_sendmsg_copy_hdr(struct io_kiocb *req,
161 struct io_async_msghdr *iomsg)
162{
163 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
164
165 iomsg->msg.msg_name = &iomsg->addr;
166 iomsg->free_iov = iomsg->fast_iov;
167 return sendmsg_copy_msghdr(&iomsg->msg, sr->umsg, sr->msg_flags,
168 &iomsg->free_iov);
169}
170
171int io_sendmsg_prep_async(struct io_kiocb *req)
172{
173 int ret;
174
175 ret = io_sendmsg_copy_hdr(req, req->async_data);
176 if (!ret)
177 req->flags |= REQ_F_NEED_CLEANUP;
178 return ret;
179}
180
181void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req)
182{
183 struct io_async_msghdr *io = req->async_data;
184
185 kfree(io->free_iov);
186}
187
188int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
189{
190 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
191
192 if (unlikely(sqe->file_index || sqe->addr2))
193 return -EINVAL;
194
195 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
196 sr->len = READ_ONCE(sqe->len);
197 sr->flags = READ_ONCE(sqe->ioprio);
198 if (sr->flags & ~IORING_RECVSEND_POLL_FIRST)
199 return -EINVAL;
200 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
201 if (sr->msg_flags & MSG_DONTWAIT)
202 req->flags |= REQ_F_NOWAIT;
203
204#ifdef CONFIG_COMPAT
205 if (req->ctx->compat)
206 sr->msg_flags |= MSG_CMSG_COMPAT;
207#endif
208 sr->done_io = 0;
209 return 0;
210}
211
212int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
213{
214 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
215 struct io_async_msghdr iomsg, *kmsg;
216 struct socket *sock;
217 unsigned flags;
218 int min_ret = 0;
219 int ret;
220
221 sock = sock_from_file(req->file);
222 if (unlikely(!sock))
223 return -ENOTSOCK;
224
225 if (req_has_async_data(req)) {
226 kmsg = req->async_data;
227 } else {
228 ret = io_sendmsg_copy_hdr(req, &iomsg);
229 if (ret)
230 return ret;
231 kmsg = &iomsg;
232 }
233
234 if (!(req->flags & REQ_F_POLLED) &&
235 (sr->flags & IORING_RECVSEND_POLL_FIRST))
43e0bbbd 236 return io_setup_async_msg(req, kmsg, issue_flags);
f9ead18c
JA		 237
238 flags = sr->msg_flags;
239 if (issue_flags & IO_URING_F_NONBLOCK)
240 flags |= MSG_DONTWAIT;
241 if (flags & MSG_WAITALL)
242 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
243
244 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
245
246 if (ret < min_ret) {
247 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
43e0bbbd 248 return io_setup_async_msg(req, kmsg, issue_flags);
f9ead18c
JA		 249 if (ret == -ERESTARTSYS)
250 ret = -EINTR;
251 if (ret > 0 && io_net_retry(sock, flags)) {
252 sr->done_io += ret;
253 req->flags |= REQ_F_PARTIAL_IO;
43e0bbbd 254 return io_setup_async_msg(req, kmsg, issue_flags);
f9ead18c
JA		 255 }
256 req_set_fail(req);
257 }
258 /* fast path, check for non-NULL to avoid function call */
259 if (kmsg->free_iov)
260 kfree(kmsg->free_iov);
261 req->flags &= ~REQ_F_NEED_CLEANUP;
43e0bbbd 262 io_netmsg_recycle(req, issue_flags);
f9ead18c
JA		 263 if (ret >= 0)
264 ret += sr->done_io;
265 else if (sr->done_io)
266 ret = sr->done_io;
267 io_req_set_res(req, ret, 0);
268 return IOU_OK;
269}
270
271int io_send(struct io_kiocb *req, unsigned int issue_flags)
272{
273 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
274 struct msghdr msg;
275 struct iovec iov;
276 struct socket *sock;
277 unsigned flags;
278 int min_ret = 0;
279 int ret;
280
281 if (!(req->flags & REQ_F_POLLED) &&
282 (sr->flags & IORING_RECVSEND_POLL_FIRST))
283 return -EAGAIN;
284
285 sock = sock_from_file(req->file);
286 if (unlikely(!sock))
287 return -ENOTSOCK;
288
289 ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
290 if (unlikely(ret))
291 return ret;
292
293 msg.msg_name = NULL;
294 msg.msg_control = NULL;
295 msg.msg_controllen = 0;
296 msg.msg_namelen = 0;
297
298 flags = sr->msg_flags;
299 if (issue_flags & IO_URING_F_NONBLOCK)
300 flags |= MSG_DONTWAIT;
301 if (flags & MSG_WAITALL)
302 min_ret = iov_iter_count(&msg.msg_iter);
303
304 msg.msg_flags = flags;
305 ret = sock_sendmsg(sock, &msg);
306 if (ret < min_ret) {
307 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
308 return -EAGAIN;
309 if (ret == -ERESTARTSYS)
310 ret = -EINTR;
311 if (ret > 0 && io_net_retry(sock, flags)) {
312 sr->len -= ret;
313 sr->buf += ret;
314 sr->done_io += ret;
315 req->flags |= REQ_F_PARTIAL_IO;
316 return -EAGAIN;
317 }
318 req_set_fail(req);
319 }
320 if (ret >= 0)
321 ret += sr->done_io;
322 else if (sr->done_io)
323 ret = sr->done_io;
324 io_req_set_res(req, ret, 0);
325 return IOU_OK;
326}
327
328static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
329 struct io_async_msghdr *iomsg)
330{
331 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
332 struct iovec __user *uiov;
333 size_t iov_len;
334 int ret;
335
336 ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg,
337 &iomsg->uaddr, &uiov, &iov_len);
338 if (ret)
339 return ret;
340
341 if (req->flags & REQ_F_BUFFER_SELECT) {
5702196e
DY		 342 if (iov_len == 0) {
343 sr->len = iomsg->fast_iov[0].iov_len = 0;
344 iomsg->fast_iov[0].iov_base = NULL;
345 iomsg->free_iov = NULL;
346 } else if (iov_len > 1) {
f9ead18c 347 return -EINVAL;
5702196e
DY		 348 } else {
349 if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov)))
350 return -EFAULT;
351 sr->len = iomsg->fast_iov[0].iov_len;
352 iomsg->free_iov = NULL;
353 }
f9ead18c
JA		 354 } else {
355 iomsg->free_iov = iomsg->fast_iov;
356 ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
357 &iomsg->free_iov, &iomsg->msg.msg_iter,
358 false);
359 if (ret > 0)
360 ret = 0;
361 }
362
363 return ret;
364}
365
366#ifdef CONFIG_COMPAT
367static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
368 struct io_async_msghdr *iomsg)
369{
370 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
371 struct compat_iovec __user *uiov;
372 compat_uptr_t ptr;
373 compat_size_t len;
374 int ret;
375
376 ret = __get_compat_msghdr(&iomsg->msg, sr->umsg_compat, &iomsg->uaddr,
377 &ptr, &len);
378 if (ret)
379 return ret;
380
381 uiov = compat_ptr(ptr);
382 if (req->flags & REQ_F_BUFFER_SELECT) {
383 compat_ssize_t clen;
384
385 if (len > 1)
386 return -EINVAL;
387 if (!access_ok(uiov, sizeof(*uiov)))
388 return -EFAULT;
389 if (__get_user(clen, &uiov->iov_len))
390 return -EFAULT;
391 if (clen < 0)
392 return -EINVAL;
393 sr->len = clen;
394 iomsg->free_iov = NULL;
395 } else {
396 iomsg->free_iov = iomsg->fast_iov;
397 ret = __import_iovec(READ, (struct iovec __user *)uiov, len,
398 UIO_FASTIOV, &iomsg->free_iov,
399 &iomsg->msg.msg_iter, true);
400 if (ret < 0)
401 return ret;
402 }
403
404 return 0;
405}
406#endif
407
408static int io_recvmsg_copy_hdr(struct io_kiocb *req,
409 struct io_async_msghdr *iomsg)
410{
411 iomsg->msg.msg_name = &iomsg->addr;
412
413#ifdef CONFIG_COMPAT
414 if (req->ctx->compat)
415 return __io_compat_recvmsg_copy_hdr(req, iomsg);
416#endif
417
418 return __io_recvmsg_copy_hdr(req, iomsg);
419}
420
421int io_recvmsg_prep_async(struct io_kiocb *req)
422{
423 int ret;
424
425 ret = io_recvmsg_copy_hdr(req, req->async_data);
426 if (!ret)
427 req->flags |= REQ_F_NEED_CLEANUP;
428 return ret;
429}
430
b3fdea6e
DY		 431#define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT)
432
f9ead18c
JA		 433int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
434{
435 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
436
437 if (unlikely(sqe->file_index || sqe->addr2))
438 return -EINVAL;
439
440 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
441 sr->len = READ_ONCE(sqe->len);
442 sr->flags = READ_ONCE(sqe->ioprio);
b3fdea6e 443 if (sr->flags & ~(RECVMSG_FLAGS))
f9ead18c
JA		 444 return -EINVAL;
445 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
446 if (sr->msg_flags & MSG_DONTWAIT)
447 req->flags |= REQ_F_NOWAIT;
448 if (sr->msg_flags & MSG_ERRQUEUE)
449 req->flags |= REQ_F_CLEAR_POLLIN;
b3fdea6e 450 if (sr->flags & IORING_RECV_MULTISHOT) {
cf0dd952
DY		 451 if (req->opcode == IORING_OP_RECVMSG)
452 return -EINVAL;
b3fdea6e
DY		 453 if (!(req->flags & REQ_F_BUFFER_SELECT))
454 return -EINVAL;
455 if (sr->msg_flags & MSG_WAITALL)
456 return -EINVAL;
457 if (req->opcode == IORING_OP_RECV && sr->len)
458 return -EINVAL;
459 req->flags |= REQ_F_APOLL_MULTISHOT;
460 }
f9ead18c
JA		 461
462#ifdef CONFIG_COMPAT
463 if (req->ctx->compat)
464 sr->msg_flags |= MSG_CMSG_COMPAT;
465#endif
466 sr->done_io = 0;
467 return 0;
468}
469
b3fdea6e
DY		 470static inline void io_recv_prep_retry(struct io_kiocb *req)
471{
472 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
473
474 sr->done_io = 0;
475 sr->len = 0; /* get from the provided buffer */
476}
477
478/*
cf0dd952 479 * Finishes io_recv
b3fdea6e
DY		 480 *
481 * Returns true if it is actually finished, or false if it should run
482 * again (for multishot).
483 */
484static inline bool io_recv_finish(struct io_kiocb *req, int *ret, unsigned int cflags)
485{
486 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
487 io_req_set_res(req, *ret, cflags);
488 *ret = IOU_OK;
489 return true;
490 }
491
492 if (*ret > 0) {
493 if (io_post_aux_cqe(req->ctx, req->cqe.user_data, *ret,
494 cflags | IORING_CQE_F_MORE, false)) {
495 io_recv_prep_retry(req);
496 return false;
497 }
498 /*
499 * Otherwise stop multishot but use the current result.
500 * Probably will end up going into overflow, but this means
501 * we cannot trust the ordering anymore
502 */
503 }
504
505 io_req_set_res(req, *ret, cflags);
506
507 if (req->flags & REQ_F_POLLED)
508 *ret = IOU_STOP_MULTISHOT;
e2df2ccb
DY		 509 else
510 *ret = IOU_OK;
b3fdea6e
DY		 511 return true;
512}
513
f9ead18c
JA		 514int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
515{
516 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
517 struct io_async_msghdr iomsg, *kmsg;
518 struct socket *sock;
519 unsigned int cflags;
520 unsigned flags;
521 int ret, min_ret = 0;
522 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
523
524 sock = sock_from_file(req->file);
525 if (unlikely(!sock))
526 return -ENOTSOCK;
527
528 if (req_has_async_data(req)) {
529 kmsg = req->async_data;
530 } else {
531 ret = io_recvmsg_copy_hdr(req, &iomsg);
532 if (ret)
533 return ret;
534 kmsg = &iomsg;
535 }
536
537 if (!(req->flags & REQ_F_POLLED) &&
538 (sr->flags & IORING_RECVSEND_POLL_FIRST))
43e0bbbd 539 return io_setup_async_msg(req, kmsg, issue_flags);
f9ead18c
JA		 540
541 if (io_do_buffer_select(req)) {
542 void __user *buf;
543
cf0dd952 544 buf = io_buffer_select(req, &sr->len, issue_flags);
f9ead18c
JA		 545 if (!buf)
546 return -ENOBUFS;
547 kmsg->fast_iov[0].iov_base = buf;
cf0dd952 548 kmsg->fast_iov[0].iov_len = sr->len;
f9ead18c 549 iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov, 1,
cf0dd952 550 sr->len);
f9ead18c
JA		 551 }
552
553 flags = sr->msg_flags;
554 if (force_nonblock)
555 flags |= MSG_DONTWAIT;
556 if (flags & MSG_WAITALL)
557 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
558
559 kmsg->msg.msg_get_inq = 1;
560 ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, kmsg->uaddr, flags);
561 if (ret < min_ret) {
cf0dd952 562 if (ret == -EAGAIN && force_nonblock)
43e0bbbd 563 return io_setup_async_msg(req, kmsg, issue_flags);
f9ead18c
JA		 564 if (ret == -ERESTARTSYS)
565 ret = -EINTR;
566 if (ret > 0 && io_net_retry(sock, flags)) {
567 sr->done_io += ret;
568 req->flags |= REQ_F_PARTIAL_IO;
43e0bbbd 569 return io_setup_async_msg(req, kmsg, issue_flags);
f9ead18c
JA		 570 }
571 req_set_fail(req);
572 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
573 req_set_fail(req);
574 }
575
576 /* fast path, check for non-NULL to avoid function call */
577 if (kmsg->free_iov)
578 kfree(kmsg->free_iov);
43e0bbbd 579 io_netmsg_recycle(req, issue_flags);
f9ead18c 580 req->flags &= ~REQ_F_NEED_CLEANUP;
d4e097da 581 if (ret > 0)
f9ead18c
JA		 582 ret += sr->done_io;
583 else if (sr->done_io)
584 ret = sr->done_io;
d4e097da
DY		 585 else
586 io_kbuf_recycle(req, issue_flags);
587
f9ead18c
JA		 588 cflags = io_put_kbuf(req, issue_flags);
589 if (kmsg->msg.msg_inq)
590 cflags |= IORING_CQE_F_SOCK_NONEMPTY;
b3fdea6e 591
cf0dd952
DY		 592 io_req_set_res(req, ret, cflags);
593 return IOU_OK;
f9ead18c
JA		 594}
595
596int io_recv(struct io_kiocb *req, unsigned int issue_flags)
597{
598 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
599 struct msghdr msg;
600 struct socket *sock;
601 struct iovec iov;
602 unsigned int cflags;
603 unsigned flags;
604 int ret, min_ret = 0;
605 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
b3fdea6e 606 size_t len = sr->len;
f9ead18c
JA		 607
608 if (!(req->flags & REQ_F_POLLED) &&
609 (sr->flags & IORING_RECVSEND_POLL_FIRST))
610 return -EAGAIN;
611
612 sock = sock_from_file(req->file);
613 if (unlikely(!sock))
614 return -ENOTSOCK;
615
b3fdea6e 616retry_multishot:
f9ead18c
JA		 617 if (io_do_buffer_select(req)) {
618 void __user *buf;
619
b3fdea6e 620 buf = io_buffer_select(req, &len, issue_flags);
f9ead18c
JA		 621 if (!buf)
622 return -ENOBUFS;
623 sr->buf = buf;
624 }
625
b3fdea6e 626 ret = import_single_range(READ, sr->buf, len, &iov, &msg.msg_iter);
f9ead18c
JA		 627 if (unlikely(ret))
628 goto out_free;
629
630 msg.msg_name = NULL;
631 msg.msg_namelen = 0;
632 msg.msg_control = NULL;
633 msg.msg_get_inq = 1;
634 msg.msg_flags = 0;
635 msg.msg_controllen = 0;
636 msg.msg_iocb = NULL;
637
638 flags = sr->msg_flags;
639 if (force_nonblock)
640 flags |= MSG_DONTWAIT;
641 if (flags & MSG_WAITALL)
642 min_ret = iov_iter_count(&msg.msg_iter);
643
644 ret = sock_recvmsg(sock, &msg, flags);
645 if (ret < min_ret) {
b3fdea6e
DY		 646 if (ret == -EAGAIN && force_nonblock) {
647 if ((req->flags & IO_APOLL_MULTI_POLLED) == IO_APOLL_MULTI_POLLED) {
648 io_kbuf_recycle(req, issue_flags);
649 return IOU_ISSUE_SKIP_COMPLETE;
650 }
651
f9ead18c 652 return -EAGAIN;
b3fdea6e 653 }
f9ead18c
JA		 654 if (ret == -ERESTARTSYS)
655 ret = -EINTR;
656 if (ret > 0 && io_net_retry(sock, flags)) {
657 sr->len -= ret;
658 sr->buf += ret;
659 sr->done_io += ret;
660 req->flags |= REQ_F_PARTIAL_IO;
661 return -EAGAIN;
662 }
663 req_set_fail(req);
664 } else if ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
665out_free:
666 req_set_fail(req);
667 }
668
d4e097da 669 if (ret > 0)
f9ead18c
JA		 670 ret += sr->done_io;
671 else if (sr->done_io)
672 ret = sr->done_io;
d4e097da
DY		 673 else
674 io_kbuf_recycle(req, issue_flags);
675
f9ead18c
JA		 676 cflags = io_put_kbuf(req, issue_flags);
677 if (msg.msg_inq)
678 cflags |= IORING_CQE_F_SOCK_NONEMPTY;
b3fdea6e
DY		 679
680 if (!io_recv_finish(req, &ret, cflags))
681 goto retry_multishot;
682
683 return ret;
f9ead18c
JA		 684}
685
686int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
687{
688 struct io_accept *accept = io_kiocb_to_cmd(req);
689 unsigned flags;
690
691 if (sqe->len || sqe->buf_index)
692 return -EINVAL;
693
694 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
695 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
696 accept->flags = READ_ONCE(sqe->accept_flags);
697 accept->nofile = rlimit(RLIMIT_NOFILE);
698 flags = READ_ONCE(sqe->ioprio);
699 if (flags & ~IORING_ACCEPT_MULTISHOT)
700 return -EINVAL;
701
702 accept->file_slot = READ_ONCE(sqe->file_index);
703 if (accept->file_slot) {
704 if (accept->flags & SOCK_CLOEXEC)
705 return -EINVAL;
706 if (flags & IORING_ACCEPT_MULTISHOT &&
707 accept->file_slot != IORING_FILE_INDEX_ALLOC)
708 return -EINVAL;
709 }
710 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
711 return -EINVAL;
712 if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK))
713 accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK;
714 if (flags & IORING_ACCEPT_MULTISHOT)
715 req->flags |= REQ_F_APOLL_MULTISHOT;
716 return 0;
717}
718
719int io_accept(struct io_kiocb *req, unsigned int issue_flags)
720{
721 struct io_ring_ctx *ctx = req->ctx;
722 struct io_accept *accept = io_kiocb_to_cmd(req);
723 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
724 unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
725 bool fixed = !!accept->file_slot;
726 struct file *file;
727 int ret, fd;
728
729retry:
730 if (!fixed) {
731 fd = __get_unused_fd_flags(accept->flags, accept->nofile);
732 if (unlikely(fd < 0))
733 return fd;
734 }
735 file = do_accept(req->file, file_flags, accept->addr, accept->addr_len,
736 accept->flags);
737 if (IS_ERR(file)) {
738 if (!fixed)
739 put_unused_fd(fd);
740 ret = PTR_ERR(file);
741 if (ret == -EAGAIN && force_nonblock) {
742 /*
743 * if it's multishot and polled, we don't need to
744 * return EAGAIN to arm the poll infra since it
745 * has already been done
746 */
747 if ((req->flags & IO_APOLL_MULTI_POLLED) ==
748 IO_APOLL_MULTI_POLLED)
749 ret = IOU_ISSUE_SKIP_COMPLETE;
750 return ret;
751 }
752 if (ret == -ERESTARTSYS)
753 ret = -EINTR;
754 req_set_fail(req);
755 } else if (!fixed) {
756 fd_install(fd, file);
757 ret = fd;
758 } else {
759 ret = io_fixed_fd_install(req, issue_flags, file,
760 accept->file_slot);
761 }
762
763 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
764 io_req_set_res(req, ret, 0);
765 return IOU_OK;
766 }
f9ead18c 767
cbd25748
DY		 768 if (ret >= 0 &&
769 io_post_aux_cqe(ctx, req->cqe.user_data, ret, IORING_CQE_F_MORE, false))
d245bca6 770 goto retry;
cbd25748
DY		 771
772 io_req_set_res(req, ret, 0);
773 if (req->flags & REQ_F_POLLED)
774 return IOU_STOP_MULTISHOT;
775 return IOU_OK;
f9ead18c
JA		 776}
777
778int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
779{
780 struct io_socket *sock = io_kiocb_to_cmd(req);
781
782 if (sqe->addr || sqe->rw_flags || sqe->buf_index)
783 return -EINVAL;
784
785 sock->domain = READ_ONCE(sqe->fd);
786 sock->type = READ_ONCE(sqe->off);
787 sock->protocol = READ_ONCE(sqe->len);
788 sock->file_slot = READ_ONCE(sqe->file_index);
789 sock->nofile = rlimit(RLIMIT_NOFILE);
790
791 sock->flags = sock->type & ~SOCK_TYPE_MASK;
792 if (sock->file_slot && (sock->flags & SOCK_CLOEXEC))
793 return -EINVAL;
794 if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
795 return -EINVAL;
796 return 0;
797}
798
799int io_socket(struct io_kiocb *req, unsigned int issue_flags)
800{
801 struct io_socket *sock = io_kiocb_to_cmd(req);
802 bool fixed = !!sock->file_slot;
803 struct file *file;
804 int ret, fd;
805
806 if (!fixed) {
807 fd = __get_unused_fd_flags(sock->flags, sock->nofile);
808 if (unlikely(fd < 0))
809 return fd;
810 }
811 file = __sys_socket_file(sock->domain, sock->type, sock->protocol);
812 if (IS_ERR(file)) {
813 if (!fixed)
814 put_unused_fd(fd);
815 ret = PTR_ERR(file);
816 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
817 return -EAGAIN;
818 if (ret == -ERESTARTSYS)
819 ret = -EINTR;
820 req_set_fail(req);
821 } else if (!fixed) {
822 fd_install(fd, file);
823 ret = fd;
824 } else {
825 ret = io_fixed_fd_install(req, issue_flags, file,
826 sock->file_slot);
827 }
828 io_req_set_res(req, ret, 0);
829 return IOU_OK;
830}
831
832int io_connect_prep_async(struct io_kiocb *req)
833{
834 struct io_async_connect *io = req->async_data;
835 struct io_connect *conn = io_kiocb_to_cmd(req);
836
837 return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address);
838}
839
840int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
841{
842 struct io_connect *conn = io_kiocb_to_cmd(req);
843
844 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
845 return -EINVAL;
846
847 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
848 conn->addr_len = READ_ONCE(sqe->addr2);
849 return 0;
850}
851
852int io_connect(struct io_kiocb *req, unsigned int issue_flags)
853{
854 struct io_connect *connect = io_kiocb_to_cmd(req);
855 struct io_async_connect __io, *io;
856 unsigned file_flags;
857 int ret;
858 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
859
860 if (req_has_async_data(req)) {
861 io = req->async_data;
862 } else {
863 ret = move_addr_to_kernel(connect->addr,
864 connect->addr_len,
865 &__io.address);
866 if (ret)
867 goto out;
868 io = &__io;
869 }
870
871 file_flags = force_nonblock ? O_NONBLOCK : 0;
872
873 ret = __sys_connect_file(req->file, &io->address,
874 connect->addr_len, file_flags);
875 if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
876 if (req_has_async_data(req))
877 return -EAGAIN;
878 if (io_alloc_async_data(req)) {
879 ret = -ENOMEM;
880 goto out;
881 }
882 memcpy(req->async_data, &__io, sizeof(__io));
883 return -EAGAIN;
884 }
885 if (ret == -ERESTARTSYS)
886 ret = -EINTR;
887out:
888 if (ret < 0)
889 req_set_fail(req);
890 io_req_set_res(req, ret, 0);
891 return IOU_OK;
892}
43e0bbbd
JA		 893
894void io_netmsg_cache_free(struct io_cache_entry *entry)
895{
896 kfree(container_of(entry, struct io_async_msghdr, cache));
897}
f9ead18c 898#endif
Linux 6.x block layer and io_uring tree(s)