projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
io_uring: support 0 length iov in buffer select in compat
[linux-block.git] / io_uring / net.c
CommitLineData
f9ead18c
JA		 1// SPDX-License-Identifier: GPL-2.0
2#include <linux/kernel.h>
3#include <linux/errno.h>
4#include <linux/file.h>
5#include <linux/slab.h>
6#include <linux/net.h>
7#include <linux/compat.h>
8#include <net/compat.h>
9#include <linux/io_uring.h>
10
11#include <uapi/linux/io_uring.h>
12
f9ead18c 13#include "io_uring.h"
3b77495a 14#include "kbuf.h"
43e0bbbd 15#include "alloc_cache.h"
f9ead18c
JA		 16#include "net.h"
17
18#if defined(CONFIG_NET)
19struct io_shutdown {
20 struct file *file;
21 int how;
22};
23
24struct io_accept {
25 struct file *file;
26 struct sockaddr __user *addr;
27 int __user *addr_len;
28 int flags;
29 u32 file_slot;
30 unsigned long nofile;
31};
32
33struct io_socket {
34 struct file *file;
35 int domain;
36 int type;
37 int protocol;
38 int flags;
39 u32 file_slot;
40 unsigned long nofile;
41};
42
43struct io_connect {
44 struct file *file;
45 struct sockaddr __user *addr;
46 int addr_len;
47};
48
49struct io_sr_msg {
50 struct file *file;
51 union {
52 struct compat_msghdr __user *umsg_compat;
53 struct user_msghdr __user *umsg;
54 void __user *buf;
55 };
56 int msg_flags;
57 size_t len;
58 size_t done_io;
59 unsigned int flags;
60};
61
62#define IO_APOLL_MULTI_POLLED (REQ_F_APOLL_MULTISHOT | REQ_F_POLLED)
63
64int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
65{
66 struct io_shutdown *shutdown = io_kiocb_to_cmd(req);
67
68 if (unlikely(sqe->off || sqe->addr || sqe->rw_flags ||
69 sqe->buf_index || sqe->splice_fd_in))
70 return -EINVAL;
71
72 shutdown->how = READ_ONCE(sqe->len);
73 return 0;
74}
75
76int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
77{
78 struct io_shutdown *shutdown = io_kiocb_to_cmd(req);
79 struct socket *sock;
80 int ret;
81
82 if (issue_flags & IO_URING_F_NONBLOCK)
83 return -EAGAIN;
84
85 sock = sock_from_file(req->file);
86 if (unlikely(!sock))
87 return -ENOTSOCK;
88
89 ret = __sys_shutdown_sock(sock, shutdown->how);
90 io_req_set_res(req, ret, 0);
91 return IOU_OK;
92}
93
94static bool io_net_retry(struct socket *sock, int flags)
95{
96 if (!(flags & MSG_WAITALL))
97 return false;
98 return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET;
99}
100
43e0bbbd
JA		 101static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags)
102{
103 struct io_async_msghdr *hdr = req->async_data;
104
105 if (!hdr || issue_flags & IO_URING_F_UNLOCKED)
106 return;
107
108 /* Let normal cleanup path reap it if we fail adding to the cache */
109 if (io_alloc_cache_put(&req->ctx->netmsg_cache, &hdr->cache)) {
110 req->async_data = NULL;
111 req->flags &= ~REQ_F_ASYNC_DATA;
112 }
113}
114
115static struct io_async_msghdr *io_recvmsg_alloc_async(struct io_kiocb *req,
116 unsigned int issue_flags)
117{
118 struct io_ring_ctx *ctx = req->ctx;
119 struct io_cache_entry *entry;
120
121 if (!(issue_flags & IO_URING_F_UNLOCKED) &&
122 (entry = io_alloc_cache_get(&ctx->netmsg_cache)) != NULL) {
123 struct io_async_msghdr *hdr;
124
125 hdr = container_of(entry, struct io_async_msghdr, cache);
126 req->flags |= REQ_F_ASYNC_DATA;
127 req->async_data = hdr;
128 return hdr;
129 }
130
131 if (!io_alloc_async_data(req))
132 return req->async_data;
133
134 return NULL;
135}
136
f9ead18c 137static int io_setup_async_msg(struct io_kiocb *req,
43e0bbbd
JA		 138 struct io_async_msghdr *kmsg,
139 unsigned int issue_flags)
f9ead18c
JA		 140{
141 struct io_async_msghdr *async_msg = req->async_data;
142
143 if (async_msg)
144 return -EAGAIN;
43e0bbbd
JA		 145 async_msg = io_recvmsg_alloc_async(req, issue_flags);
146 if (!async_msg) {
f9ead18c
JA		 147 kfree(kmsg->free_iov);
148 return -ENOMEM;
149 }
f9ead18c
JA		 150 req->flags |= REQ_F_NEED_CLEANUP;
151 memcpy(async_msg, kmsg, sizeof(*kmsg));
152 async_msg->msg.msg_name = &async_msg->addr;
153 /* if were using fast_iov, set it to the new one */
154 if (!async_msg->free_iov)
155 async_msg->msg.msg_iter.iov = async_msg->fast_iov;
156
157 return -EAGAIN;
158}
159
160static int io_sendmsg_copy_hdr(struct io_kiocb *req,
161 struct io_async_msghdr *iomsg)
162{
163 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
164
165 iomsg->msg.msg_name = &iomsg->addr;
166 iomsg->free_iov = iomsg->fast_iov;
167 return sendmsg_copy_msghdr(&iomsg->msg, sr->umsg, sr->msg_flags,
168 &iomsg->free_iov);
169}
170
171int io_sendmsg_prep_async(struct io_kiocb *req)
172{
173 int ret;
174
175 ret = io_sendmsg_copy_hdr(req, req->async_data);
176 if (!ret)
177 req->flags |= REQ_F_NEED_CLEANUP;
178 return ret;
179}
180
181void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req)
182{
183 struct io_async_msghdr *io = req->async_data;
184
185 kfree(io->free_iov);
186}
187
188int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
189{
190 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
191
192 if (unlikely(sqe->file_index || sqe->addr2))
193 return -EINVAL;
194
195 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
196 sr->len = READ_ONCE(sqe->len);
197 sr->flags = READ_ONCE(sqe->ioprio);
198 if (sr->flags & ~IORING_RECVSEND_POLL_FIRST)
199 return -EINVAL;
200 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
201 if (sr->msg_flags & MSG_DONTWAIT)
202 req->flags |= REQ_F_NOWAIT;
203
204#ifdef CONFIG_COMPAT
205 if (req->ctx->compat)
206 sr->msg_flags |= MSG_CMSG_COMPAT;
207#endif
208 sr->done_io = 0;
209 return 0;
210}
211
212int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
213{
214 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
215 struct io_async_msghdr iomsg, *kmsg;
216 struct socket *sock;
217 unsigned flags;
218 int min_ret = 0;
219 int ret;
220
221 sock = sock_from_file(req->file);
222 if (unlikely(!sock))
223 return -ENOTSOCK;
224
225 if (req_has_async_data(req)) {
226 kmsg = req->async_data;
227 } else {
228 ret = io_sendmsg_copy_hdr(req, &iomsg);
229 if (ret)
230 return ret;
231 kmsg = &iomsg;
232 }
233
234 if (!(req->flags & REQ_F_POLLED) &&
235 (sr->flags & IORING_RECVSEND_POLL_FIRST))
43e0bbbd 236 return io_setup_async_msg(req, kmsg, issue_flags);
f9ead18c
JA		 237
238 flags = sr->msg_flags;
239 if (issue_flags & IO_URING_F_NONBLOCK)
240 flags |= MSG_DONTWAIT;
241 if (flags & MSG_WAITALL)
242 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
243
244 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
245
246 if (ret < min_ret) {
247 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
43e0bbbd 248 return io_setup_async_msg(req, kmsg, issue_flags);
f9ead18c
JA		 249 if (ret == -ERESTARTSYS)
250 ret = -EINTR;
251 if (ret > 0 && io_net_retry(sock, flags)) {
252 sr->done_io += ret;
253 req->flags |= REQ_F_PARTIAL_IO;
43e0bbbd 254 return io_setup_async_msg(req, kmsg, issue_flags);
f9ead18c
JA		 255 }
256 req_set_fail(req);
257 }
258 /* fast path, check for non-NULL to avoid function call */
259 if (kmsg->free_iov)
260 kfree(kmsg->free_iov);
261 req->flags &= ~REQ_F_NEED_CLEANUP;
43e0bbbd 262 io_netmsg_recycle(req, issue_flags);
f9ead18c
JA		 263 if (ret >= 0)
264 ret += sr->done_io;
265 else if (sr->done_io)
266 ret = sr->done_io;
267 io_req_set_res(req, ret, 0);
268 return IOU_OK;
269}
270
271int io_send(struct io_kiocb *req, unsigned int issue_flags)
272{
273 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
274 struct msghdr msg;
275 struct iovec iov;
276 struct socket *sock;
277 unsigned flags;
278 int min_ret = 0;
279 int ret;
280
281 if (!(req->flags & REQ_F_POLLED) &&
282 (sr->flags & IORING_RECVSEND_POLL_FIRST))
283 return -EAGAIN;
284
285 sock = sock_from_file(req->file);
286 if (unlikely(!sock))
287 return -ENOTSOCK;
288
289 ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
290 if (unlikely(ret))
291 return ret;
292
293 msg.msg_name = NULL;
294 msg.msg_control = NULL;
295 msg.msg_controllen = 0;
296 msg.msg_namelen = 0;
297
298 flags = sr->msg_flags;
299 if (issue_flags & IO_URING_F_NONBLOCK)
300 flags |= MSG_DONTWAIT;
301 if (flags & MSG_WAITALL)
302 min_ret = iov_iter_count(&msg.msg_iter);
303
304 msg.msg_flags = flags;
305 ret = sock_sendmsg(sock, &msg);
306 if (ret < min_ret) {
307 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
308 return -EAGAIN;
309 if (ret == -ERESTARTSYS)
310 ret = -EINTR;
311 if (ret > 0 && io_net_retry(sock, flags)) {
312 sr->len -= ret;
313 sr->buf += ret;
314 sr->done_io += ret;
315 req->flags |= REQ_F_PARTIAL_IO;
316 return -EAGAIN;
317 }
318 req_set_fail(req);
319 }
320 if (ret >= 0)
321 ret += sr->done_io;
322 else if (sr->done_io)
323 ret = sr->done_io;
324 io_req_set_res(req, ret, 0);
325 return IOU_OK;
326}
327
328static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
329 struct io_async_msghdr *iomsg)
330{
331 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
332 struct iovec __user *uiov;
333 size_t iov_len;
334 int ret;
335
336 ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg,
337 &iomsg->uaddr, &uiov, &iov_len);
338 if (ret)
339 return ret;
340
341 if (req->flags & REQ_F_BUFFER_SELECT) {
5702196e
DY		 342 if (iov_len == 0) {
343 sr->len = iomsg->fast_iov[0].iov_len = 0;
344 iomsg->fast_iov[0].iov_base = NULL;
345 iomsg->free_iov = NULL;
346 } else if (iov_len > 1) {
f9ead18c 347 return -EINVAL;
5702196e
DY		 348 } else {
349 if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov)))
350 return -EFAULT;
351 sr->len = iomsg->fast_iov[0].iov_len;
352 iomsg->free_iov = NULL;
353 }
f9ead18c
JA		 354 } else {
355 iomsg->free_iov = iomsg->fast_iov;
356 ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
357 &iomsg->free_iov, &iomsg->msg.msg_iter,
358 false);
359 if (ret > 0)
360 ret = 0;
361 }
362
363 return ret;
364}
365
366#ifdef CONFIG_COMPAT
367static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
368 struct io_async_msghdr *iomsg)
369{
370 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
371 struct compat_iovec __user *uiov;
372 compat_uptr_t ptr;
373 compat_size_t len;
374 int ret;
375
376 ret = __get_compat_msghdr(&iomsg->msg, sr->umsg_compat, &iomsg->uaddr,
377 &ptr, &len);
378 if (ret)
379 return ret;
380
381 uiov = compat_ptr(ptr);
382 if (req->flags & REQ_F_BUFFER_SELECT) {
383 compat_ssize_t clen;
384
6d2f75a0
DY		 385 if (len == 0) {
386 sr->len = 0;
387 iomsg->free_iov = NULL;
388 } else if (len > 1) {
f9ead18c 389 return -EINVAL;
6d2f75a0
DY		 390 } else {
391 if (!access_ok(uiov, sizeof(*uiov)))
392 return -EFAULT;
393 if (__get_user(clen, &uiov->iov_len))
394 return -EFAULT;
395 if (clen < 0)
396 return -EINVAL;
397 sr->len = clen;
398 iomsg->free_iov = NULL;
399 }
f9ead18c
JA		 400 } else {
401 iomsg->free_iov = iomsg->fast_iov;
402 ret = __import_iovec(READ, (struct iovec __user *)uiov, len,
403 UIO_FASTIOV, &iomsg->free_iov,
404 &iomsg->msg.msg_iter, true);
405 if (ret < 0)
406 return ret;
407 }
408
409 return 0;
410}
411#endif
412
413static int io_recvmsg_copy_hdr(struct io_kiocb *req,
414 struct io_async_msghdr *iomsg)
415{
416 iomsg->msg.msg_name = &iomsg->addr;
417
418#ifdef CONFIG_COMPAT
419 if (req->ctx->compat)
420 return __io_compat_recvmsg_copy_hdr(req, iomsg);
421#endif
422
423 return __io_recvmsg_copy_hdr(req, iomsg);
424}
425
426int io_recvmsg_prep_async(struct io_kiocb *req)
427{
428 int ret;
429
430 ret = io_recvmsg_copy_hdr(req, req->async_data);
431 if (!ret)
432 req->flags |= REQ_F_NEED_CLEANUP;
433 return ret;
434}
435
b3fdea6e
DY		 436#define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT)
437
f9ead18c
JA		 438int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
439{
440 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
441
442 if (unlikely(sqe->file_index || sqe->addr2))
443 return -EINVAL;
444
445 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
446 sr->len = READ_ONCE(sqe->len);
447 sr->flags = READ_ONCE(sqe->ioprio);
b3fdea6e 448 if (sr->flags & ~(RECVMSG_FLAGS))
f9ead18c
JA		 449 return -EINVAL;
450 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
451 if (sr->msg_flags & MSG_DONTWAIT)
452 req->flags |= REQ_F_NOWAIT;
453 if (sr->msg_flags & MSG_ERRQUEUE)
454 req->flags |= REQ_F_CLEAR_POLLIN;
b3fdea6e 455 if (sr->flags & IORING_RECV_MULTISHOT) {
cf0dd952
DY		 456 if (req->opcode == IORING_OP_RECVMSG)
457 return -EINVAL;
b3fdea6e
DY		 458 if (!(req->flags & REQ_F_BUFFER_SELECT))
459 return -EINVAL;
460 if (sr->msg_flags & MSG_WAITALL)
461 return -EINVAL;
462 if (req->opcode == IORING_OP_RECV && sr->len)
463 return -EINVAL;
464 req->flags |= REQ_F_APOLL_MULTISHOT;
465 }
f9ead18c
JA		 466
467#ifdef CONFIG_COMPAT
468 if (req->ctx->compat)
469 sr->msg_flags |= MSG_CMSG_COMPAT;
470#endif
471 sr->done_io = 0;
472 return 0;
473}
474
b3fdea6e
DY		 475static inline void io_recv_prep_retry(struct io_kiocb *req)
476{
477 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
478
479 sr->done_io = 0;
480 sr->len = 0; /* get from the provided buffer */
481}
482
483/*
cf0dd952 484 * Finishes io_recv
b3fdea6e
DY		 485 *
486 * Returns true if it is actually finished, or false if it should run
487 * again (for multishot).
488 */
489static inline bool io_recv_finish(struct io_kiocb *req, int *ret, unsigned int cflags)
490{
491 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
492 io_req_set_res(req, *ret, cflags);
493 *ret = IOU_OK;
494 return true;
495 }
496
497 if (*ret > 0) {
498 if (io_post_aux_cqe(req->ctx, req->cqe.user_data, *ret,
499 cflags | IORING_CQE_F_MORE, false)) {
500 io_recv_prep_retry(req);
501 return false;
502 }
503 /*
504 * Otherwise stop multishot but use the current result.
505 * Probably will end up going into overflow, but this means
506 * we cannot trust the ordering anymore
507 */
508 }
509
510 io_req_set_res(req, *ret, cflags);
511
512 if (req->flags & REQ_F_POLLED)
513 *ret = IOU_STOP_MULTISHOT;
e2df2ccb
DY		 514 else
515 *ret = IOU_OK;
b3fdea6e
DY		 516 return true;
517}
518
f9ead18c
JA		 519int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
520{
521 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
522 struct io_async_msghdr iomsg, *kmsg;
523 struct socket *sock;
524 unsigned int cflags;
525 unsigned flags;
526 int ret, min_ret = 0;
527 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
528
529 sock = sock_from_file(req->file);
530 if (unlikely(!sock))
531 return -ENOTSOCK;
532
533 if (req_has_async_data(req)) {
534 kmsg = req->async_data;
535 } else {
536 ret = io_recvmsg_copy_hdr(req, &iomsg);
537 if (ret)
538 return ret;
539 kmsg = &iomsg;
540 }
541
542 if (!(req->flags & REQ_F_POLLED) &&
543 (sr->flags & IORING_RECVSEND_POLL_FIRST))
43e0bbbd 544 return io_setup_async_msg(req, kmsg, issue_flags);
f9ead18c
JA		 545
546 if (io_do_buffer_select(req)) {
547 void __user *buf;
548
cf0dd952 549 buf = io_buffer_select(req, &sr->len, issue_flags);
f9ead18c
JA		 550 if (!buf)
551 return -ENOBUFS;
552 kmsg->fast_iov[0].iov_base = buf;
cf0dd952 553 kmsg->fast_iov[0].iov_len = sr->len;
f9ead18c 554 iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov, 1,
cf0dd952 555 sr->len);
f9ead18c
JA		 556 }
557
558 flags = sr->msg_flags;
559 if (force_nonblock)
560 flags |= MSG_DONTWAIT;
561 if (flags & MSG_WAITALL)
562 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
563
564 kmsg->msg.msg_get_inq = 1;
565 ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, kmsg->uaddr, flags);
566 if (ret < min_ret) {
cf0dd952 567 if (ret == -EAGAIN && force_nonblock)
43e0bbbd 568 return io_setup_async_msg(req, kmsg, issue_flags);
f9ead18c
JA		 569 if (ret == -ERESTARTSYS)
570 ret = -EINTR;
571 if (ret > 0 && io_net_retry(sock, flags)) {
572 sr->done_io += ret;
573 req->flags |= REQ_F_PARTIAL_IO;
43e0bbbd 574 return io_setup_async_msg(req, kmsg, issue_flags);
f9ead18c
JA		 575 }
576 req_set_fail(req);
577 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
578 req_set_fail(req);
579 }
580
581 /* fast path, check for non-NULL to avoid function call */
582 if (kmsg->free_iov)
583 kfree(kmsg->free_iov);
43e0bbbd 584 io_netmsg_recycle(req, issue_flags);
f9ead18c 585 req->flags &= ~REQ_F_NEED_CLEANUP;
d4e097da 586 if (ret > 0)
f9ead18c
JA		 587 ret += sr->done_io;
588 else if (sr->done_io)
589 ret = sr->done_io;
d4e097da
DY		 590 else
591 io_kbuf_recycle(req, issue_flags);
592
f9ead18c
JA		 593 cflags = io_put_kbuf(req, issue_flags);
594 if (kmsg->msg.msg_inq)
595 cflags |= IORING_CQE_F_SOCK_NONEMPTY;
b3fdea6e 596
cf0dd952
DY		 597 io_req_set_res(req, ret, cflags);
598 return IOU_OK;
f9ead18c
JA		 599}
600
601int io_recv(struct io_kiocb *req, unsigned int issue_flags)
602{
603 struct io_sr_msg *sr = io_kiocb_to_cmd(req);
604 struct msghdr msg;
605 struct socket *sock;
606 struct iovec iov;
607 unsigned int cflags;
608 unsigned flags;
609 int ret, min_ret = 0;
610 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
b3fdea6e 611 size_t len = sr->len;
f9ead18c
JA		 612
613 if (!(req->flags & REQ_F_POLLED) &&
614 (sr->flags & IORING_RECVSEND_POLL_FIRST))
615 return -EAGAIN;
616
617 sock = sock_from_file(req->file);
618 if (unlikely(!sock))
619 return -ENOTSOCK;
620
b3fdea6e 621retry_multishot:
f9ead18c
JA		 622 if (io_do_buffer_select(req)) {
623 void __user *buf;
624
b3fdea6e 625 buf = io_buffer_select(req, &len, issue_flags);
f9ead18c
JA		 626 if (!buf)
627 return -ENOBUFS;
628 sr->buf = buf;
629 }
630
b3fdea6e 631 ret = import_single_range(READ, sr->buf, len, &iov, &msg.msg_iter);
f9ead18c
JA		 632 if (unlikely(ret))
633 goto out_free;
634
635 msg.msg_name = NULL;
636 msg.msg_namelen = 0;
637 msg.msg_control = NULL;
638 msg.msg_get_inq = 1;
639 msg.msg_flags = 0;
640 msg.msg_controllen = 0;
641 msg.msg_iocb = NULL;
642
643 flags = sr->msg_flags;
644 if (force_nonblock)
645 flags |= MSG_DONTWAIT;
646 if (flags & MSG_WAITALL)
647 min_ret = iov_iter_count(&msg.msg_iter);
648
649 ret = sock_recvmsg(sock, &msg, flags);
650 if (ret < min_ret) {
b3fdea6e
DY		 651 if (ret == -EAGAIN && force_nonblock) {
652 if ((req->flags & IO_APOLL_MULTI_POLLED) == IO_APOLL_MULTI_POLLED) {
653 io_kbuf_recycle(req, issue_flags);
654 return IOU_ISSUE_SKIP_COMPLETE;
655 }
656
f9ead18c 657 return -EAGAIN;
b3fdea6e 658 }
f9ead18c
JA		 659 if (ret == -ERESTARTSYS)
660 ret = -EINTR;
661 if (ret > 0 && io_net_retry(sock, flags)) {
662 sr->len -= ret;
663 sr->buf += ret;
664 sr->done_io += ret;
665 req->flags |= REQ_F_PARTIAL_IO;
666 return -EAGAIN;
667 }
668 req_set_fail(req);
669 } else if ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
670out_free:
671 req_set_fail(req);
672 }
673
d4e097da 674 if (ret > 0)
f9ead18c
JA		 675 ret += sr->done_io;
676 else if (sr->done_io)
677 ret = sr->done_io;
d4e097da
DY		 678 else
679 io_kbuf_recycle(req, issue_flags);
680
f9ead18c
JA		 681 cflags = io_put_kbuf(req, issue_flags);
682 if (msg.msg_inq)
683 cflags |= IORING_CQE_F_SOCK_NONEMPTY;
b3fdea6e
DY		 684
685 if (!io_recv_finish(req, &ret, cflags))
686 goto retry_multishot;
687
688 return ret;
f9ead18c
JA		 689}
690
691int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
692{
693 struct io_accept *accept = io_kiocb_to_cmd(req);
694 unsigned flags;
695
696 if (sqe->len || sqe->buf_index)
697 return -EINVAL;
698
699 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
700 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
701 accept->flags = READ_ONCE(sqe->accept_flags);
702 accept->nofile = rlimit(RLIMIT_NOFILE);
703 flags = READ_ONCE(sqe->ioprio);
704 if (flags & ~IORING_ACCEPT_MULTISHOT)
705 return -EINVAL;
706
707 accept->file_slot = READ_ONCE(sqe->file_index);
708 if (accept->file_slot) {
709 if (accept->flags & SOCK_CLOEXEC)
710 return -EINVAL;
711 if (flags & IORING_ACCEPT_MULTISHOT &&
712 accept->file_slot != IORING_FILE_INDEX_ALLOC)
713 return -EINVAL;
714 }
715 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
716 return -EINVAL;
717 if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK))
718 accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK;
719 if (flags & IORING_ACCEPT_MULTISHOT)
720 req->flags |= REQ_F_APOLL_MULTISHOT;
721 return 0;
722}
723
724int io_accept(struct io_kiocb *req, unsigned int issue_flags)
725{
726 struct io_ring_ctx *ctx = req->ctx;
727 struct io_accept *accept = io_kiocb_to_cmd(req);
728 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
729 unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
730 bool fixed = !!accept->file_slot;
731 struct file *file;
732 int ret, fd;
733
734retry:
735 if (!fixed) {
736 fd = __get_unused_fd_flags(accept->flags, accept->nofile);
737 if (unlikely(fd < 0))
738 return fd;
739 }
740 file = do_accept(req->file, file_flags, accept->addr, accept->addr_len,
741 accept->flags);
742 if (IS_ERR(file)) {
743 if (!fixed)
744 put_unused_fd(fd);
745 ret = PTR_ERR(file);
746 if (ret == -EAGAIN && force_nonblock) {
747 /*
748 * if it's multishot and polled, we don't need to
749 * return EAGAIN to arm the poll infra since it
750 * has already been done
751 */
752 if ((req->flags & IO_APOLL_MULTI_POLLED) ==
753 IO_APOLL_MULTI_POLLED)
754 ret = IOU_ISSUE_SKIP_COMPLETE;
755 return ret;
756 }
757 if (ret == -ERESTARTSYS)
758 ret = -EINTR;
759 req_set_fail(req);
760 } else if (!fixed) {
761 fd_install(fd, file);
762 ret = fd;
763 } else {
764 ret = io_fixed_fd_install(req, issue_flags, file,
765 accept->file_slot);
766 }
767
768 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
769 io_req_set_res(req, ret, 0);
770 return IOU_OK;
771 }
f9ead18c 772
cbd25748
DY		 773 if (ret >= 0 &&
774 io_post_aux_cqe(ctx, req->cqe.user_data, ret, IORING_CQE_F_MORE, false))
d245bca6 775 goto retry;
cbd25748
DY		 776
777 io_req_set_res(req, ret, 0);
778 if (req->flags & REQ_F_POLLED)
779 return IOU_STOP_MULTISHOT;
780 return IOU_OK;
f9ead18c
JA		 781}
782
783int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
784{
785 struct io_socket *sock = io_kiocb_to_cmd(req);
786
787 if (sqe->addr || sqe->rw_flags || sqe->buf_index)
788 return -EINVAL;
789
790 sock->domain = READ_ONCE(sqe->fd);
791 sock->type = READ_ONCE(sqe->off);
792 sock->protocol = READ_ONCE(sqe->len);
793 sock->file_slot = READ_ONCE(sqe->file_index);
794 sock->nofile = rlimit(RLIMIT_NOFILE);
795
796 sock->flags = sock->type & ~SOCK_TYPE_MASK;
797 if (sock->file_slot && (sock->flags & SOCK_CLOEXEC))
798 return -EINVAL;
799 if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
800 return -EINVAL;
801 return 0;
802}
803
804int io_socket(struct io_kiocb *req, unsigned int issue_flags)
805{
806 struct io_socket *sock = io_kiocb_to_cmd(req);
807 bool fixed = !!sock->file_slot;
808 struct file *file;
809 int ret, fd;
810
811 if (!fixed) {
812 fd = __get_unused_fd_flags(sock->flags, sock->nofile);
813 if (unlikely(fd < 0))
814 return fd;
815 }
816 file = __sys_socket_file(sock->domain, sock->type, sock->protocol);
817 if (IS_ERR(file)) {
818 if (!fixed)
819 put_unused_fd(fd);
820 ret = PTR_ERR(file);
821 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
822 return -EAGAIN;
823 if (ret == -ERESTARTSYS)
824 ret = -EINTR;
825 req_set_fail(req);
826 } else if (!fixed) {
827 fd_install(fd, file);
828 ret = fd;
829 } else {
830 ret = io_fixed_fd_install(req, issue_flags, file,
831 sock->file_slot);
832 }
833 io_req_set_res(req, ret, 0);
834 return IOU_OK;
835}
836
837int io_connect_prep_async(struct io_kiocb *req)
838{
839 struct io_async_connect *io = req->async_data;
840 struct io_connect *conn = io_kiocb_to_cmd(req);
841
842 return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address);
843}
844
845int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
846{
847 struct io_connect *conn = io_kiocb_to_cmd(req);
848
849 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
850 return -EINVAL;
851
852 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
853 conn->addr_len = READ_ONCE(sqe->addr2);
854 return 0;
855}
856
857int io_connect(struct io_kiocb *req, unsigned int issue_flags)
858{
859 struct io_connect *connect = io_kiocb_to_cmd(req);
860 struct io_async_connect __io, *io;
861 unsigned file_flags;
862 int ret;
863 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
864
865 if (req_has_async_data(req)) {
866 io = req->async_data;
867 } else {
868 ret = move_addr_to_kernel(connect->addr,
869 connect->addr_len,
870 &__io.address);
871 if (ret)
872 goto out;
873 io = &__io;
874 }
875
876 file_flags = force_nonblock ? O_NONBLOCK : 0;
877
878 ret = __sys_connect_file(req->file, &io->address,
879 connect->addr_len, file_flags);
880 if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
881 if (req_has_async_data(req))
882 return -EAGAIN;
883 if (io_alloc_async_data(req)) {
884 ret = -ENOMEM;
885 goto out;
886 }
887 memcpy(req->async_data, &__io, sizeof(__io));
888 return -EAGAIN;
889 }
890 if (ret == -ERESTARTSYS)
891 ret = -EINTR;
892out:
893 if (ret < 0)
894 req_set_fail(req);
895 io_req_set_res(req, ret, 0);
896 return IOU_OK;
897}
43e0bbbd
JA		 898
899void io_netmsg_cache_free(struct io_cache_entry *entry)
900{
901 kfree(container_of(entry, struct io_async_msghdr, cache));
902}
f9ead18c 903#endif
Linux 6.x block layer and io_uring tree(s)