projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
io_uring/rsrc: optimise io_rsrc_put allocation
[linux-block.git] / io_uring / rsrc.c
CommitLineData
73572984
JA		 1// SPDX-License-Identifier: GPL-2.0
2#include <linux/kernel.h>
3#include <linux/errno.h>
4#include <linux/fs.h>
5#include <linux/file.h>
6#include <linux/mm.h>
7#include <linux/slab.h>
8#include <linux/nospec.h>
9#include <linux/hugetlb.h>
10#include <linux/compat.h>
11#include <linux/io_uring.h>
12
13#include <uapi/linux/io_uring.h>
14
73572984
JA		 15#include "io_uring.h"
16#include "openclose.h"
17#include "rsrc.h"
18
19struct io_rsrc_update {
20 struct file *file;
21 u64 arg;
22 u32 nr_args;
23 u32 offset;
24};
25
26static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
27 struct io_mapped_ubuf **pimu,
28 struct page **last_hpage);
29
73572984
JA		 30/* only define max */
31#define IORING_MAX_FIXED_FILES (1U << 20)
32#define IORING_MAX_REG_BUFFERS (1U << 14)
33
6a9ce66f 34int __io_account_mem(struct user_struct *user, unsigned long nr_pages)
73572984
JA		 35{
36 unsigned long page_limit, cur_pages, new_pages;
37
6a9ce66f
PB		 38 if (!nr_pages)
39 return 0;
40
73572984
JA		 41 /* Don't allow more pages than we can safely lock */
42 page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
43
4ccc6db0 44 cur_pages = atomic_long_read(&user->locked_vm);
73572984 45 do {
73572984
JA		 46 new_pages = cur_pages + nr_pages;
47 if (new_pages > page_limit)
48 return -ENOMEM;
4ccc6db0
UB		 49 } while (!atomic_long_try_cmpxchg(&user->locked_vm,
50 &cur_pages, new_pages));
73572984
JA		 51 return 0;
52}
53
54static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
55{
56 if (ctx->user)
57 __io_unaccount_mem(ctx->user, nr_pages);
58
59 if (ctx->mm_account)
60 atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm);
61}
62
63static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
64{
65 int ret;
66
67 if (ctx->user) {
68 ret = __io_account_mem(ctx->user, nr_pages);
69 if (ret)
70 return ret;
71 }
72
73 if (ctx->mm_account)
74 atomic64_add(nr_pages, &ctx->mm_account->pinned_vm);
75
76 return 0;
77}
78
79static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
80 void __user *arg, unsigned index)
81{
82 struct iovec __user *src;
83
84#ifdef CONFIG_COMPAT
85 if (ctx->compat) {
86 struct compat_iovec __user *ciovs;
87 struct compat_iovec ciov;
88
89 ciovs = (struct compat_iovec __user *) arg;
90 if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
91 return -EFAULT;
92
93 dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
94 dst->iov_len = ciov.iov_len;
95 return 0;
96 }
97#endif
98 src = (struct iovec __user *) arg;
99 if (copy_from_user(dst, &src[index], sizeof(*dst)))
100 return -EFAULT;
101 return 0;
102}
103
104static int io_buffer_validate(struct iovec *iov)
105{
106 unsigned long tmp, acct_len = iov->iov_len + (PAGE_SIZE - 1);
107
108 /*
109 * Don't impose further limits on the size and buffer
110 * constraints here, we'll -EINVAL later when IO is
111 * submitted if they are wrong.
112 */
113 if (!iov->iov_base)
114 return iov->iov_len ? -EFAULT : 0;
115 if (!iov->iov_len)
116 return -EFAULT;
117
118 /* arbitrary limit, but we need something */
119 if (iov->iov_len > SZ_1G)
120 return -EFAULT;
121
122 if (check_add_overflow((unsigned long)iov->iov_base, acct_len, &tmp))
123 return -EOVERFLOW;
124
125 return 0;
126}
127
128static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot)
129{
130 struct io_mapped_ubuf *imu = *slot;
131 unsigned int i;
132
133 if (imu != ctx->dummy_ubuf) {
134 for (i = 0; i < imu->nr_bvecs; i++)
135 unpin_user_page(imu->bvec[i].bv_page);
136 if (imu->acct_pages)
137 io_unaccount_mem(ctx, imu->acct_pages);
138 kvfree(imu);
139 }
140 *slot = NULL;
141}
142
ff7c75ec
PB		 143static void io_rsrc_put_work_one(struct io_rsrc_data *rsrc_data,
144 struct io_rsrc_put *prsrc)
145{
146 struct io_ring_ctx *ctx = rsrc_data->ctx;
147
148 if (prsrc->tag) {
149 if (ctx->flags & IORING_SETUP_IOPOLL) {
150 mutex_lock(&ctx->uring_lock);
151 io_post_aux_cqe(ctx, prsrc->tag, 0, 0);
152 mutex_unlock(&ctx->uring_lock);
153 } else {
154 io_post_aux_cqe(ctx, prsrc->tag, 0, 0);
155 }
156 }
157 rsrc_data->do_put(ctx, prsrc);
158}
159
73572984
JA		 160static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
161{
162 struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
73572984
JA		 163 struct io_rsrc_put *prsrc, *tmp;
164
ff7c75ec
PB		 165 if (ref_node->inline_items)
166 io_rsrc_put_work_one(rsrc_data, &ref_node->item);
167
c824986c 168 list_for_each_entry_safe(prsrc, tmp, &ref_node->item_list, list) {
73572984 169 list_del(&prsrc->list);
ff7c75ec 170 io_rsrc_put_work_one(rsrc_data, prsrc);
73572984
JA		 171 kfree(prsrc);
172 }
173
174 io_rsrc_node_destroy(ref_node);
175 if (atomic_dec_and_test(&rsrc_data->refs))
176 complete(&rsrc_data->done);
177}
178
179void io_rsrc_put_work(struct work_struct *work)
180{
181 struct io_ring_ctx *ctx;
182 struct llist_node *node;
183
184 ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work);
185 node = llist_del_all(&ctx->rsrc_put_llist);
186
187 while (node) {
188 struct io_rsrc_node *ref_node;
189 struct llist_node *next = node->next;
190
191 ref_node = llist_entry(node, struct io_rsrc_node, llist);
192 __io_rsrc_put_work(ref_node);
193 node = next;
194 }
195}
196
d34b1b0b
PB		 197void io_rsrc_put_tw(struct callback_head *cb)
198{
199 struct io_ring_ctx *ctx = container_of(cb, struct io_ring_ctx,
200 rsrc_put_tw);
201
202 io_rsrc_put_work(&ctx->rsrc_put_work.work);
203}
204
73572984
JA		 205void io_wait_rsrc_data(struct io_rsrc_data *data)
206{
207 if (data && !atomic_dec_and_test(&data->refs))
208 wait_for_completion(&data->done);
209}
210
211void io_rsrc_node_destroy(struct io_rsrc_node *ref_node)
212{
73572984
JA		 213 kfree(ref_node);
214}
215
ef8ae64f
PB		 216void io_rsrc_node_ref_zero(struct io_rsrc_node *node)
217 __must_hold(&node->rsrc_data->ctx->uring_lock)
73572984 218{
73572984 219 struct io_ring_ctx *ctx = node->rsrc_data->ctx;
73572984
JA		 220 bool first_add = false;
221 unsigned long delay = HZ;
222
73572984
JA		 223 node->done = true;
224
225 /* if we are mid-quiesce then do not delay */
226 if (node->rsrc_data->quiesce)
227 delay = 0;
228
229 while (!list_empty(&ctx->rsrc_ref_list)) {
230 node = list_first_entry(&ctx->rsrc_ref_list,
231 struct io_rsrc_node, node);
232 /* recycle ref nodes in order */
233 if (!node->done)
234 break;
235 list_del(&node->node);
236 first_add |= llist_add(&node->llist, &ctx->rsrc_put_llist);
237 }
73572984 238
d34b1b0b
PB		 239 if (!first_add)
240 return;
241
242 if (ctx->submitter_task) {
243 if (!task_work_add(ctx->submitter_task, &ctx->rsrc_put_tw,
244 ctx->notify_method))
245 return;
246 }
247 mod_delayed_work(system_wq, &ctx->rsrc_put_work, delay);
73572984
JA		 248}
249
250static struct io_rsrc_node *io_rsrc_node_alloc(void)
251{
252 struct io_rsrc_node *ref_node;
253
254 ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL);
255 if (!ref_node)
256 return NULL;
257
ef8ae64f 258 ref_node->refs = 1;
73572984 259 INIT_LIST_HEAD(&ref_node->node);
c824986c 260 INIT_LIST_HEAD(&ref_node->item_list);
73572984 261 ref_node->done = false;
ff7c75ec 262 ref_node->inline_items = 0;
73572984
JA		 263 return ref_node;
264}
265
266void io_rsrc_node_switch(struct io_ring_ctx *ctx,
267 struct io_rsrc_data *data_to_kill)
268 __must_hold(&ctx->uring_lock)
269{
270 WARN_ON_ONCE(!ctx->rsrc_backup_node);
271 WARN_ON_ONCE(data_to_kill && !ctx->rsrc_node);
272
73572984
JA		 273 if (data_to_kill) {
274 struct io_rsrc_node *rsrc_node = ctx->rsrc_node;
275
276 rsrc_node->rsrc_data = data_to_kill;
73572984 277 list_add_tail(&rsrc_node->node, &ctx->rsrc_ref_list);
73572984
JA		 278
279 atomic_inc(&data_to_kill->refs);
b8fb5b4f 280 /* put master ref */
ef8ae64f 281 io_put_rsrc_node(rsrc_node);
73572984
JA		 282 ctx->rsrc_node = NULL;
283 }
284
285 if (!ctx->rsrc_node) {
286 ctx->rsrc_node = ctx->rsrc_backup_node;
287 ctx->rsrc_backup_node = NULL;
288 }
289}
290
291int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
292{
293 if (ctx->rsrc_backup_node)
294 return 0;
295 ctx->rsrc_backup_node = io_rsrc_node_alloc();
296 return ctx->rsrc_backup_node ? 0 : -ENOMEM;
297}
298
299__cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
300 struct io_ring_ctx *ctx)
301{
302 int ret;
303
304 /* As we may drop ->uring_lock, other task may have started quiesce */
305 if (data->quiesce)
306 return -ENXIO;
77e3202a
PB		 307 ret = io_rsrc_node_switch_start(ctx);
308 if (ret)
309 return ret;
310 io_rsrc_node_switch(ctx, data);
311
312 /* kill initial ref, already quiesced if zero */
313 if (atomic_dec_and_test(&data->refs))
314 return 0;
73572984
JA		 315
316 data->quiesce = true;
77e3202a 317 mutex_unlock(&ctx->uring_lock);
73572984 318 do {
ef67fcb4 319 ret = io_run_task_work_sig(ctx);
77e3202a
PB		 320 if (ret < 0) {
321 atomic_inc(&data->refs);
322 /* wait for all works potentially completing data->done */
323 flush_delayed_work(&ctx->rsrc_put_work);
324 reinit_completion(&data->done);
325 mutex_lock(&ctx->uring_lock);
326 break;
327 }
ef67fcb4 328
73572984
JA		 329 flush_delayed_work(&ctx->rsrc_put_work);
330 ret = wait_for_completion_interruptible(&data->done);
331 if (!ret) {
332 mutex_lock(&ctx->uring_lock);
0ced756f 333 if (atomic_read(&data->refs) <= 0)
73572984 334 break;
0ced756f
PB		 335 /*
336 * it has been revived by another thread while
337 * we were unlocked
338 */
339 mutex_unlock(&ctx->uring_lock);
73572984 340 }
77e3202a 341 } while (1);
73572984
JA		 342 data->quiesce = false;
343
344 return ret;
345}
346
347static void io_free_page_table(void **table, size_t size)
348{
349 unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE);
350
351 for (i = 0; i < nr_tables; i++)
352 kfree(table[i]);
353 kfree(table);
354}
355
356static void io_rsrc_data_free(struct io_rsrc_data *data)
357{
358 size_t size = data->nr * sizeof(data->tags[0][0]);
359
360 if (data->tags)
361 io_free_page_table((void **)data->tags, size);
362 kfree(data);
363}
364
365static __cold void **io_alloc_page_table(size_t size)
366{
367 unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE);
368 size_t init_size = size;
369 void **table;
370
371 table = kcalloc(nr_tables, sizeof(*table), GFP_KERNEL_ACCOUNT);
372 if (!table)
373 return NULL;
374
375 for (i = 0; i < nr_tables; i++) {
376 unsigned int this_size = min_t(size_t, size, PAGE_SIZE);
377
378 table[i] = kzalloc(this_size, GFP_KERNEL_ACCOUNT);
379 if (!table[i]) {
380 io_free_page_table(table, init_size);
381 return NULL;
382 }
383 size -= this_size;
384 }
385 return table;
386}
387
388__cold static int io_rsrc_data_alloc(struct io_ring_ctx *ctx,
389 rsrc_put_fn *do_put, u64 __user *utags,
390 unsigned nr, struct io_rsrc_data **pdata)
391{
392 struct io_rsrc_data *data;
6acd352d 393 int ret = 0;
73572984
JA		 394 unsigned i;
395
396 data = kzalloc(sizeof(*data), GFP_KERNEL);
397 if (!data)
398 return -ENOMEM;
399 data->tags = (u64 **)io_alloc_page_table(nr * sizeof(data->tags[0][0]));
400 if (!data->tags) {
401 kfree(data);
402 return -ENOMEM;
403 }
404
405 data->nr = nr;
406 data->ctx = ctx;
407 data->do_put = do_put;
408 if (utags) {
409 ret = -EFAULT;
410 for (i = 0; i < nr; i++) {
411 u64 *tag_slot = io_get_tag_slot(data, i);
412
413 if (copy_from_user(tag_slot, &utags[i],
414 sizeof(*tag_slot)))
415 goto fail;
416 }
417 }
418
419 atomic_set(&data->refs, 1);
420 init_completion(&data->done);
421 *pdata = data;
422 return 0;
423fail:
424 io_rsrc_data_free(data);
425 return ret;
426}
427
428static int __io_sqe_files_update(struct io_ring_ctx *ctx,
429 struct io_uring_rsrc_update2 *up,
430 unsigned nr_args)
431{
432 u64 __user *tags = u64_to_user_ptr(up->tags);
433 __s32 __user *fds = u64_to_user_ptr(up->data);
434 struct io_rsrc_data *data = ctx->file_data;
435 struct io_fixed_file *file_slot;
436 struct file *file;
437 int fd, i, err = 0;
438 unsigned int done;
439 bool needs_switch = false;
440
441 if (!ctx->file_data)
442 return -ENXIO;
443 if (up->offset + nr_args > ctx->nr_user_files)
444 return -EINVAL;
445
446 for (done = 0; done < nr_args; done++) {
447 u64 tag = 0;
448
449 if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) ||
450 copy_from_user(&fd, &fds[done], sizeof(fd))) {
451 err = -EFAULT;
452 break;
453 }
454 if ((fd == IORING_REGISTER_FILES_SKIP || fd == -1) && tag) {
455 err = -EINVAL;
456 break;
457 }
458 if (fd == IORING_REGISTER_FILES_SKIP)
459 continue;
460
461 i = array_index_nospec(up->offset + done, ctx->nr_user_files);
462 file_slot = io_fixed_file_slot(&ctx->file_table, i);
463
464 if (file_slot->file_ptr) {
465 file = (struct file *)(file_slot->file_ptr & FFS_MASK);
466 err = io_queue_rsrc_removal(data, i, ctx->rsrc_node, file);
467 if (err)
468 break;
469 file_slot->file_ptr = 0;
470 io_file_bitmap_clear(&ctx->file_table, i);
471 needs_switch = true;
472 }
473 if (fd != -1) {
474 file = fget(fd);
475 if (!file) {
476 err = -EBADF;
477 break;
478 }
479 /*
480 * Don't allow io_uring instances to be registered. If
481 * UNIX isn't enabled, then this causes a reference
482 * cycle and this instance can never get freed. If UNIX
483 * is enabled we'll handle it just fine, but there's
484 * still no point in allowing a ring fd as it doesn't
485 * support regular read/write anyway.
486 */
487 if (io_is_uring_fops(file)) {
488 fput(file);
489 err = -EBADF;
490 break;
491 }
492 err = io_scm_file_account(ctx, file);
493 if (err) {
494 fput(file);
495 break;
496 }
497 *io_get_tag_slot(data, i) = tag;
498 io_fixed_file_set(file_slot, file);
499 io_file_bitmap_set(&ctx->file_table, i);
500 }
501 }
502
503 if (needs_switch)
504 io_rsrc_node_switch(ctx, data);
505 return done ? done : err;
506}
507
508static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
509 struct io_uring_rsrc_update2 *up,
510 unsigned int nr_args)
511{
512 u64 __user *tags = u64_to_user_ptr(up->tags);
513 struct iovec iov, __user *iovs = u64_to_user_ptr(up->data);
514 struct page *last_hpage = NULL;
515 bool needs_switch = false;
516 __u32 done;
517 int i, err;
518
519 if (!ctx->buf_data)
520 return -ENXIO;
521 if (up->offset + nr_args > ctx->nr_user_bufs)
522 return -EINVAL;
523
524 for (done = 0; done < nr_args; done++) {
525 struct io_mapped_ubuf *imu;
526 int offset = up->offset + done;
527 u64 tag = 0;
528
529 err = io_copy_iov(ctx, &iov, iovs, done);
530 if (err)
531 break;
532 if (tags && copy_from_user(&tag, &tags[done], sizeof(tag))) {
533 err = -EFAULT;
534 break;
535 }
536 err = io_buffer_validate(&iov);
537 if (err)
538 break;
539 if (!iov.iov_base && tag) {
540 err = -EINVAL;
541 break;
542 }
543 err = io_sqe_buffer_register(ctx, &iov, &imu, &last_hpage);
544 if (err)
545 break;
546
547 i = array_index_nospec(offset, ctx->nr_user_bufs);
548 if (ctx->user_bufs[i] != ctx->dummy_ubuf) {
549 err = io_queue_rsrc_removal(ctx->buf_data, i,
550 ctx->rsrc_node, ctx->user_bufs[i]);
551 if (unlikely(err)) {
552 io_buffer_unmap(ctx, &imu);
553 break;
554 }
5ff4fdff 555 ctx->user_bufs[i] = ctx->dummy_ubuf;
73572984
JA		 556 needs_switch = true;
557 }
558
559 ctx->user_bufs[i] = imu;
560 *io_get_tag_slot(ctx->buf_data, offset) = tag;
561 }
562
563 if (needs_switch)
564 io_rsrc_node_switch(ctx, ctx->buf_data);
565 return done ? done : err;
566}
567
568static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
569 struct io_uring_rsrc_update2 *up,
570 unsigned nr_args)
571{
572 __u32 tmp;
573 int err;
574
575 if (check_add_overflow(up->offset, nr_args, &tmp))
576 return -EOVERFLOW;
577 err = io_rsrc_node_switch_start(ctx);
578 if (err)
579 return err;
580
581 switch (type) {
582 case IORING_RSRC_FILE:
583 return __io_sqe_files_update(ctx, up, nr_args);
584 case IORING_RSRC_BUFFER:
585 return __io_sqe_buffers_update(ctx, up, nr_args);
586 }
587 return -EINVAL;
588}
589
590int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
591 unsigned nr_args)
592{
593 struct io_uring_rsrc_update2 up;
594
595 if (!nr_args)
596 return -EINVAL;
597 memset(&up, 0, sizeof(up));
598 if (copy_from_user(&up, arg, sizeof(struct io_uring_rsrc_update)))
599 return -EFAULT;
600 if (up.resv || up.resv2)
601 return -EINVAL;
602 return __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, nr_args);
603}
604
605int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
606 unsigned size, unsigned type)
607{
608 struct io_uring_rsrc_update2 up;
609
610 if (size != sizeof(up))
611 return -EINVAL;
612 if (copy_from_user(&up, arg, sizeof(up)))
613 return -EFAULT;
614 if (!up.nr || up.resv || up.resv2)
615 return -EINVAL;
616 return __io_register_rsrc_update(ctx, type, &up, up.nr);
617}
618
619__cold int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
620 unsigned int size, unsigned int type)
621{
622 struct io_uring_rsrc_register rr;
623
624 /* keep it extendible */
625 if (size != sizeof(rr))
626 return -EINVAL;
627
628 memset(&rr, 0, sizeof(rr));
629 if (copy_from_user(&rr, arg, size))
630 return -EFAULT;
631 if (!rr.nr || rr.resv2)
632 return -EINVAL;
633 if (rr.flags & ~IORING_RSRC_REGISTER_SPARSE)
634 return -EINVAL;
635
636 switch (type) {
637 case IORING_RSRC_FILE:
638 if (rr.flags & IORING_RSRC_REGISTER_SPARSE && rr.data)
639 break;
640 return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data),
641 rr.nr, u64_to_user_ptr(rr.tags));
642 case IORING_RSRC_BUFFER:
643 if (rr.flags & IORING_RSRC_REGISTER_SPARSE && rr.data)
644 break;
645 return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data),
646 rr.nr, u64_to_user_ptr(rr.tags));
647 }
648 return -EINVAL;
649}
650
d9808ceb 651int io_files_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
73572984 652{
f2ccb5ae 653 struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update);
73572984
JA		 654
655 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
656 return -EINVAL;
657 if (sqe->rw_flags || sqe->splice_fd_in)
658 return -EINVAL;
659
660 up->offset = READ_ONCE(sqe->off);
661 up->nr_args = READ_ONCE(sqe->len);
662 if (!up->nr_args)
663 return -EINVAL;
664 up->arg = READ_ONCE(sqe->addr);
665 return 0;
666}
667
668static int io_files_update_with_index_alloc(struct io_kiocb *req,
669 unsigned int issue_flags)
670{
f2ccb5ae 671 struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update);
73572984
JA		 672 __s32 __user *fds = u64_to_user_ptr(up->arg);
673 unsigned int done;
674 struct file *file;
675 int ret, fd;
676
677 if (!req->ctx->file_data)
678 return -ENXIO;
679
680 for (done = 0; done < up->nr_args; done++) {
681 if (copy_from_user(&fd, &fds[done], sizeof(fd))) {
682 ret = -EFAULT;
683 break;
684 }
685
686 file = fget(fd);
687 if (!file) {
688 ret = -EBADF;
689 break;
690 }
691 ret = io_fixed_fd_install(req, issue_flags, file,
692 IORING_FILE_INDEX_ALLOC);
693 if (ret < 0)
694 break;
695 if (copy_to_user(&fds[done], &ret, sizeof(ret))) {
f110ed84 696 __io_close_fixed(req->ctx, issue_flags, ret);
73572984
JA		 697 ret = -EFAULT;
698 break;
699 }
700 }
701
702 if (done)
703 return done;
704 return ret;
705}
706
d9808ceb 707int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
73572984 708{
f2ccb5ae 709 struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update);
73572984
JA		 710 struct io_ring_ctx *ctx = req->ctx;
711 struct io_uring_rsrc_update2 up2;
712 int ret;
713
714 up2.offset = up->offset;
715 up2.data = up->arg;
716 up2.nr = 0;
717 up2.tags = 0;
718 up2.resv = 0;
719 up2.resv2 = 0;
720
721 if (up->offset == IORING_FILE_INDEX_ALLOC) {
722 ret = io_files_update_with_index_alloc(req, issue_flags);
723 } else {
724 io_ring_submit_lock(ctx, issue_flags);
725 ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE,
726 &up2, up->nr_args);
727 io_ring_submit_unlock(ctx, issue_flags);
728 }
729
730 if (ret < 0)
731 req_set_fail(req);
732 io_req_set_res(req, ret, 0);
733 return IOU_OK;
734}
735
736int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
737 struct io_rsrc_node *node, void *rsrc)
738{
739 u64 *tag_slot = io_get_tag_slot(data, idx);
740 struct io_rsrc_put *prsrc;
ff7c75ec 741 bool inline_item = true;
73572984 742
ff7c75ec
PB		 743 if (!node->inline_items) {
744 prsrc = &node->item;
745 node->inline_items++;
746 } else {
747 prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
748 if (!prsrc)
749 return -ENOMEM;
750 inline_item = false;
751 }
73572984
JA		 752
753 prsrc->tag = *tag_slot;
754 *tag_slot = 0;
755 prsrc->rsrc = rsrc;
ff7c75ec
PB		 756 if (!inline_item)
757 list_add(&prsrc->list, &node->item_list);
73572984
JA		 758 return 0;
759}
760
761void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
762{
73572984
JA		 763 int i;
764
765 for (i = 0; i < ctx->nr_user_files; i++) {
766 struct file *file = io_file_from_index(&ctx->file_table, i);
767
38eddb2c
PB		 768 /* skip scm accounted files, they'll be freed by ->ring_sock */
769 if (!file || io_file_need_scm(file))
73572984
JA		 770 continue;
771 io_file_bitmap_clear(&ctx->file_table, i);
772 fput(file);
773 }
73572984
JA		 774
775#if defined(CONFIG_UNIX)
776 if (ctx->ring_sock) {
777 struct sock *sock = ctx->ring_sock->sk;
778 struct sk_buff *skb;
779
780 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
781 kfree_skb(skb);
782 }
783#endif
784 io_free_file_tables(&ctx->file_table);
02a4d923 785 io_file_table_set_alloc_range(ctx, 0, 0);
73572984
JA		 786 io_rsrc_data_free(ctx->file_data);
787 ctx->file_data = NULL;
788 ctx->nr_user_files = 0;
789}
790
791int io_sqe_files_unregister(struct io_ring_ctx *ctx)
792{
793 unsigned nr = ctx->nr_user_files;
794 int ret;
795
796 if (!ctx->file_data)
797 return -ENXIO;
798
799 /*
800 * Quiesce may unlock ->uring_lock, and while it's not held
801 * prevent new requests using the table.
802 */
803 ctx->nr_user_files = 0;
804 ret = io_rsrc_ref_quiesce(ctx->file_data, ctx);
805 ctx->nr_user_files = nr;
806 if (!ret)
807 __io_sqe_files_unregister(ctx);
808 return ret;
809}
810
811/*
812 * Ensure the UNIX gc is aware of our file set, so we are certain that
813 * the io_uring can be safely unregistered on process exit, even if we have
814 * loops in the file referencing. We account only files that can hold other
815 * files because otherwise they can't form a loop and so are not interesting
816 * for GC.
817 */
818int __io_scm_file_account(struct io_ring_ctx *ctx, struct file *file)
819{
820#if defined(CONFIG_UNIX)
821 struct sock *sk = ctx->ring_sock->sk;
822 struct sk_buff_head *head = &sk->sk_receive_queue;
823 struct scm_fp_list *fpl;
824 struct sk_buff *skb;
825
826 if (likely(!io_file_need_scm(file)))
827 return 0;
828
829 /*
830 * See if we can merge this file into an existing skb SCM_RIGHTS
831 * file set. If there's no room, fall back to allocating a new skb
832 * and filling it in.
833 */
834 spin_lock_irq(&head->lock);
835 skb = skb_peek(head);
836 if (skb && UNIXCB(skb).fp->count < SCM_MAX_FD)
837 __skb_unlink(skb, head);
838 else
839 skb = NULL;
840 spin_unlock_irq(&head->lock);
841
842 if (!skb) {
843 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
844 if (!fpl)
845 return -ENOMEM;
846
847 skb = alloc_skb(0, GFP_KERNEL);
848 if (!skb) {
849 kfree(fpl);
850 return -ENOMEM;
851 }
852
853 fpl->user = get_uid(current_user());
854 fpl->max = SCM_MAX_FD;
855 fpl->count = 0;
856
857 UNIXCB(skb).fp = fpl;
858 skb->sk = sk;
0091bfc8 859 skb->scm_io_uring = 1;
73572984
JA		 860 skb->destructor = unix_destruct_scm;
861 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
862 }
863
864 fpl = UNIXCB(skb).fp;
865 fpl->fp[fpl->count++] = get_file(file);
866 unix_inflight(fpl->user, file);
867 skb_queue_head(head, skb);
868 fput(file);
869#endif
870 return 0;
871}
872
873static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
874{
875 struct file *file = prsrc->file;
876#if defined(CONFIG_UNIX)
877 struct sock *sock = ctx->ring_sock->sk;
878 struct sk_buff_head list, *head = &sock->sk_receive_queue;
879 struct sk_buff *skb;
880 int i;
881
882 if (!io_file_need_scm(file)) {
883 fput(file);
884 return;
885 }
886
887 __skb_queue_head_init(&list);
888
889 /*
890 * Find the skb that holds this file in its SCM_RIGHTS. When found,
891 * remove this entry and rearrange the file array.
892 */
893 skb = skb_dequeue(head);
894 while (skb) {
895 struct scm_fp_list *fp;
896
897 fp = UNIXCB(skb).fp;
898 for (i = 0; i < fp->count; i++) {
899 int left;
900
901 if (fp->fp[i] != file)
902 continue;
903
904 unix_notinflight(fp->user, fp->fp[i]);
905 left = fp->count - 1 - i;
906 if (left) {
907 memmove(&fp->fp[i], &fp->fp[i + 1],
908 left * sizeof(struct file *));
909 }
910 fp->count--;
911 if (!fp->count) {
912 kfree_skb(skb);
913 skb = NULL;
914 } else {
915 __skb_queue_tail(&list, skb);
916 }
917 fput(file);
918 file = NULL;
919 break;
920 }
921
922 if (!file)
923 break;
924
925 __skb_queue_tail(&list, skb);
926
927 skb = skb_dequeue(head);
928 }
929
930 if (skb_peek(&list)) {
931 spin_lock_irq(&head->lock);
932 while ((skb = __skb_dequeue(&list)) != NULL)
933 __skb_queue_tail(head, skb);
934 spin_unlock_irq(&head->lock);
935 }
936#else
937 fput(file);
938#endif
939}
940
941int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
942 unsigned nr_args, u64 __user *tags)
943{
944 __s32 __user *fds = (__s32 __user *) arg;
945 struct file *file;
946 int fd, ret;
947 unsigned i;
948
949 if (ctx->file_data)
950 return -EBUSY;
951 if (!nr_args)
952 return -EINVAL;
953 if (nr_args > IORING_MAX_FIXED_FILES)
954 return -EMFILE;
955 if (nr_args > rlimit(RLIMIT_NOFILE))
956 return -EMFILE;
957 ret = io_rsrc_node_switch_start(ctx);
958 if (ret)
959 return ret;
960 ret = io_rsrc_data_alloc(ctx, io_rsrc_file_put, tags, nr_args,
961 &ctx->file_data);
962 if (ret)
963 return ret;
964
965 if (!io_alloc_file_tables(&ctx->file_table, nr_args)) {
966 io_rsrc_data_free(ctx->file_data);
967 ctx->file_data = NULL;
968 return -ENOMEM;
969 }
970
971 for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
972 struct io_fixed_file *file_slot;
973
974 if (fds && copy_from_user(&fd, &fds[i], sizeof(fd))) {
975 ret = -EFAULT;
976 goto fail;
977 }
978 /* allow sparse sets */
979 if (!fds || fd == -1) {
980 ret = -EINVAL;
981 if (unlikely(*io_get_tag_slot(ctx->file_data, i)))
982 goto fail;
983 continue;
984 }
985
986 file = fget(fd);
987 ret = -EBADF;
988 if (unlikely(!file))
989 goto fail;
990
991 /*
992 * Don't allow io_uring instances to be registered. If UNIX
993 * isn't enabled, then this causes a reference cycle and this
994 * instance can never get freed. If UNIX is enabled we'll
995 * handle it just fine, but there's still no point in allowing
996 * a ring fd as it doesn't support regular read/write anyway.
997 */
998 if (io_is_uring_fops(file)) {
999 fput(file);
1000 goto fail;
1001 }
1002 ret = io_scm_file_account(ctx, file);
1003 if (ret) {
1004 fput(file);
1005 goto fail;
1006 }
1007 file_slot = io_fixed_file_slot(&ctx->file_table, i);
1008 io_fixed_file_set(file_slot, file);
1009 io_file_bitmap_set(&ctx->file_table, i);
1010 }
1011
6e73dffb
PB		 1012 /* default it to the whole table */
1013 io_file_table_set_alloc_range(ctx, 0, ctx->nr_user_files);
73572984
JA		 1014 io_rsrc_node_switch(ctx, NULL);
1015 return 0;
1016fail:
1017 __io_sqe_files_unregister(ctx);
1018 return ret;
1019}
1020
1021static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
1022{
1023 io_buffer_unmap(ctx, &prsrc->buf);
1024 prsrc->buf = NULL;
1025}
1026
1027void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
1028{
1029 unsigned int i;
1030
1031 for (i = 0; i < ctx->nr_user_bufs; i++)
1032 io_buffer_unmap(ctx, &ctx->user_bufs[i]);
1033 kfree(ctx->user_bufs);
1034 io_rsrc_data_free(ctx->buf_data);
1035 ctx->user_bufs = NULL;
1036 ctx->buf_data = NULL;
1037 ctx->nr_user_bufs = 0;
1038}
1039
1040int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
1041{
1042 unsigned nr = ctx->nr_user_bufs;
1043 int ret;
1044
1045 if (!ctx->buf_data)
1046 return -ENXIO;
1047
1048 /*
1049 * Quiesce may unlock ->uring_lock, and while it's not held
1050 * prevent new requests using the table.
1051 */
1052 ctx->nr_user_bufs = 0;
1053 ret = io_rsrc_ref_quiesce(ctx->buf_data, ctx);
1054 ctx->nr_user_bufs = nr;
1055 if (!ret)
1056 __io_sqe_buffers_unregister(ctx);
1057 return ret;
1058}
1059
1060/*
1061 * Not super efficient, but this is just a registration time. And we do cache
1062 * the last compound head, so generally we'll only do a full search if we don't
1063 * match that one.
1064 *
1065 * We check if the given compound head page has already been accounted, to
1066 * avoid double accounting it. This allows us to account the full size of the
1067 * page, not just the constituent pages of a huge page.
1068 */
1069static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,
1070 int nr_pages, struct page *hpage)
1071{
1072 int i, j;
1073
1074 /* check current page array */
1075 for (i = 0; i < nr_pages; i++) {
1076 if (!PageCompound(pages[i]))
1077 continue;
1078 if (compound_head(pages[i]) == hpage)
1079 return true;
1080 }
1081
1082 /* check previously registered pages */
1083 for (i = 0; i < ctx->nr_user_bufs; i++) {
1084 struct io_mapped_ubuf *imu = ctx->user_bufs[i];
1085
1086 for (j = 0; j < imu->nr_bvecs; j++) {
1087 if (!PageCompound(imu->bvec[j].bv_page))
1088 continue;
1089 if (compound_head(imu->bvec[j].bv_page) == hpage)
1090 return true;
1091 }
1092 }
1093
1094 return false;
1095}
1096
1097static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
1098 int nr_pages, struct io_mapped_ubuf *imu,
1099 struct page **last_hpage)
1100{
1101 int i, ret;
1102
1103 imu->acct_pages = 0;
1104 for (i = 0; i < nr_pages; i++) {
1105 if (!PageCompound(pages[i])) {
1106 imu->acct_pages++;
1107 } else {
1108 struct page *hpage;
1109
1110 hpage = compound_head(pages[i]);
1111 if (hpage == *last_hpage)
1112 continue;
1113 *last_hpage = hpage;
1114 if (headpage_already_acct(ctx, pages, i, hpage))
1115 continue;
1116 imu->acct_pages += page_size(hpage) >> PAGE_SHIFT;
1117 }
1118 }
1119
1120 if (!imu->acct_pages)
1121 return 0;
1122
1123 ret = io_account_mem(ctx, imu->acct_pages);
1124 if (ret)
1125 imu->acct_pages = 0;
1126 return ret;
1127}
1128
1129struct page **io_pin_pages(unsigned long ubuf, unsigned long len, int *npages)
1130{
1131 unsigned long start, end, nr_pages;
1132 struct vm_area_struct **vmas = NULL;
1133 struct page **pages = NULL;
1134 int i, pret, ret = -ENOMEM;
1135
1136 end = (ubuf + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1137 start = ubuf >> PAGE_SHIFT;
1138 nr_pages = end - start;
1139
1140 pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
1141 if (!pages)
1142 goto done;
1143
1144 vmas = kvmalloc_array(nr_pages, sizeof(struct vm_area_struct *),
1145 GFP_KERNEL);
1146 if (!vmas)
1147 goto done;
1148
1149 ret = 0;
1150 mmap_read_lock(current->mm);
1151 pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
1152 pages, vmas);
1153 if (pret == nr_pages) {
edd47826
PB		 1154 struct file *file = vmas[0]->vm_file;
1155
73572984
JA		 1156 /* don't support file backed memory */
1157 for (i = 0; i < nr_pages; i++) {
edd47826
PB		 1158 if (vmas[i]->vm_file != file) {
1159 ret = -EINVAL;
1160 break;
1161 }
1162 if (!file)
73572984 1163 continue;
edd47826 1164 if (!vma_is_shmem(vmas[i]) && !is_file_hugepages(file)) {
73572984
JA		 1165 ret = -EOPNOTSUPP;
1166 break;
1167 }
1168 }
1169 *npages = nr_pages;
1170 } else {
1171 ret = pret < 0 ? pret : -EFAULT;
1172 }
1173 mmap_read_unlock(current->mm);
1174 if (ret) {
1175 /*
1176 * if we did partial map, or found file backed vmas,
1177 * release any pages we did get
1178 */
1179 if (pret > 0)
1180 unpin_user_pages(pages, pret);
1181 goto done;
1182 }
1183 ret = 0;
1184done:
1185 kvfree(vmas);
1186 if (ret < 0) {
1187 kvfree(pages);
1188 pages = ERR_PTR(ret);
1189 }
1190 return pages;
1191}
1192
1193static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
1194 struct io_mapped_ubuf **pimu,
1195 struct page **last_hpage)
1196{
1197 struct io_mapped_ubuf *imu = NULL;
1198 struct page **pages = NULL;
1199 unsigned long off;
1200 size_t size;
1201 int ret, nr_pages, i;
977bc873 1202 struct folio *folio = NULL;
73572984 1203
5ff4fdff
PB		 1204 *pimu = ctx->dummy_ubuf;
1205 if (!iov->iov_base)
73572984 1206 return 0;
73572984 1207
73572984 1208 ret = -ENOMEM;
73572984
JA		 1209 pages = io_pin_pages((unsigned long) iov->iov_base, iov->iov_len,
1210 &nr_pages);
1211 if (IS_ERR(pages)) {
1212 ret = PTR_ERR(pages);
1213 pages = NULL;
1214 goto done;
1215 }
1216
57bebf80
PB		 1217 /* If it's a huge page, try to coalesce them into a single bvec entry */
1218 if (nr_pages > 1) {
1219 folio = page_folio(pages[0]);
1220 for (i = 1; i < nr_pages; i++) {
1221 if (page_folio(pages[i]) != folio) {
1222 folio = NULL;
1223 break;
1224 }
1225 }
1226 if (folio) {
d2acf789
PB		 1227 /*
1228 * The pages are bound to the folio, it doesn't
1229 * actually unpin them but drops all but one reference,
1230 * which is usually put down by io_buffer_unmap().
1231 * Note, needs a better helper.
1232 */
1233 unpin_user_pages(&pages[1], nr_pages - 1);
57bebf80
PB		 1234 nr_pages = 1;
1235 }
1236 }
1237
73572984
JA		 1238 imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);
1239 if (!imu)
1240 goto done;
1241
1242 ret = io_buffer_account_pin(ctx, pages, nr_pages, imu, last_hpage);
1243 if (ret) {
1244 unpin_user_pages(pages, nr_pages);
1245 goto done;
1246 }
1247
1248 off = (unsigned long) iov->iov_base & ~PAGE_MASK;
1249 size = iov->iov_len;
57bebf80
PB		 1250 /* store original address for later verification */
1251 imu->ubuf = (unsigned long) iov->iov_base;
1252 imu->ubuf_end = imu->ubuf + iov->iov_len;
1253 imu->nr_bvecs = nr_pages;
1254 *pimu = imu;
1255 ret = 0;
1256
1257 if (folio) {
1258 bvec_set_page(&imu->bvec[0], pages[0], size, off);
1259 goto done;
1260 }
73572984
JA		 1261 for (i = 0; i < nr_pages; i++) {
1262 size_t vec_len;
1263
1264 vec_len = min_t(size_t, size, PAGE_SIZE - off);
cc342a21 1265 bvec_set_page(&imu->bvec[i], pages[i], vec_len, off);
73572984
JA		 1266 off = 0;
1267 size -= vec_len;
1268 }
73572984
JA		 1269done:
1270 if (ret)
1271 kvfree(imu);
1272 kvfree(pages);
1273 return ret;
1274}
1275
1276static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)
1277{
1278 ctx->user_bufs = kcalloc(nr_args, sizeof(*ctx->user_bufs), GFP_KERNEL);
1279 return ctx->user_bufs ? 0 : -ENOMEM;
1280}
1281
1282int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
1283 unsigned int nr_args, u64 __user *tags)
1284{
1285 struct page *last_hpage = NULL;
1286 struct io_rsrc_data *data;
1287 int i, ret;
1288 struct iovec iov;
1289
1290 BUILD_BUG_ON(IORING_MAX_REG_BUFFERS >= (1u << 16));
1291
1292 if (ctx->user_bufs)
1293 return -EBUSY;
1294 if (!nr_args || nr_args > IORING_MAX_REG_BUFFERS)
1295 return -EINVAL;
1296 ret = io_rsrc_node_switch_start(ctx);
1297 if (ret)
1298 return ret;
1299 ret = io_rsrc_data_alloc(ctx, io_rsrc_buf_put, tags, nr_args, &data);
1300 if (ret)
1301 return ret;
1302 ret = io_buffers_map_alloc(ctx, nr_args);
1303 if (ret) {
1304 io_rsrc_data_free(data);
1305 return ret;
1306 }
1307
1308 for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) {
1309 if (arg) {
1310 ret = io_copy_iov(ctx, &iov, arg, i);
1311 if (ret)
1312 break;
1313 ret = io_buffer_validate(&iov);
1314 if (ret)
1315 break;
1316 } else {
1317 memset(&iov, 0, sizeof(iov));
1318 }
1319
1320 if (!iov.iov_base && *io_get_tag_slot(data, i)) {
1321 ret = -EINVAL;
1322 break;
1323 }
1324
1325 ret = io_sqe_buffer_register(ctx, &iov, &ctx->user_bufs[i],
1326 &last_hpage);
1327 if (ret)
1328 break;
1329 }
1330
1331 WARN_ON_ONCE(ctx->buf_data);
1332
1333 ctx->buf_data = data;
1334 if (ret)
1335 __io_sqe_buffers_unregister(ctx);
1336 else
1337 io_rsrc_node_switch(ctx, NULL);
1338 return ret;
1339}
c059f785
PB		 1340
1341int io_import_fixed(int ddir, struct iov_iter *iter,
1342 struct io_mapped_ubuf *imu,
1343 u64 buf_addr, size_t len)
1344{
1345 u64 buf_end;
1346 size_t offset;
1347
1348 if (WARN_ON_ONCE(!imu))
1349 return -EFAULT;
1350 if (unlikely(check_add_overflow(buf_addr, (u64)len, &buf_end)))
1351 return -EFAULT;
1352 /* not inside the mapped region */
1353 if (unlikely(buf_addr < imu->ubuf || buf_end > imu->ubuf_end))
1354 return -EFAULT;
1355
1356 /*
6bf65a1b 1357 * Might not be a start of buffer, set size appropriately
c059f785
PB		 1358 * and advance us to the beginning.
1359 */
1360 offset = buf_addr - imu->ubuf;
1361 iov_iter_bvec(iter, ddir, imu->bvec, imu->nr_bvecs, offset + len);
1362
1363 if (offset) {
1364 /*
1365 * Don't use iov_iter_advance() here, as it's really slow for
1366 * using the latter parts of a big fixed buffer - it iterates
1367 * over each segment manually. We can cheat a bit here, because
1368 * we know that:
1369 *
1370 * 1) it's a BVEC iter, we set it up
1371 * 2) all bvecs are PAGE_SIZE in size, except potentially the
1372 * first and last bvec
1373 *
1374 * So just find our index, and adjust the iterator afterwards.
1375 * If the offset is within the first bvec (or the whole first
1376 * bvec, just use iov_iter_advance(). This makes it easier
1377 * since we can just skip the first segment, which may not
1378 * be PAGE_SIZE aligned.
1379 */
1380 const struct bio_vec *bvec = imu->bvec;
1381
1382 if (offset <= bvec->bv_len) {
57bebf80
PB		 1383 /*
1384 * Note, huge pages buffers consists of one large
1385 * bvec entry and should always go this way. The other
1386 * branch doesn't expect non PAGE_SIZE'd chunks.
1387 */
b000ae0e
PB		 1388 iter->bvec = bvec;
1389 iter->nr_segs = bvec->bv_len;
1390 iter->count -= offset;
1391 iter->iov_offset = offset;
c059f785
PB		 1392 } else {
1393 unsigned long seg_skip;
1394
1395 /* skip first vec */
1396 offset -= bvec->bv_len;
1397 seg_skip = 1 + (offset >> PAGE_SHIFT);
1398
1399 iter->bvec = bvec + seg_skip;
1400 iter->nr_segs -= seg_skip;
1401 iter->count -= bvec->bv_len + offset;
1402 iter->iov_offset = offset & ~PAGE_MASK;
1403 }
1404 }
1405
1406 return 0;
1407}
Linux 6.x block layer and io_uring tree(s)