projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
io_uring/rsrc: optimise io_rsrc_data refcounting
[linux-block.git] / io_uring / rsrc.c
CommitLineData
73572984
JA		 1// SPDX-License-Identifier: GPL-2.0
2#include <linux/kernel.h>
3#include <linux/errno.h>
4#include <linux/fs.h>
5#include <linux/file.h>
6#include <linux/mm.h>
7#include <linux/slab.h>
8#include <linux/nospec.h>
9#include <linux/hugetlb.h>
10#include <linux/compat.h>
11#include <linux/io_uring.h>
12
13#include <uapi/linux/io_uring.h>
14
73572984
JA		 15#include "io_uring.h"
16#include "openclose.h"
17#include "rsrc.h"
18
19struct io_rsrc_update {
20 struct file *file;
21 u64 arg;
22 u32 nr_args;
23 u32 offset;
24};
25
26static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
27 struct io_mapped_ubuf **pimu,
28 struct page **last_hpage);
29
73572984
JA		 30/* only define max */
31#define IORING_MAX_FIXED_FILES (1U << 20)
32#define IORING_MAX_REG_BUFFERS (1U << 14)
33
757ef468
PB		 34static inline bool io_put_rsrc_data_ref(struct io_rsrc_data *rsrc_data)
35{
36 return !--rsrc_data->refs;
37}
38
6a9ce66f 39int __io_account_mem(struct user_struct *user, unsigned long nr_pages)
73572984
JA		 40{
41 unsigned long page_limit, cur_pages, new_pages;
42
6a9ce66f
PB		 43 if (!nr_pages)
44 return 0;
45
73572984
JA		 46 /* Don't allow more pages than we can safely lock */
47 page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
48
4ccc6db0 49 cur_pages = atomic_long_read(&user->locked_vm);
73572984 50 do {
73572984
JA		 51 new_pages = cur_pages + nr_pages;
52 if (new_pages > page_limit)
53 return -ENOMEM;
4ccc6db0
UB		 54 } while (!atomic_long_try_cmpxchg(&user->locked_vm,
55 &cur_pages, new_pages));
73572984
JA		 56 return 0;
57}
58
59static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
60{
61 if (ctx->user)
62 __io_unaccount_mem(ctx->user, nr_pages);
63
64 if (ctx->mm_account)
65 atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm);
66}
67
68static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
69{
70 int ret;
71
72 if (ctx->user) {
73 ret = __io_account_mem(ctx->user, nr_pages);
74 if (ret)
75 return ret;
76 }
77
78 if (ctx->mm_account)
79 atomic64_add(nr_pages, &ctx->mm_account->pinned_vm);
80
81 return 0;
82}
83
84static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
85 void __user *arg, unsigned index)
86{
87 struct iovec __user *src;
88
89#ifdef CONFIG_COMPAT
90 if (ctx->compat) {
91 struct compat_iovec __user *ciovs;
92 struct compat_iovec ciov;
93
94 ciovs = (struct compat_iovec __user *) arg;
95 if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
96 return -EFAULT;
97
98 dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
99 dst->iov_len = ciov.iov_len;
100 return 0;
101 }
102#endif
103 src = (struct iovec __user *) arg;
104 if (copy_from_user(dst, &src[index], sizeof(*dst)))
105 return -EFAULT;
106 return 0;
107}
108
109static int io_buffer_validate(struct iovec *iov)
110{
111 unsigned long tmp, acct_len = iov->iov_len + (PAGE_SIZE - 1);
112
113 /*
114 * Don't impose further limits on the size and buffer
115 * constraints here, we'll -EINVAL later when IO is
116 * submitted if they are wrong.
117 */
118 if (!iov->iov_base)
119 return iov->iov_len ? -EFAULT : 0;
120 if (!iov->iov_len)
121 return -EFAULT;
122
123 /* arbitrary limit, but we need something */
124 if (iov->iov_len > SZ_1G)
125 return -EFAULT;
126
127 if (check_add_overflow((unsigned long)iov->iov_base, acct_len, &tmp))
128 return -EOVERFLOW;
129
130 return 0;
131}
132
133static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot)
134{
135 struct io_mapped_ubuf *imu = *slot;
136 unsigned int i;
137
138 if (imu != ctx->dummy_ubuf) {
139 for (i = 0; i < imu->nr_bvecs; i++)
140 unpin_user_page(imu->bvec[i].bv_page);
141 if (imu->acct_pages)
142 io_unaccount_mem(ctx, imu->acct_pages);
143 kvfree(imu);
144 }
145 *slot = NULL;
146}
147
ff7c75ec
PB		 148static void io_rsrc_put_work_one(struct io_rsrc_data *rsrc_data,
149 struct io_rsrc_put *prsrc)
150{
151 struct io_ring_ctx *ctx = rsrc_data->ctx;
152
36b9818a
PB		 153 if (prsrc->tag)
154 io_post_aux_cqe(ctx, prsrc->tag, 0, 0);
ff7c75ec
PB		 155 rsrc_data->do_put(ctx, prsrc);
156}
157
73572984
JA		 158static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
159{
160 struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
73572984
JA		 161 struct io_rsrc_put *prsrc, *tmp;
162
ff7c75ec
PB		 163 if (ref_node->inline_items)
164 io_rsrc_put_work_one(rsrc_data, &ref_node->item);
165
c824986c 166 list_for_each_entry_safe(prsrc, tmp, &ref_node->item_list, list) {
73572984 167 list_del(&prsrc->list);
ff7c75ec 168 io_rsrc_put_work_one(rsrc_data, prsrc);
73572984
JA		 169 kfree(prsrc);
170 }
171
9eae8655 172 io_rsrc_node_destroy(rsrc_data->ctx, ref_node);
757ef468 173 if (io_put_rsrc_data_ref(rsrc_data))
73572984
JA		 174 complete(&rsrc_data->done);
175}
176
73572984
JA		 177void io_wait_rsrc_data(struct io_rsrc_data *data)
178{
757ef468 179 if (data && !io_put_rsrc_data_ref(data))
73572984
JA		 180 wait_for_completion(&data->done);
181}
182
9eae8655 183void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *node)
73572984 184{
9eae8655
PB		 185 if (!io_alloc_cache_put(&ctx->rsrc_node_cache, &node->cache))
186 kfree(node);
73572984
JA		 187}
188
ef8ae64f
PB		 189void io_rsrc_node_ref_zero(struct io_rsrc_node *node)
190 __must_hold(&node->rsrc_data->ctx->uring_lock)
73572984 191{
73572984 192 struct io_ring_ctx *ctx = node->rsrc_data->ctx;
73572984 193
73572984 194 node->done = true;
73572984
JA		 195 while (!list_empty(&ctx->rsrc_ref_list)) {
196 node = list_first_entry(&ctx->rsrc_ref_list,
197 struct io_rsrc_node, node);
198 /* recycle ref nodes in order */
199 if (!node->done)
200 break;
73572984 201
36b9818a
PB		 202 list_del(&node->node);
203 __io_rsrc_put_work(node);
d34b1b0b 204 }
73572984
JA		 205}
206
9eae8655 207static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
73572984
JA		 208{
209 struct io_rsrc_node *ref_node;
9eae8655
PB		 210 struct io_cache_entry *entry;
211
212 entry = io_alloc_cache_get(&ctx->rsrc_node_cache);
213 if (entry) {
214 ref_node = container_of(entry, struct io_rsrc_node, cache);
215 } else {
216 ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL);
217 if (!ref_node)
218 return NULL;
219 }
73572984 220
ef8ae64f 221 ref_node->refs = 1;
73572984 222 INIT_LIST_HEAD(&ref_node->node);
c824986c 223 INIT_LIST_HEAD(&ref_node->item_list);
73572984 224 ref_node->done = false;
ff7c75ec 225 ref_node->inline_items = 0;
73572984
JA		 226 return ref_node;
227}
228
229void io_rsrc_node_switch(struct io_ring_ctx *ctx,
230 struct io_rsrc_data *data_to_kill)
231 __must_hold(&ctx->uring_lock)
232{
233 WARN_ON_ONCE(!ctx->rsrc_backup_node);
234 WARN_ON_ONCE(data_to_kill && !ctx->rsrc_node);
235
73572984
JA		 236 if (data_to_kill) {
237 struct io_rsrc_node *rsrc_node = ctx->rsrc_node;
238
239 rsrc_node->rsrc_data = data_to_kill;
73572984 240 list_add_tail(&rsrc_node->node, &ctx->rsrc_ref_list);
73572984 241
757ef468 242 data_to_kill->refs++;
b8fb5b4f 243 /* put master ref */
1f2c8f61 244 io_put_rsrc_node(ctx, rsrc_node);
73572984
JA		 245 ctx->rsrc_node = NULL;
246 }
247
248 if (!ctx->rsrc_node) {
249 ctx->rsrc_node = ctx->rsrc_backup_node;
250 ctx->rsrc_backup_node = NULL;
251 }
252}
253
254int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
255{
256 if (ctx->rsrc_backup_node)
257 return 0;
9eae8655 258 ctx->rsrc_backup_node = io_rsrc_node_alloc(ctx);
73572984
JA		 259 return ctx->rsrc_backup_node ? 0 : -ENOMEM;
260}
261
262__cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
263 struct io_ring_ctx *ctx)
264{
265 int ret;
266
267 /* As we may drop ->uring_lock, other task may have started quiesce */
268 if (data->quiesce)
269 return -ENXIO;
77e3202a
PB		 270 ret = io_rsrc_node_switch_start(ctx);
271 if (ret)
272 return ret;
273 io_rsrc_node_switch(ctx, data);
274
757ef468
PB		 275 /* kill initial ref */
276 if (io_put_rsrc_data_ref(data))
77e3202a 277 return 0;
73572984
JA		 278
279 data->quiesce = true;
77e3202a 280 mutex_unlock(&ctx->uring_lock);
73572984 281 do {
ef67fcb4 282 ret = io_run_task_work_sig(ctx);
77e3202a 283 if (ret < 0) {
77e3202a 284 mutex_lock(&ctx->uring_lock);
757ef468
PB		 285 if (!data->refs) {
286 ret = 0;
287 } else {
288 /* restore the master reference */
289 data->refs++;
290 }
77e3202a
PB		 291 break;
292 }
73572984
JA		 293 ret = wait_for_completion_interruptible(&data->done);
294 if (!ret) {
295 mutex_lock(&ctx->uring_lock);
757ef468 296 if (!data->refs)
73572984 297 break;
0ced756f
PB		 298 /*
299 * it has been revived by another thread while
300 * we were unlocked
301 */
302 mutex_unlock(&ctx->uring_lock);
73572984 303 }
77e3202a 304 } while (1);
73572984
JA		 305 data->quiesce = false;
306
307 return ret;
308}
309
310static void io_free_page_table(void **table, size_t size)
311{
312 unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE);
313
314 for (i = 0; i < nr_tables; i++)
315 kfree(table[i]);
316 kfree(table);
317}
318
319static void io_rsrc_data_free(struct io_rsrc_data *data)
320{
321 size_t size = data->nr * sizeof(data->tags[0][0]);
322
323 if (data->tags)
324 io_free_page_table((void **)data->tags, size);
325 kfree(data);
326}
327
328static __cold void **io_alloc_page_table(size_t size)
329{
330 unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE);
331 size_t init_size = size;
332 void **table;
333
334 table = kcalloc(nr_tables, sizeof(*table), GFP_KERNEL_ACCOUNT);
335 if (!table)
336 return NULL;
337
338 for (i = 0; i < nr_tables; i++) {
339 unsigned int this_size = min_t(size_t, size, PAGE_SIZE);
340
341 table[i] = kzalloc(this_size, GFP_KERNEL_ACCOUNT);
342 if (!table[i]) {
343 io_free_page_table(table, init_size);
344 return NULL;
345 }
346 size -= this_size;
347 }
348 return table;
349}
350
351__cold static int io_rsrc_data_alloc(struct io_ring_ctx *ctx,
352 rsrc_put_fn *do_put, u64 __user *utags,
353 unsigned nr, struct io_rsrc_data **pdata)
354{
355 struct io_rsrc_data *data;
6acd352d 356 int ret = 0;
73572984
JA		 357 unsigned i;
358
359 data = kzalloc(sizeof(*data), GFP_KERNEL);
360 if (!data)
361 return -ENOMEM;
362 data->tags = (u64 **)io_alloc_page_table(nr * sizeof(data->tags[0][0]));
363 if (!data->tags) {
364 kfree(data);
365 return -ENOMEM;
366 }
367
368 data->nr = nr;
369 data->ctx = ctx;
370 data->do_put = do_put;
757ef468 371 data->refs = 1;
73572984
JA		 372 if (utags) {
373 ret = -EFAULT;
374 for (i = 0; i < nr; i++) {
375 u64 *tag_slot = io_get_tag_slot(data, i);
376
377 if (copy_from_user(tag_slot, &utags[i],
378 sizeof(*tag_slot)))
379 goto fail;
380 }
381 }
73572984
JA		 382 init_completion(&data->done);
383 *pdata = data;
384 return 0;
385fail:
386 io_rsrc_data_free(data);
387 return ret;
388}
389
390static int __io_sqe_files_update(struct io_ring_ctx *ctx,
391 struct io_uring_rsrc_update2 *up,
392 unsigned nr_args)
393{
394 u64 __user *tags = u64_to_user_ptr(up->tags);
395 __s32 __user *fds = u64_to_user_ptr(up->data);
396 struct io_rsrc_data *data = ctx->file_data;
397 struct io_fixed_file *file_slot;
398 struct file *file;
399 int fd, i, err = 0;
400 unsigned int done;
401 bool needs_switch = false;
402
403 if (!ctx->file_data)
404 return -ENXIO;
405 if (up->offset + nr_args > ctx->nr_user_files)
406 return -EINVAL;
407
408 for (done = 0; done < nr_args; done++) {
409 u64 tag = 0;
410
411 if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) ||
412 copy_from_user(&fd, &fds[done], sizeof(fd))) {
413 err = -EFAULT;
414 break;
415 }
416 if ((fd == IORING_REGISTER_FILES_SKIP || fd == -1) && tag) {
417 err = -EINVAL;
418 break;
419 }
420 if (fd == IORING_REGISTER_FILES_SKIP)
421 continue;
422
423 i = array_index_nospec(up->offset + done, ctx->nr_user_files);
424 file_slot = io_fixed_file_slot(&ctx->file_table, i);
425
426 if (file_slot->file_ptr) {
427 file = (struct file *)(file_slot->file_ptr & FFS_MASK);
428 err = io_queue_rsrc_removal(data, i, ctx->rsrc_node, file);
429 if (err)
430 break;
431 file_slot->file_ptr = 0;
432 io_file_bitmap_clear(&ctx->file_table, i);
433 needs_switch = true;
434 }
435 if (fd != -1) {
436 file = fget(fd);
437 if (!file) {
438 err = -EBADF;
439 break;
440 }
441 /*
442 * Don't allow io_uring instances to be registered. If
443 * UNIX isn't enabled, then this causes a reference
444 * cycle and this instance can never get freed. If UNIX
445 * is enabled we'll handle it just fine, but there's
446 * still no point in allowing a ring fd as it doesn't
447 * support regular read/write anyway.
448 */
449 if (io_is_uring_fops(file)) {
450 fput(file);
451 err = -EBADF;
452 break;
453 }
454 err = io_scm_file_account(ctx, file);
455 if (err) {
456 fput(file);
457 break;
458 }
459 *io_get_tag_slot(data, i) = tag;
460 io_fixed_file_set(file_slot, file);
461 io_file_bitmap_set(&ctx->file_table, i);
462 }
463 }
464
465 if (needs_switch)
466 io_rsrc_node_switch(ctx, data);
467 return done ? done : err;
468}
469
470static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
471 struct io_uring_rsrc_update2 *up,
472 unsigned int nr_args)
473{
474 u64 __user *tags = u64_to_user_ptr(up->tags);
475 struct iovec iov, __user *iovs = u64_to_user_ptr(up->data);
476 struct page *last_hpage = NULL;
477 bool needs_switch = false;
478 __u32 done;
479 int i, err;
480
481 if (!ctx->buf_data)
482 return -ENXIO;
483 if (up->offset + nr_args > ctx->nr_user_bufs)
484 return -EINVAL;
485
486 for (done = 0; done < nr_args; done++) {
487 struct io_mapped_ubuf *imu;
488 int offset = up->offset + done;
489 u64 tag = 0;
490
491 err = io_copy_iov(ctx, &iov, iovs, done);
492 if (err)
493 break;
494 if (tags && copy_from_user(&tag, &tags[done], sizeof(tag))) {
495 err = -EFAULT;
496 break;
497 }
498 err = io_buffer_validate(&iov);
499 if (err)
500 break;
501 if (!iov.iov_base && tag) {
502 err = -EINVAL;
503 break;
504 }
505 err = io_sqe_buffer_register(ctx, &iov, &imu, &last_hpage);
506 if (err)
507 break;
508
509 i = array_index_nospec(offset, ctx->nr_user_bufs);
510 if (ctx->user_bufs[i] != ctx->dummy_ubuf) {
511 err = io_queue_rsrc_removal(ctx->buf_data, i,
512 ctx->rsrc_node, ctx->user_bufs[i]);
513 if (unlikely(err)) {
514 io_buffer_unmap(ctx, &imu);
515 break;
516 }
5ff4fdff 517 ctx->user_bufs[i] = ctx->dummy_ubuf;
73572984
JA		 518 needs_switch = true;
519 }
520
521 ctx->user_bufs[i] = imu;
522 *io_get_tag_slot(ctx->buf_data, offset) = tag;
523 }
524
525 if (needs_switch)
526 io_rsrc_node_switch(ctx, ctx->buf_data);
527 return done ? done : err;
528}
529
530static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
531 struct io_uring_rsrc_update2 *up,
532 unsigned nr_args)
533{
534 __u32 tmp;
535 int err;
536
537 if (check_add_overflow(up->offset, nr_args, &tmp))
538 return -EOVERFLOW;
539 err = io_rsrc_node_switch_start(ctx);
540 if (err)
541 return err;
542
543 switch (type) {
544 case IORING_RSRC_FILE:
545 return __io_sqe_files_update(ctx, up, nr_args);
546 case IORING_RSRC_BUFFER:
547 return __io_sqe_buffers_update(ctx, up, nr_args);
548 }
549 return -EINVAL;
550}
551
552int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
553 unsigned nr_args)
554{
555 struct io_uring_rsrc_update2 up;
556
557 if (!nr_args)
558 return -EINVAL;
559 memset(&up, 0, sizeof(up));
560 if (copy_from_user(&up, arg, sizeof(struct io_uring_rsrc_update)))
561 return -EFAULT;
562 if (up.resv || up.resv2)
563 return -EINVAL;
564 return __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, nr_args);
565}
566
567int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
568 unsigned size, unsigned type)
569{
570 struct io_uring_rsrc_update2 up;
571
572 if (size != sizeof(up))
573 return -EINVAL;
574 if (copy_from_user(&up, arg, sizeof(up)))
575 return -EFAULT;
576 if (!up.nr || up.resv || up.resv2)
577 return -EINVAL;
578 return __io_register_rsrc_update(ctx, type, &up, up.nr);
579}
580
581__cold int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
582 unsigned int size, unsigned int type)
583{
584 struct io_uring_rsrc_register rr;
585
586 /* keep it extendible */
587 if (size != sizeof(rr))
588 return -EINVAL;
589
590 memset(&rr, 0, sizeof(rr));
591 if (copy_from_user(&rr, arg, size))
592 return -EFAULT;
593 if (!rr.nr || rr.resv2)
594 return -EINVAL;
595 if (rr.flags & ~IORING_RSRC_REGISTER_SPARSE)
596 return -EINVAL;
597
598 switch (type) {
599 case IORING_RSRC_FILE:
600 if (rr.flags & IORING_RSRC_REGISTER_SPARSE && rr.data)
601 break;
602 return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data),
603 rr.nr, u64_to_user_ptr(rr.tags));
604 case IORING_RSRC_BUFFER:
605 if (rr.flags & IORING_RSRC_REGISTER_SPARSE && rr.data)
606 break;
607 return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data),
608 rr.nr, u64_to_user_ptr(rr.tags));
609 }
610 return -EINVAL;
611}
612
d9808ceb 613int io_files_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
73572984 614{
f2ccb5ae 615 struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update);
73572984
JA		 616
617 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
618 return -EINVAL;
619 if (sqe->rw_flags || sqe->splice_fd_in)
620 return -EINVAL;
621
622 up->offset = READ_ONCE(sqe->off);
623 up->nr_args = READ_ONCE(sqe->len);
624 if (!up->nr_args)
625 return -EINVAL;
626 up->arg = READ_ONCE(sqe->addr);
627 return 0;
628}
629
630static int io_files_update_with_index_alloc(struct io_kiocb *req,
631 unsigned int issue_flags)
632{
f2ccb5ae 633 struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update);
73572984
JA		 634 __s32 __user *fds = u64_to_user_ptr(up->arg);
635 unsigned int done;
636 struct file *file;
637 int ret, fd;
638
639 if (!req->ctx->file_data)
640 return -ENXIO;
641
642 for (done = 0; done < up->nr_args; done++) {
643 if (copy_from_user(&fd, &fds[done], sizeof(fd))) {
644 ret = -EFAULT;
645 break;
646 }
647
648 file = fget(fd);
649 if (!file) {
650 ret = -EBADF;
651 break;
652 }
653 ret = io_fixed_fd_install(req, issue_flags, file,
654 IORING_FILE_INDEX_ALLOC);
655 if (ret < 0)
656 break;
657 if (copy_to_user(&fds[done], &ret, sizeof(ret))) {
f110ed84 658 __io_close_fixed(req->ctx, issue_flags, ret);
73572984
JA		 659 ret = -EFAULT;
660 break;
661 }
662 }
663
664 if (done)
665 return done;
666 return ret;
667}
668
d9808ceb 669int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
73572984 670{
f2ccb5ae 671 struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update);
73572984
JA		 672 struct io_ring_ctx *ctx = req->ctx;
673 struct io_uring_rsrc_update2 up2;
674 int ret;
675
676 up2.offset = up->offset;
677 up2.data = up->arg;
678 up2.nr = 0;
679 up2.tags = 0;
680 up2.resv = 0;
681 up2.resv2 = 0;
682
683 if (up->offset == IORING_FILE_INDEX_ALLOC) {
684 ret = io_files_update_with_index_alloc(req, issue_flags);
685 } else {
686 io_ring_submit_lock(ctx, issue_flags);
687 ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE,
688 &up2, up->nr_args);
689 io_ring_submit_unlock(ctx, issue_flags);
690 }
691
692 if (ret < 0)
693 req_set_fail(req);
694 io_req_set_res(req, ret, 0);
695 return IOU_OK;
696}
697
698int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
699 struct io_rsrc_node *node, void *rsrc)
700{
701 u64 *tag_slot = io_get_tag_slot(data, idx);
702 struct io_rsrc_put *prsrc;
ff7c75ec 703 bool inline_item = true;
73572984 704
ff7c75ec
PB		 705 if (!node->inline_items) {
706 prsrc = &node->item;
707 node->inline_items++;
708 } else {
709 prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
710 if (!prsrc)
711 return -ENOMEM;
712 inline_item = false;
713 }
73572984
JA		 714
715 prsrc->tag = *tag_slot;
716 *tag_slot = 0;
717 prsrc->rsrc = rsrc;
ff7c75ec
PB		 718 if (!inline_item)
719 list_add(&prsrc->list, &node->item_list);
73572984
JA		 720 return 0;
721}
722
723void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
724{
73572984
JA		 725 int i;
726
727 for (i = 0; i < ctx->nr_user_files; i++) {
728 struct file *file = io_file_from_index(&ctx->file_table, i);
729
38eddb2c
PB		 730 /* skip scm accounted files, they'll be freed by ->ring_sock */
731 if (!file || io_file_need_scm(file))
73572984
JA		 732 continue;
733 io_file_bitmap_clear(&ctx->file_table, i);
734 fput(file);
735 }
73572984
JA		 736
737#if defined(CONFIG_UNIX)
738 if (ctx->ring_sock) {
739 struct sock *sock = ctx->ring_sock->sk;
740 struct sk_buff *skb;
741
742 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
743 kfree_skb(skb);
744 }
745#endif
746 io_free_file_tables(&ctx->file_table);
02a4d923 747 io_file_table_set_alloc_range(ctx, 0, 0);
73572984
JA		 748 io_rsrc_data_free(ctx->file_data);
749 ctx->file_data = NULL;
750 ctx->nr_user_files = 0;
751}
752
753int io_sqe_files_unregister(struct io_ring_ctx *ctx)
754{
755 unsigned nr = ctx->nr_user_files;
756 int ret;
757
758 if (!ctx->file_data)
759 return -ENXIO;
760
761 /*
762 * Quiesce may unlock ->uring_lock, and while it's not held
763 * prevent new requests using the table.
764 */
765 ctx->nr_user_files = 0;
766 ret = io_rsrc_ref_quiesce(ctx->file_data, ctx);
767 ctx->nr_user_files = nr;
768 if (!ret)
769 __io_sqe_files_unregister(ctx);
770 return ret;
771}
772
773/*
774 * Ensure the UNIX gc is aware of our file set, so we are certain that
775 * the io_uring can be safely unregistered on process exit, even if we have
776 * loops in the file referencing. We account only files that can hold other
777 * files because otherwise they can't form a loop and so are not interesting
778 * for GC.
779 */
780int __io_scm_file_account(struct io_ring_ctx *ctx, struct file *file)
781{
782#if defined(CONFIG_UNIX)
783 struct sock *sk = ctx->ring_sock->sk;
784 struct sk_buff_head *head = &sk->sk_receive_queue;
785 struct scm_fp_list *fpl;
786 struct sk_buff *skb;
787
788 if (likely(!io_file_need_scm(file)))
789 return 0;
790
791 /*
792 * See if we can merge this file into an existing skb SCM_RIGHTS
793 * file set. If there's no room, fall back to allocating a new skb
794 * and filling it in.
795 */
796 spin_lock_irq(&head->lock);
797 skb = skb_peek(head);
798 if (skb && UNIXCB(skb).fp->count < SCM_MAX_FD)
799 __skb_unlink(skb, head);
800 else
801 skb = NULL;
802 spin_unlock_irq(&head->lock);
803
804 if (!skb) {
805 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
806 if (!fpl)
807 return -ENOMEM;
808
809 skb = alloc_skb(0, GFP_KERNEL);
810 if (!skb) {
811 kfree(fpl);
812 return -ENOMEM;
813 }
814
815 fpl->user = get_uid(current_user());
816 fpl->max = SCM_MAX_FD;
817 fpl->count = 0;
818
819 UNIXCB(skb).fp = fpl;
820 skb->sk = sk;
0091bfc8 821 skb->scm_io_uring = 1;
73572984
JA		 822 skb->destructor = unix_destruct_scm;
823 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
824 }
825
826 fpl = UNIXCB(skb).fp;
827 fpl->fp[fpl->count++] = get_file(file);
828 unix_inflight(fpl->user, file);
829 skb_queue_head(head, skb);
830 fput(file);
831#endif
832 return 0;
833}
834
835static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
836{
837 struct file *file = prsrc->file;
838#if defined(CONFIG_UNIX)
839 struct sock *sock = ctx->ring_sock->sk;
840 struct sk_buff_head list, *head = &sock->sk_receive_queue;
841 struct sk_buff *skb;
842 int i;
843
844 if (!io_file_need_scm(file)) {
845 fput(file);
846 return;
847 }
848
849 __skb_queue_head_init(&list);
850
851 /*
852 * Find the skb that holds this file in its SCM_RIGHTS. When found,
853 * remove this entry and rearrange the file array.
854 */
855 skb = skb_dequeue(head);
856 while (skb) {
857 struct scm_fp_list *fp;
858
859 fp = UNIXCB(skb).fp;
860 for (i = 0; i < fp->count; i++) {
861 int left;
862
863 if (fp->fp[i] != file)
864 continue;
865
866 unix_notinflight(fp->user, fp->fp[i]);
867 left = fp->count - 1 - i;
868 if (left) {
869 memmove(&fp->fp[i], &fp->fp[i + 1],
870 left * sizeof(struct file *));
871 }
872 fp->count--;
873 if (!fp->count) {
874 kfree_skb(skb);
875 skb = NULL;
876 } else {
877 __skb_queue_tail(&list, skb);
878 }
879 fput(file);
880 file = NULL;
881 break;
882 }
883
884 if (!file)
885 break;
886
887 __skb_queue_tail(&list, skb);
888
889 skb = skb_dequeue(head);
890 }
891
892 if (skb_peek(&list)) {
893 spin_lock_irq(&head->lock);
894 while ((skb = __skb_dequeue(&list)) != NULL)
895 __skb_queue_tail(head, skb);
896 spin_unlock_irq(&head->lock);
897 }
898#else
899 fput(file);
900#endif
901}
902
903int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
904 unsigned nr_args, u64 __user *tags)
905{
906 __s32 __user *fds = (__s32 __user *) arg;
907 struct file *file;
908 int fd, ret;
909 unsigned i;
910
911 if (ctx->file_data)
912 return -EBUSY;
913 if (!nr_args)
914 return -EINVAL;
915 if (nr_args > IORING_MAX_FIXED_FILES)
916 return -EMFILE;
917 if (nr_args > rlimit(RLIMIT_NOFILE))
918 return -EMFILE;
919 ret = io_rsrc_node_switch_start(ctx);
920 if (ret)
921 return ret;
922 ret = io_rsrc_data_alloc(ctx, io_rsrc_file_put, tags, nr_args,
923 &ctx->file_data);
924 if (ret)
925 return ret;
926
927 if (!io_alloc_file_tables(&ctx->file_table, nr_args)) {
928 io_rsrc_data_free(ctx->file_data);
929 ctx->file_data = NULL;
930 return -ENOMEM;
931 }
932
933 for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
934 struct io_fixed_file *file_slot;
935
936 if (fds && copy_from_user(&fd, &fds[i], sizeof(fd))) {
937 ret = -EFAULT;
938 goto fail;
939 }
940 /* allow sparse sets */
941 if (!fds || fd == -1) {
942 ret = -EINVAL;
943 if (unlikely(*io_get_tag_slot(ctx->file_data, i)))
944 goto fail;
945 continue;
946 }
947
948 file = fget(fd);
949 ret = -EBADF;
950 if (unlikely(!file))
951 goto fail;
952
953 /*
954 * Don't allow io_uring instances to be registered. If UNIX
955 * isn't enabled, then this causes a reference cycle and this
956 * instance can never get freed. If UNIX is enabled we'll
957 * handle it just fine, but there's still no point in allowing
958 * a ring fd as it doesn't support regular read/write anyway.
959 */
960 if (io_is_uring_fops(file)) {
961 fput(file);
962 goto fail;
963 }
964 ret = io_scm_file_account(ctx, file);
965 if (ret) {
966 fput(file);
967 goto fail;
968 }
969 file_slot = io_fixed_file_slot(&ctx->file_table, i);
970 io_fixed_file_set(file_slot, file);
971 io_file_bitmap_set(&ctx->file_table, i);
972 }
973
6e73dffb
PB		 974 /* default it to the whole table */
975 io_file_table_set_alloc_range(ctx, 0, ctx->nr_user_files);
73572984
JA		 976 io_rsrc_node_switch(ctx, NULL);
977 return 0;
978fail:
979 __io_sqe_files_unregister(ctx);
980 return ret;
981}
982
983static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
984{
985 io_buffer_unmap(ctx, &prsrc->buf);
986 prsrc->buf = NULL;
987}
988
989void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
990{
991 unsigned int i;
992
993 for (i = 0; i < ctx->nr_user_bufs; i++)
994 io_buffer_unmap(ctx, &ctx->user_bufs[i]);
995 kfree(ctx->user_bufs);
996 io_rsrc_data_free(ctx->buf_data);
997 ctx->user_bufs = NULL;
998 ctx->buf_data = NULL;
999 ctx->nr_user_bufs = 0;
1000}
1001
1002int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
1003{
1004 unsigned nr = ctx->nr_user_bufs;
1005 int ret;
1006
1007 if (!ctx->buf_data)
1008 return -ENXIO;
1009
1010 /*
1011 * Quiesce may unlock ->uring_lock, and while it's not held
1012 * prevent new requests using the table.
1013 */
1014 ctx->nr_user_bufs = 0;
1015 ret = io_rsrc_ref_quiesce(ctx->buf_data, ctx);
1016 ctx->nr_user_bufs = nr;
1017 if (!ret)
1018 __io_sqe_buffers_unregister(ctx);
1019 return ret;
1020}
1021
1022/*
1023 * Not super efficient, but this is just a registration time. And we do cache
1024 * the last compound head, so generally we'll only do a full search if we don't
1025 * match that one.
1026 *
1027 * We check if the given compound head page has already been accounted, to
1028 * avoid double accounting it. This allows us to account the full size of the
1029 * page, not just the constituent pages of a huge page.
1030 */
1031static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,
1032 int nr_pages, struct page *hpage)
1033{
1034 int i, j;
1035
1036 /* check current page array */
1037 for (i = 0; i < nr_pages; i++) {
1038 if (!PageCompound(pages[i]))
1039 continue;
1040 if (compound_head(pages[i]) == hpage)
1041 return true;
1042 }
1043
1044 /* check previously registered pages */
1045 for (i = 0; i < ctx->nr_user_bufs; i++) {
1046 struct io_mapped_ubuf *imu = ctx->user_bufs[i];
1047
1048 for (j = 0; j < imu->nr_bvecs; j++) {
1049 if (!PageCompound(imu->bvec[j].bv_page))
1050 continue;
1051 if (compound_head(imu->bvec[j].bv_page) == hpage)
1052 return true;
1053 }
1054 }
1055
1056 return false;
1057}
1058
1059static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
1060 int nr_pages, struct io_mapped_ubuf *imu,
1061 struct page **last_hpage)
1062{
1063 int i, ret;
1064
1065 imu->acct_pages = 0;
1066 for (i = 0; i < nr_pages; i++) {
1067 if (!PageCompound(pages[i])) {
1068 imu->acct_pages++;
1069 } else {
1070 struct page *hpage;
1071
1072 hpage = compound_head(pages[i]);
1073 if (hpage == *last_hpage)
1074 continue;
1075 *last_hpage = hpage;
1076 if (headpage_already_acct(ctx, pages, i, hpage))
1077 continue;
1078 imu->acct_pages += page_size(hpage) >> PAGE_SHIFT;
1079 }
1080 }
1081
1082 if (!imu->acct_pages)
1083 return 0;
1084
1085 ret = io_account_mem(ctx, imu->acct_pages);
1086 if (ret)
1087 imu->acct_pages = 0;
1088 return ret;
1089}
1090
1091struct page **io_pin_pages(unsigned long ubuf, unsigned long len, int *npages)
1092{
1093 unsigned long start, end, nr_pages;
1094 struct vm_area_struct **vmas = NULL;
1095 struct page **pages = NULL;
1096 int i, pret, ret = -ENOMEM;
1097
1098 end = (ubuf + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1099 start = ubuf >> PAGE_SHIFT;
1100 nr_pages = end - start;
1101
1102 pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
1103 if (!pages)
1104 goto done;
1105
1106 vmas = kvmalloc_array(nr_pages, sizeof(struct vm_area_struct *),
1107 GFP_KERNEL);
1108 if (!vmas)
1109 goto done;
1110
1111 ret = 0;
1112 mmap_read_lock(current->mm);
1113 pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
1114 pages, vmas);
1115 if (pret == nr_pages) {
edd47826
PB		 1116 struct file *file = vmas[0]->vm_file;
1117
73572984
JA		 1118 /* don't support file backed memory */
1119 for (i = 0; i < nr_pages; i++) {
edd47826
PB		 1120 if (vmas[i]->vm_file != file) {
1121 ret = -EINVAL;
1122 break;
1123 }
1124 if (!file)
73572984 1125 continue;
edd47826 1126 if (!vma_is_shmem(vmas[i]) && !is_file_hugepages(file)) {
73572984
JA		 1127 ret = -EOPNOTSUPP;
1128 break;
1129 }
1130 }
1131 *npages = nr_pages;
1132 } else {
1133 ret = pret < 0 ? pret : -EFAULT;
1134 }
1135 mmap_read_unlock(current->mm);
1136 if (ret) {
1137 /*
1138 * if we did partial map, or found file backed vmas,
1139 * release any pages we did get
1140 */
1141 if (pret > 0)
1142 unpin_user_pages(pages, pret);
1143 goto done;
1144 }
1145 ret = 0;
1146done:
1147 kvfree(vmas);
1148 if (ret < 0) {
1149 kvfree(pages);
1150 pages = ERR_PTR(ret);
1151 }
1152 return pages;
1153}
1154
1155static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
1156 struct io_mapped_ubuf **pimu,
1157 struct page **last_hpage)
1158{
1159 struct io_mapped_ubuf *imu = NULL;
1160 struct page **pages = NULL;
1161 unsigned long off;
1162 size_t size;
1163 int ret, nr_pages, i;
977bc873 1164 struct folio *folio = NULL;
73572984 1165
5ff4fdff
PB		 1166 *pimu = ctx->dummy_ubuf;
1167 if (!iov->iov_base)
73572984 1168 return 0;
73572984 1169
73572984 1170 ret = -ENOMEM;
73572984
JA		 1171 pages = io_pin_pages((unsigned long) iov->iov_base, iov->iov_len,
1172 &nr_pages);
1173 if (IS_ERR(pages)) {
1174 ret = PTR_ERR(pages);
1175 pages = NULL;
1176 goto done;
1177 }
1178
57bebf80
PB		 1179 /* If it's a huge page, try to coalesce them into a single bvec entry */
1180 if (nr_pages > 1) {
1181 folio = page_folio(pages[0]);
1182 for (i = 1; i < nr_pages; i++) {
1183 if (page_folio(pages[i]) != folio) {
1184 folio = NULL;
1185 break;
1186 }
1187 }
1188 if (folio) {
d2acf789
PB		 1189 /*
1190 * The pages are bound to the folio, it doesn't
1191 * actually unpin them but drops all but one reference,
1192 * which is usually put down by io_buffer_unmap().
1193 * Note, needs a better helper.
1194 */
1195 unpin_user_pages(&pages[1], nr_pages - 1);
57bebf80
PB		 1196 nr_pages = 1;
1197 }
1198 }
1199
73572984
JA		 1200 imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);
1201 if (!imu)
1202 goto done;
1203
1204 ret = io_buffer_account_pin(ctx, pages, nr_pages, imu, last_hpage);
1205 if (ret) {
1206 unpin_user_pages(pages, nr_pages);
1207 goto done;
1208 }
1209
1210 off = (unsigned long) iov->iov_base & ~PAGE_MASK;
1211 size = iov->iov_len;
57bebf80
PB		 1212 /* store original address for later verification */
1213 imu->ubuf = (unsigned long) iov->iov_base;
1214 imu->ubuf_end = imu->ubuf + iov->iov_len;
1215 imu->nr_bvecs = nr_pages;
1216 *pimu = imu;
1217 ret = 0;
1218
1219 if (folio) {
1220 bvec_set_page(&imu->bvec[0], pages[0], size, off);
1221 goto done;
1222 }
73572984
JA		 1223 for (i = 0; i < nr_pages; i++) {
1224 size_t vec_len;
1225
1226 vec_len = min_t(size_t, size, PAGE_SIZE - off);
cc342a21 1227 bvec_set_page(&imu->bvec[i], pages[i], vec_len, off);
73572984
JA		 1228 off = 0;
1229 size -= vec_len;
1230 }
73572984
JA		 1231done:
1232 if (ret)
1233 kvfree(imu);
1234 kvfree(pages);
1235 return ret;
1236}
1237
1238static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)
1239{
1240 ctx->user_bufs = kcalloc(nr_args, sizeof(*ctx->user_bufs), GFP_KERNEL);
1241 return ctx->user_bufs ? 0 : -ENOMEM;
1242}
1243
1244int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
1245 unsigned int nr_args, u64 __user *tags)
1246{
1247 struct page *last_hpage = NULL;
1248 struct io_rsrc_data *data;
1249 int i, ret;
1250 struct iovec iov;
1251
1252 BUILD_BUG_ON(IORING_MAX_REG_BUFFERS >= (1u << 16));
1253
1254 if (ctx->user_bufs)
1255 return -EBUSY;
1256 if (!nr_args || nr_args > IORING_MAX_REG_BUFFERS)
1257 return -EINVAL;
1258 ret = io_rsrc_node_switch_start(ctx);
1259 if (ret)
1260 return ret;
1261 ret = io_rsrc_data_alloc(ctx, io_rsrc_buf_put, tags, nr_args, &data);
1262 if (ret)
1263 return ret;
1264 ret = io_buffers_map_alloc(ctx, nr_args);
1265 if (ret) {
1266 io_rsrc_data_free(data);
1267 return ret;
1268 }
1269
1270 for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) {
1271 if (arg) {
1272 ret = io_copy_iov(ctx, &iov, arg, i);
1273 if (ret)
1274 break;
1275 ret = io_buffer_validate(&iov);
1276 if (ret)
1277 break;
1278 } else {
1279 memset(&iov, 0, sizeof(iov));
1280 }
1281
1282 if (!iov.iov_base && *io_get_tag_slot(data, i)) {
1283 ret = -EINVAL;
1284 break;
1285 }
1286
1287 ret = io_sqe_buffer_register(ctx, &iov, &ctx->user_bufs[i],
1288 &last_hpage);
1289 if (ret)
1290 break;
1291 }
1292
1293 WARN_ON_ONCE(ctx->buf_data);
1294
1295 ctx->buf_data = data;
1296 if (ret)
1297 __io_sqe_buffers_unregister(ctx);
1298 else
1299 io_rsrc_node_switch(ctx, NULL);
1300 return ret;
1301}
c059f785
PB		 1302
1303int io_import_fixed(int ddir, struct iov_iter *iter,
1304 struct io_mapped_ubuf *imu,
1305 u64 buf_addr, size_t len)
1306{
1307 u64 buf_end;
1308 size_t offset;
1309
1310 if (WARN_ON_ONCE(!imu))
1311 return -EFAULT;
1312 if (unlikely(check_add_overflow(buf_addr, (u64)len, &buf_end)))
1313 return -EFAULT;
1314 /* not inside the mapped region */
1315 if (unlikely(buf_addr < imu->ubuf || buf_end > imu->ubuf_end))
1316 return -EFAULT;
1317
1318 /*
6bf65a1b 1319 * Might not be a start of buffer, set size appropriately
c059f785
PB		 1320 * and advance us to the beginning.
1321 */
1322 offset = buf_addr - imu->ubuf;
1323 iov_iter_bvec(iter, ddir, imu->bvec, imu->nr_bvecs, offset + len);
1324
1325 if (offset) {
1326 /*
1327 * Don't use iov_iter_advance() here, as it's really slow for
1328 * using the latter parts of a big fixed buffer - it iterates
1329 * over each segment manually. We can cheat a bit here, because
1330 * we know that:
1331 *
1332 * 1) it's a BVEC iter, we set it up
1333 * 2) all bvecs are PAGE_SIZE in size, except potentially the
1334 * first and last bvec
1335 *
1336 * So just find our index, and adjust the iterator afterwards.
1337 * If the offset is within the first bvec (or the whole first
1338 * bvec, just use iov_iter_advance(). This makes it easier
1339 * since we can just skip the first segment, which may not
1340 * be PAGE_SIZE aligned.
1341 */
1342 const struct bio_vec *bvec = imu->bvec;
1343
1344 if (offset <= bvec->bv_len) {
57bebf80
PB		 1345 /*
1346 * Note, huge pages buffers consists of one large
1347 * bvec entry and should always go this way. The other
1348 * branch doesn't expect non PAGE_SIZE'd chunks.
1349 */
b000ae0e
PB		 1350 iter->bvec = bvec;
1351 iter->nr_segs = bvec->bv_len;
1352 iter->count -= offset;
1353 iter->iov_offset = offset;
c059f785
PB		 1354 } else {
1355 unsigned long seg_skip;
1356
1357 /* skip first vec */
1358 offset -= bvec->bv_len;
1359 seg_skip = 1 + (offset >> PAGE_SHIFT);
1360
1361 iter->bvec = bvec + seg_skip;
1362 iter->nr_segs -= seg_skip;
1363 iter->count -= bvec->bv_len + offset;
1364 iter->iov_offset = offset & ~PAGE_MASK;
1365 }
1366 }
1367
1368 return 0;
1369}
Linux 6.x block layer and io_uring tree(s)