Commit | Line | Data |
---|---|---|
eb42cebb PB |
1 | #include <linux/kernel.h> |
2 | #include <linux/errno.h> | |
3 | #include <linux/file.h> | |
4 | #include <linux/slab.h> | |
5 | #include <linux/net.h> | |
6 | #include <linux/io_uring.h> | |
7 | ||
8 | #include "io_uring.h" | |
9 | #include "notif.h" | |
68ef5578 | 10 | #include "rsrc.h" |
eb42cebb | 11 | |
14b146b6 | 12 | static void __io_notif_complete_tw(struct io_kiocb *notif, bool *locked) |
eb42cebb | 13 | { |
14b146b6 | 14 | struct io_notif_data *nd = io_notif_to_data(notif); |
eb42cebb PB |
15 | struct io_ring_ctx *ctx = notif->ctx; |
16 | ||
14b146b6 PB |
17 | if (nd->account_pages && ctx->user) { |
18 | __io_unaccount_mem(ctx->user, nd->account_pages); | |
19 | nd->account_pages = 0; | |
e29e3bd4 | 20 | } |
14b146b6 | 21 | io_req_task_complete(notif, locked); |
eb42cebb PB |
22 | } |
23 | ||
eb42cebb PB |
24 | static void io_uring_tx_zerocopy_callback(struct sk_buff *skb, |
25 | struct ubuf_info *uarg, | |
26 | bool success) | |
27 | { | |
14b146b6 PB |
28 | struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg); |
29 | struct io_kiocb *notif = cmd_to_io_kiocb(nd); | |
eb4a299b | 30 | |
14b146b6 PB |
31 | if (refcount_dec_and_test(&uarg->refcnt)) { |
32 | notif->io_task_work.func = __io_notif_complete_tw; | |
33 | io_req_task_work_add(notif); | |
eb4a299b PB |
34 | } |
35 | } | |
36 | ||
b48c312b | 37 | struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) |
eb42cebb PB |
38 | __must_hold(&ctx->uring_lock) |
39 | { | |
14b146b6 PB |
40 | struct io_kiocb *notif; |
41 | struct io_notif_data *nd; | |
42 | ||
43 | if (unlikely(!io_alloc_req_refill(ctx))) | |
44 | return NULL; | |
45 | notif = io_alloc_req(ctx); | |
46 | notif->opcode = IORING_OP_NOP; | |
47 | notif->flags = 0; | |
48 | notif->file = NULL; | |
49 | notif->task = current; | |
50 | io_get_task_refs(1); | |
51 | notif->rsrc_node = NULL; | |
52 | io_req_set_rsrc_node(notif, ctx, 0); | |
14b146b6 PB |
53 | |
54 | nd = io_notif_to_data(notif); | |
55 | nd->account_pages = 0; | |
56 | nd->uarg.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; | |
57 | nd->uarg.callback = io_uring_tx_zerocopy_callback; | |
14b146b6 | 58 | refcount_set(&nd->uarg.refcnt, 1); |
eb42cebb PB |
59 | return notif; |
60 | } | |
61 | ||
b48c312b | 62 | void io_notif_flush(struct io_kiocb *notif) |
2cacedc8 | 63 | __must_hold(&slot->notif->ctx->uring_lock) |
eb42cebb | 64 | { |
14b146b6 | 65 | struct io_notif_data *nd = io_notif_to_data(notif); |
eb42cebb | 66 | |
eb42cebb | 67 | /* drop slot's master ref */ |
53bdc88a PB |
68 | if (refcount_dec_and_test(&nd->uarg.refcnt)) { |
69 | notif->io_task_work.func = __io_notif_complete_tw; | |
70 | io_req_task_work_add(notif); | |
71 | } | |
eb42cebb | 72 | } |