4 #include <linux/errno.h>
5 #include <linux/lockdep.h>
6 #include "io_uring_types.h"
10 IOU_ISSUE_SKIP_COMPLETE = -EIOCBQUEUED,
13 static inline void req_set_fail(struct io_kiocb *req)
15 req->flags |= REQ_F_FAIL;
16 if (req->flags & REQ_F_CQE_SKIP) {
17 req->flags &= ~REQ_F_CQE_SKIP;
18 req->flags |= REQ_F_SKIP_LINK_CQES;
22 static inline void io_req_set_res(struct io_kiocb *req, s32 res, u32 cflags)
25 req->cqe.flags = cflags;
28 static inline bool req_has_async_data(struct io_kiocb *req)
30 return req->flags & REQ_F_ASYNC_DATA;
33 static inline void io_put_file(struct file *file)
39 static inline void io_ring_submit_unlock(struct io_ring_ctx *ctx,
42 lockdep_assert_held(&ctx->uring_lock);
43 if (issue_flags & IO_URING_F_UNLOCKED)
44 mutex_unlock(&ctx->uring_lock);
47 static inline void io_ring_submit_lock(struct io_ring_ctx *ctx,
51 * "Normal" inline submissions always hold the uring_lock, since we
52 * grab it from the system call. Same is true for the SQPOLL offload.
53 * The only exception is when we've detached the request and issue it
54 * from an async worker thread, grab the lock for that case.
56 if (issue_flags & IO_URING_F_UNLOCKED)
57 mutex_lock(&ctx->uring_lock);
58 lockdep_assert_held(&ctx->uring_lock);
61 static inline void io_commit_cqring(struct io_ring_ctx *ctx)
63 /* order cqe stores with ring update */
64 smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);
67 static inline bool io_sqring_full(struct io_ring_ctx *ctx)
69 struct io_rings *r = ctx->rings;
71 return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == ctx->sq_entries;
74 static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
76 struct io_rings *rings = ctx->rings;
78 /* make sure SQ entry isn't read before tail */
79 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
82 static inline bool io_run_task_work(void)
84 if (test_thread_flag(TIF_NOTIFY_SIGNAL) || task_work_pending(current)) {
85 __set_current_state(TASK_RUNNING);
86 clear_notify_signal();
87 if (task_work_pending(current))
95 int io_run_task_work_sig(void);
96 void io_req_complete_failed(struct io_kiocb *req, s32 res);
97 void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);
98 void io_req_complete_post(struct io_kiocb *req);
99 void __io_req_complete_post(struct io_kiocb *req);
100 bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res,
102 void io_cqring_ev_posted(struct io_ring_ctx *ctx);
104 struct page **io_pin_pages(unsigned long ubuf, unsigned long len, int *npages);
106 struct file *io_file_get_normal(struct io_kiocb *req, int fd);
107 struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
108 unsigned issue_flags);
110 bool io_is_uring_fops(struct file *file);
111 bool io_alloc_async_data(struct io_kiocb *req);
112 void io_req_task_work_add(struct io_kiocb *req);
113 void io_req_tw_post_queue(struct io_kiocb *req, s32 res, u32 cflags);
114 void io_req_task_complete(struct io_kiocb *req, bool *locked);
115 void io_req_task_queue_fail(struct io_kiocb *req, int ret);
116 void io_req_task_submit(struct io_kiocb *req, bool *locked);
117 void tctx_task_work(struct callback_head *cb);
118 __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd);
119 int io_uring_alloc_task_context(struct task_struct *task,
120 struct io_ring_ctx *ctx);
122 int io_poll_issue(struct io_kiocb *req, bool *locked);
123 int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr);
124 int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin);
126 struct io_wq_work *io_wq_free_work(struct io_wq_work *work);
127 void io_wq_submit_work(struct io_wq_work *work);
129 void io_free_req(struct io_kiocb *req);
130 void io_queue_next(struct io_kiocb *req);
132 bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task,
135 #define io_for_each_link(pos, head) \
136 for (pos = (head); pos; pos = pos->link)