Commit | Line | Data |
---|---|---|
21b4aa5d JA |
1 | #include <sys/types.h> |
2 | #include <sys/stat.h> | |
3 | #include <sys/mman.h> | |
4 | #include <unistd.h> | |
5 | #include <errno.h> | |
6 | #include <string.h> | |
7 | ||
8 | #include "liburing.h" | |
9 | #include "barrier.h" | |
10 | ||
004d564f JA |
11 | static int __io_uring_get_cqe(struct io_uring *ring, |
12 | struct io_uring_cqe **cqe_ptr, int wait) | |
21b4aa5d JA |
13 | { |
14 | struct io_uring_cq *cq = &ring->cq; | |
15 | const unsigned mask = *cq->kring_mask; | |
16 | unsigned head; | |
17 | int ret; | |
18 | ||
19 | *cqe_ptr = NULL; | |
20 | head = *cq->khead; | |
21 | do { | |
22 | /* | |
23 | * It's necessary to use a read_barrier() before reading | |
24 | * the CQ tail, since the kernel updates it locklessly. The | |
25 | * kernel has the matching store barrier for the update. The | |
26 | * kernel also ensures that previous stores to CQEs are ordered | |
27 | * with the tail update. | |
28 | */ | |
29 | read_barrier(); | |
30 | if (head != *cq->ktail) { | |
31 | *cqe_ptr = &cq->cqes[head & mask]; | |
32 | break; | |
33 | } | |
34 | if (!wait) | |
35 | break; | |
36 | ret = io_uring_enter(ring->ring_fd, 0, 1, | |
37 | IORING_ENTER_GETEVENTS, NULL); | |
38 | if (ret < 0) | |
39 | return -errno; | |
40 | } while (1); | |
41 | ||
21b4aa5d JA |
42 | return 0; |
43 | } | |
44 | ||
45 | /* | |
004d564f JA |
46 | * Return an IO completion, if one is readily available. Returns 0 with |
47 | * cqe_ptr filled in on success, -errno on failure. | |
21b4aa5d | 48 | */ |
004d564f | 49 | int io_uring_peek_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr) |
21b4aa5d | 50 | { |
004d564f | 51 | return __io_uring_get_cqe(ring, cqe_ptr, 0); |
21b4aa5d JA |
52 | } |
53 | ||
54 | /* | |
004d564f JA |
55 | * Return an IO completion, waiting for it if necessary. Returns 0 with |
56 | * cqe_ptr filled in on success, -errno on failure. | |
21b4aa5d | 57 | */ |
004d564f | 58 | int io_uring_wait_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr) |
21b4aa5d | 59 | { |
004d564f | 60 | return __io_uring_get_cqe(ring, cqe_ptr, 1); |
21b4aa5d JA |
61 | } |
62 | ||
63 | /* | |
64 | * Submit sqes acquired from io_uring_get_sqe() to the kernel. | |
65 | * | |
66 | * Returns number of sqes submitted | |
67 | */ | |
68 | int io_uring_submit(struct io_uring *ring) | |
69 | { | |
70 | struct io_uring_sq *sq = &ring->sq; | |
71 | const unsigned mask = *sq->kring_mask; | |
004d564f | 72 | unsigned ktail, ktail_next, submitted, to_submit; |
21b4aa5d JA |
73 | int ret; |
74 | ||
75 | /* | |
76 | * If we have pending IO in the kring, submit it first. We need a | |
77 | * read barrier here to match the kernels store barrier when updating | |
78 | * the SQ head. | |
79 | */ | |
80 | read_barrier(); | |
81 | if (*sq->khead != *sq->ktail) { | |
82 | submitted = *sq->kring_entries; | |
83 | goto submit; | |
84 | } | |
85 | ||
86 | if (sq->sqe_head == sq->sqe_tail) | |
87 | return 0; | |
88 | ||
89 | /* | |
90 | * Fill in sqes that we have queued up, adding them to the kernel ring | |
91 | */ | |
92 | submitted = 0; | |
93 | ktail = ktail_next = *sq->ktail; | |
004d564f JA |
94 | to_submit = sq->sqe_tail - sq->sqe_head; |
95 | while (to_submit--) { | |
21b4aa5d JA |
96 | ktail_next++; |
97 | read_barrier(); | |
98 | ||
99 | sq->array[ktail & mask] = sq->sqe_head & mask; | |
100 | ktail = ktail_next; | |
101 | ||
102 | sq->sqe_head++; | |
103 | submitted++; | |
104 | } | |
105 | ||
106 | if (!submitted) | |
107 | return 0; | |
108 | ||
109 | if (*sq->ktail != ktail) { | |
110 | /* | |
111 | * First write barrier ensures that the SQE stores are updated | |
112 | * with the tail update. This is needed so that the kernel | |
113 | * will never see a tail update without the preceeding sQE | |
114 | * stores being done. | |
115 | */ | |
116 | write_barrier(); | |
117 | *sq->ktail = ktail; | |
118 | /* | |
119 | * The kernel has the matching read barrier for reading the | |
120 | * SQ tail. | |
121 | */ | |
122 | write_barrier(); | |
123 | } | |
124 | ||
125 | submit: | |
126 | ret = io_uring_enter(ring->ring_fd, submitted, 0, | |
127 | IORING_ENTER_GETEVENTS, NULL); | |
128 | if (ret < 0) | |
129 | return -errno; | |
130 | ||
004d564f | 131 | return ret; |
21b4aa5d JA |
132 | } |
133 | ||
134 | /* | |
135 | * Return an sqe to fill. Application must later call io_uring_submit() | |
136 | * when it's ready to tell the kernel about it. The caller may call this | |
137 | * function multiple times before calling io_uring_submit(). | |
138 | * | |
139 | * Returns a vacant sqe, or NULL if we're full. | |
140 | */ | |
141 | struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring) | |
142 | { | |
143 | struct io_uring_sq *sq = &ring->sq; | |
144 | unsigned next = sq->sqe_tail + 1; | |
145 | struct io_uring_sqe *sqe; | |
146 | ||
147 | /* | |
148 | * All sqes are used | |
149 | */ | |
150 | if (next - sq->sqe_head > *sq->kring_entries) | |
151 | return NULL; | |
152 | ||
153 | sqe = &sq->sqes[sq->sqe_tail & *sq->kring_mask]; | |
154 | sq->sqe_tail = next; | |
155 | return sqe; | |
156 | } |