Commit | Line | Data |
---|---|---|
a4f4fdd7 | 1 | /* |
da751ca9 JA |
2 | * syslet engine |
3 | * | |
4 | * IO engine that does regular pread(2)/pwrite(2) to transfer data, but | |
5 | * with syslets to make the execution async. | |
a4f4fdd7 JA |
6 | * |
7 | */ | |
8 | #include <stdio.h> | |
9 | #include <stdlib.h> | |
10 | #include <unistd.h> | |
11 | #include <errno.h> | |
12 | #include <assert.h> | |
12cbb469 | 13 | #include <malloc.h> |
b8846354 | 14 | #include <asm/unistd.h> |
a4f4fdd7 JA |
15 | |
16 | #include "../fio.h" | |
12cbb469 JA |
17 | #include "../indirect.h" |
18 | #include "../syslet.h" | |
a4f4fdd7 JA |
19 | |
20 | #ifdef FIO_HAVE_SYSLET | |
21 | ||
1760e679 JA |
22 | #ifdef __NR_pread64 |
23 | #define __NR_fio_pread __NR_pread64 | |
24 | #define __NR_fio_pwrite __NR_pwrite64 | |
25 | #else | |
26 | #define __NR_fio_pread __NR_pread | |
27 | #define __NR_fio_pwrite __NR_pwrite | |
28 | #endif | |
29 | ||
a4f4fdd7 JA |
30 | struct syslet_data { |
31 | struct io_u **events; | |
32 | unsigned int nr_events; | |
33 | ||
12cbb469 JA |
34 | struct syslet_ring *ring; |
35 | void *stack; | |
a4f4fdd7 JA |
36 | }; |
37 | ||
12cbb469 | 38 | static void fio_syslet_complete(struct thread_data *td, struct io_u *io_u) |
9ff9de69 JA |
39 | { |
40 | struct syslet_data *sd = td->io_ops->data; | |
9ff9de69 | 41 | |
12cbb469 JA |
42 | assert(sd->nr_events < td->o.iodepth); |
43 | sd->events[sd->nr_events++] = io_u; | |
44 | } | |
9ff9de69 | 45 | |
12cbb469 JA |
46 | static void syslet_complete_nr(struct thread_data *td, unsigned int nr) |
47 | { | |
48 | struct syslet_data *sd = td->io_ops->data; | |
49 | unsigned int i; | |
9ff9de69 | 50 | |
12cbb469 JA |
51 | for (i = 0; i < nr; i++) { |
52 | unsigned int idx = (i + sd->ring->user_tail) % td->o.iodepth; | |
53 | struct syslet_completion *comp = &sd->ring->comp[idx]; | |
54 | struct io_u *io_u = (struct io_u *) (long) comp->caller_data; | |
5b38ee84 | 55 | |
12cbb469 JA |
56 | io_u->resid = io_u->xfer_buflen - comp->status; |
57 | fio_syslet_complete(td, io_u); | |
58 | } | |
9ff9de69 JA |
59 | } |
60 | ||
12cbb469 JA |
61 | |
62 | static void fio_syslet_wait_for_events(struct thread_data *td) | |
a4f4fdd7 JA |
63 | { |
64 | struct syslet_data *sd = td->io_ops->data; | |
12cbb469 JA |
65 | struct syslet_ring *ring = sd->ring; |
66 | unsigned int events; | |
a4f4fdd7 | 67 | |
12cbb469 | 68 | events = 0; |
a4f4fdd7 | 69 | do { |
12cbb469 JA |
70 | unsigned int kh = ring->kernel_head; |
71 | int ret; | |
a4f4fdd7 | 72 | |
12cbb469 JA |
73 | /* |
74 | * first reap events that are already completed | |
75 | */ | |
76 | if (ring->user_tail != kh) { | |
77 | unsigned int nr = kh - ring->user_tail; | |
a4f4fdd7 | 78 | |
12cbb469 JA |
79 | syslet_complete_nr(td, nr); |
80 | events += nr; | |
81 | ring->user_tail = kh; | |
82 | continue; | |
83 | } | |
a4f4fdd7 | 84 | |
12cbb469 JA |
85 | /* |
86 | * block waiting for at least one event | |
87 | */ | |
88 | ret = syscall(__NR_syslet_ring_wait, ring, ring->user_tail); | |
89 | assert(!ret); | |
90 | } while (!events); | |
a4f4fdd7 JA |
91 | } |
92 | ||
93 | static int fio_syslet_getevents(struct thread_data *td, int min, | |
94 | int fio_unused max, | |
95 | struct timespec fio_unused *t) | |
96 | { | |
97 | struct syslet_data *sd = td->io_ops->data; | |
a4f4fdd7 JA |
98 | long ret; |
99 | ||
100 | do { | |
a4f4fdd7 JA |
101 | /* |
102 | * do we have enough immediate completions? | |
103 | */ | |
104 | if (sd->nr_events >= (unsigned int) min) | |
105 | break; | |
106 | ||
12cbb469 | 107 | fio_syslet_wait_for_events(td); |
a4f4fdd7 JA |
108 | } while (1); |
109 | ||
110 | ret = sd->nr_events; | |
111 | sd->nr_events = 0; | |
112 | return ret; | |
113 | } | |
114 | ||
115 | static struct io_u *fio_syslet_event(struct thread_data *td, int event) | |
116 | { | |
117 | struct syslet_data *sd = td->io_ops->data; | |
118 | ||
119 | return sd->events[event]; | |
120 | } | |
121 | ||
12cbb469 JA |
122 | static void fio_syslet_prep_sync(struct fio_file *f, |
123 | struct indirect_registers *regs) | |
a4f4fdd7 | 124 | { |
12cbb469 | 125 | FILL_IN(*regs, __NR_fsync, (long) f->fd); |
a4f4fdd7 JA |
126 | } |
127 | ||
12cbb469 JA |
128 | static void fio_syslet_prep_rw(struct io_u *io_u, struct fio_file *f, |
129 | struct indirect_registers *regs) | |
a4f4fdd7 | 130 | { |
12cbb469 | 131 | long nr; |
a4f4fdd7 | 132 | |
a4f4fdd7 JA |
133 | /* |
134 | * prepare rw | |
135 | */ | |
136 | if (io_u->ddir == DDIR_READ) | |
1760e679 | 137 | nr = __NR_fio_pread; |
a4f4fdd7 | 138 | else |
1760e679 | 139 | nr = __NR_fio_pwrite; |
a4f4fdd7 | 140 | |
12cbb469 JA |
141 | FILL_IN(*regs, nr, (long) f->fd, (long) io_u->xfer_buf, |
142 | (long) io_u->xfer_buflen, (long) io_u->offset); | |
a4f4fdd7 JA |
143 | } |
144 | ||
12cbb469 | 145 | static void fio_syslet_prep(struct io_u *io_u, struct indirect_registers *regs) |
a4f4fdd7 JA |
146 | { |
147 | struct fio_file *f = io_u->file; | |
148 | ||
149 | if (io_u->ddir == DDIR_SYNC) | |
12cbb469 | 150 | fio_syslet_prep_sync(f, regs); |
a4f4fdd7 | 151 | else |
12cbb469 | 152 | fio_syslet_prep_rw(io_u, f, regs); |
a0a930ef JA |
153 | } |
154 | ||
12cbb469 | 155 | static void ret_func(void) |
a4f4fdd7 | 156 | { |
12cbb469 | 157 | syscall(__NR_exit); |
9ff9de69 JA |
158 | } |
159 | ||
160 | static int fio_syslet_queue(struct thread_data *td, struct io_u *io_u) | |
161 | { | |
162 | struct syslet_data *sd = td->io_ops->data; | |
12cbb469 JA |
163 | union indirect_params params; |
164 | struct indirect_registers regs; | |
165 | int ret; | |
bf0dc8fa | 166 | |
7101d9c2 JA |
167 | fio_ro_check(td, io_u); |
168 | ||
12cbb469 JA |
169 | memset(¶ms, 0, sizeof(params)); |
170 | fill_syslet_args(¶ms.syslet, sd->ring, (long)io_u, ret_func, sd->stack); | |
a4f4fdd7 | 171 | |
12cbb469 | 172 | fio_syslet_prep(io_u, ®s); |
2ca50be4 | 173 | |
12cbb469 JA |
174 | ret = syscall(__NR_indirect, ®s, ¶ms, sizeof(params), 0); |
175 | if (ret == (int) io_u->xfer_buflen) { | |
176 | /* | |
177 | * completed sync, account. this also catches fsync(). | |
178 | */ | |
179 | return FIO_Q_COMPLETED; | |
180 | } else if (ret < 0) { | |
181 | /* | |
182 | * queued for async execution | |
183 | */ | |
184 | if (errno == ESYSLETPENDING) | |
185 | return FIO_Q_QUEUED; | |
186 | } | |
a4f4fdd7 | 187 | |
12cbb469 JA |
188 | io_u->error = errno; |
189 | td_verror(td, io_u->error, "xfer"); | |
190 | return FIO_Q_COMPLETED; | |
a4f4fdd7 JA |
191 | } |
192 | ||
76f58b92 JA |
193 | static int check_syslet_support(struct syslet_data *sd) |
194 | { | |
12cbb469 JA |
195 | union indirect_params params; |
196 | struct indirect_registers regs; | |
197 | pid_t pid, my_pid = getpid(); | |
76f58b92 | 198 | |
12cbb469 JA |
199 | memset(¶ms, 0, sizeof(params)); |
200 | fill_syslet_args(¶ms.syslet, sd->ring, 0, ret_func, sd->stack); | |
76f58b92 | 201 | |
12cbb469 JA |
202 | FILL_IN(regs, __NR_getpid); |
203 | ||
204 | pid = syscall(__NR_indirect, ®s, ¶ms, sizeof(params), 0); | |
205 | if (pid == my_pid) | |
206 | return 0; | |
207 | ||
208 | return 1; | |
76f58b92 JA |
209 | } |
210 | ||
a4f4fdd7 JA |
211 | static void fio_syslet_cleanup(struct thread_data *td) |
212 | { | |
213 | struct syslet_data *sd = td->io_ops->data; | |
214 | ||
215 | if (sd) { | |
a4f4fdd7 | 216 | free(sd->events); |
12cbb469 JA |
217 | free(sd->ring); |
218 | free(sd->stack); | |
a4f4fdd7 JA |
219 | free(sd); |
220 | td->io_ops->data = NULL; | |
221 | } | |
222 | } | |
223 | ||
224 | static int fio_syslet_init(struct thread_data *td) | |
225 | { | |
226 | struct syslet_data *sd; | |
12cbb469 | 227 | void *ring, *stack; |
a4f4fdd7 JA |
228 | |
229 | sd = malloc(sizeof(*sd)); | |
230 | memset(sd, 0, sizeof(*sd)); | |
12cbb469 | 231 | |
2dc1bbeb JA |
232 | sd->events = malloc(sizeof(struct io_u *) * td->o.iodepth); |
233 | memset(sd->events, 0, sizeof(struct io_u *) * td->o.iodepth); | |
12cbb469 | 234 | if (posix_memalign(&ring, sizeof(uint64_t), sizeof(struct syslet_ring))) |
db64e9bc | 235 | return 1; |
12cbb469 JA |
236 | if (posix_memalign(&stack, page_size, page_size)) |
237 | return 1; | |
238 | ||
239 | sd->ring = ring; | |
240 | sd->stack = stack; | |
241 | ||
242 | memset(sd->ring, 0, sizeof(*sd->ring)); | |
243 | sd->ring->elements = td->o.iodepth; | |
db64e9bc | 244 | |
76f58b92 JA |
245 | if (check_syslet_support(sd)) { |
246 | log_err("fio: syslets do not appear to work\n"); | |
247 | free(sd->events); | |
12cbb469 JA |
248 | free(sd->ring); |
249 | free(sd->stack); | |
76f58b92 JA |
250 | free(sd); |
251 | return 1; | |
252 | } | |
253 | ||
a4f4fdd7 | 254 | td->io_ops->data = sd; |
a4f4fdd7 JA |
255 | return 0; |
256 | } | |
257 | ||
258 | static struct ioengine_ops ioengine = { | |
259 | .name = "syslet-rw", | |
260 | .version = FIO_IOOPS_VERSION, | |
261 | .init = fio_syslet_init, | |
a4f4fdd7 JA |
262 | .queue = fio_syslet_queue, |
263 | .getevents = fio_syslet_getevents, | |
264 | .event = fio_syslet_event, | |
265 | .cleanup = fio_syslet_cleanup, | |
b5af8293 JA |
266 | .open_file = generic_open_file, |
267 | .close_file = generic_close_file, | |
a4f4fdd7 JA |
268 | }; |
269 | ||
270 | #else /* FIO_HAVE_SYSLET */ | |
271 | ||
272 | /* | |
273 | * When we have a proper configure system in place, we simply wont build | |
274 | * and install this io engine. For now install a crippled version that | |
275 | * just complains and fails to load. | |
276 | */ | |
277 | static int fio_syslet_init(struct thread_data fio_unused *td) | |
278 | { | |
279 | fprintf(stderr, "fio: syslet not available\n"); | |
280 | return 1; | |
281 | } | |
282 | ||
283 | static struct ioengine_ops ioengine = { | |
284 | .name = "syslet-rw", | |
285 | .version = FIO_IOOPS_VERSION, | |
286 | .init = fio_syslet_init, | |
287 | }; | |
288 | ||
289 | #endif /* FIO_HAVE_SYSLET */ | |
290 | ||
291 | static void fio_init fio_syslet_register(void) | |
292 | { | |
293 | register_ioengine(&ioengine); | |
294 | } | |
295 | ||
296 | static void fio_exit fio_syslet_unregister(void) | |
297 | { | |
298 | unregister_ioengine(&ioengine); | |
299 | } |