Commit | Line | Data |
---|---|---|
a4f4fdd7 | 1 | /* |
da751ca9 JA |
2 | * syslet engine |
3 | * | |
4 | * IO engine that does regular pread(2)/pwrite(2) to transfer data, but | |
5 | * with syslets to make the execution async. | |
a4f4fdd7 JA |
6 | * |
7 | */ | |
8 | #include <stdio.h> | |
9 | #include <stdlib.h> | |
10 | #include <unistd.h> | |
11 | #include <errno.h> | |
12 | #include <assert.h> | |
b8846354 | 13 | #include <asm/unistd.h> |
a4f4fdd7 JA |
14 | |
15 | #include "../fio.h" | |
a4f4fdd7 JA |
16 | |
17 | #ifdef FIO_HAVE_SYSLET | |
18 | ||
1760e679 JA |
19 | #ifdef __NR_pread64 |
20 | #define __NR_fio_pread __NR_pread64 | |
21 | #define __NR_fio_pwrite __NR_pwrite64 | |
22 | #else | |
23 | #define __NR_fio_pread __NR_pread | |
24 | #define __NR_fio_pwrite __NR_pwrite | |
25 | #endif | |
26 | ||
67f5b27e JA |
27 | #define ATOM_TO_IOU(p) ((struct io_u *) (unsigned long) (p)) |
28 | ||
a4f4fdd7 JA |
29 | struct syslet_data { |
30 | struct io_u **events; | |
31 | unsigned int nr_events; | |
32 | ||
bf0dc8fa | 33 | struct async_head_user ahu; |
a4f4fdd7 | 34 | struct syslet_uatom **ring; |
9ff9de69 JA |
35 | |
36 | struct syslet_uatom *head, *tail; | |
a4f4fdd7 JA |
37 | }; |
38 | ||
9ff9de69 JA |
39 | static void fio_syslet_complete_atom(struct thread_data *td, |
40 | struct syslet_uatom *atom) | |
41 | { | |
42 | struct syslet_data *sd = td->io_ops->data; | |
5b38ee84 | 43 | struct syslet_uatom *last; |
9ff9de69 | 44 | struct io_u *io_u; |
9ff9de69 JA |
45 | |
46 | /* | |
5b38ee84 JA |
47 | * complete from the beginning of the sequence up to (and |
48 | * including) this atom | |
9ff9de69 | 49 | */ |
5b38ee84 | 50 | last = atom; |
67f5b27e | 51 | io_u = ATOM_TO_IOU(atom); |
5b38ee84 | 52 | atom = io_u->req.head; |
9ff9de69 JA |
53 | |
54 | /* | |
55 | * now complete in right order | |
56 | */ | |
5b38ee84 | 57 | do { |
9ff9de69 JA |
58 | long ret; |
59 | ||
67f5b27e JA |
60 | io_u = ATOM_TO_IOU(atom); |
61 | ret = *(long *) (unsigned long) atom->ret_ptr; | |
e2e67912 | 62 | if (ret >= 0) |
9ff9de69 JA |
63 | io_u->resid = io_u->xfer_buflen - ret; |
64 | else if (ret < 0) | |
65 | io_u->error = ret; | |
66 | ||
2dc1bbeb | 67 | assert(sd->nr_events < td->o.iodepth); |
9ff9de69 | 68 | sd->events[sd->nr_events++] = io_u; |
9ff9de69 | 69 | |
5b38ee84 JA |
70 | if (atom == last) |
71 | break; | |
9ff9de69 | 72 | |
67f5b27e | 73 | atom = (struct syslet_uatom *) (unsigned long) atom->next; |
5b38ee84 JA |
74 | } while (1); |
75 | ||
76 | assert(!last->next); | |
9ff9de69 JA |
77 | } |
78 | ||
a4f4fdd7 JA |
79 | /* |
80 | * Inspect the ring to see if we have completed events | |
81 | */ | |
82 | static void fio_syslet_complete(struct thread_data *td) | |
83 | { | |
84 | struct syslet_data *sd = td->io_ops->data; | |
85 | ||
86 | do { | |
87 | struct syslet_uatom *atom; | |
a4f4fdd7 | 88 | |
bf0dc8fa | 89 | atom = sd->ring[sd->ahu.user_ring_idx]; |
a4f4fdd7 JA |
90 | if (!atom) |
91 | break; | |
92 | ||
bf0dc8fa | 93 | sd->ring[sd->ahu.user_ring_idx] = NULL; |
2dc1bbeb | 94 | if (++sd->ahu.user_ring_idx == td->o.iodepth) |
bf0dc8fa | 95 | sd->ahu.user_ring_idx = 0; |
a4f4fdd7 | 96 | |
9ff9de69 | 97 | fio_syslet_complete_atom(td, atom); |
a4f4fdd7 JA |
98 | } while (1); |
99 | } | |
100 | ||
101 | static int fio_syslet_getevents(struct thread_data *td, int min, | |
102 | int fio_unused max, | |
103 | struct timespec fio_unused *t) | |
104 | { | |
105 | struct syslet_data *sd = td->io_ops->data; | |
a4f4fdd7 JA |
106 | long ret; |
107 | ||
108 | do { | |
109 | fio_syslet_complete(td); | |
110 | ||
111 | /* | |
112 | * do we have enough immediate completions? | |
113 | */ | |
114 | if (sd->nr_events >= (unsigned int) min) | |
115 | break; | |
116 | ||
117 | /* | |
118 | * OK, we need to wait for some events... | |
119 | */ | |
9ff9de69 | 120 | ret = async_wait(1, sd->ahu.user_ring_idx, &sd->ahu); |
a4f4fdd7 | 121 | if (ret < 0) |
e49499f8 | 122 | return -errno; |
a4f4fdd7 JA |
123 | } while (1); |
124 | ||
125 | ret = sd->nr_events; | |
126 | sd->nr_events = 0; | |
127 | return ret; | |
128 | } | |
129 | ||
130 | static struct io_u *fio_syslet_event(struct thread_data *td, int event) | |
131 | { | |
132 | struct syslet_data *sd = td->io_ops->data; | |
133 | ||
134 | return sd->events[event]; | |
135 | } | |
136 | ||
137 | static void init_atom(struct syslet_uatom *atom, int nr, void *arg0, | |
a2e1b08a JA |
138 | void *arg1, void *arg2, void *arg3, void *ret_ptr, |
139 | unsigned long flags, void *priv) | |
a4f4fdd7 JA |
140 | { |
141 | atom->flags = flags; | |
142 | atom->nr = nr; | |
67f5b27e | 143 | atom->ret_ptr = (uint64_t) (unsigned long) ret_ptr; |
1d79a6d1 | 144 | atom->next = 0; |
67f5b27e JA |
145 | atom->arg_ptr[0] = (uint64_t) (unsigned long) arg0; |
146 | atom->arg_ptr[1] = (uint64_t) (unsigned long) arg1; | |
147 | atom->arg_ptr[2] = (uint64_t) (unsigned long) arg2; | |
148 | atom->arg_ptr[3] = (uint64_t) (unsigned long) arg3; | |
1d79a6d1 ZB |
149 | atom->arg_ptr[4] = 0; |
150 | atom->arg_ptr[5] = 0; | |
67f5b27e | 151 | atom->private = (uint64_t) (unsigned long) priv; |
a4f4fdd7 JA |
152 | } |
153 | ||
154 | /* | |
155 | * Use seek atom for sync | |
156 | */ | |
157 | static void fio_syslet_prep_sync(struct io_u *io_u, struct fio_file *f) | |
158 | { | |
a2e1b08a | 159 | init_atom(&io_u->req.atom, __NR_fsync, &f->fd, NULL, NULL, NULL, |
7d44a745 | 160 | &io_u->req.ret, 0, io_u); |
a4f4fdd7 JA |
161 | } |
162 | ||
163 | static void fio_syslet_prep_rw(struct io_u *io_u, struct fio_file *f) | |
164 | { | |
165 | int nr; | |
166 | ||
a4f4fdd7 JA |
167 | /* |
168 | * prepare rw | |
169 | */ | |
170 | if (io_u->ddir == DDIR_READ) | |
1760e679 | 171 | nr = __NR_fio_pread; |
a4f4fdd7 | 172 | else |
1760e679 | 173 | nr = __NR_fio_pwrite; |
a4f4fdd7 | 174 | |
a2e1b08a | 175 | init_atom(&io_u->req.atom, nr, &f->fd, &io_u->xfer_buf, |
7d44a745 | 176 | &io_u->xfer_buflen, &io_u->offset, &io_u->req.ret, 0, io_u); |
a4f4fdd7 JA |
177 | } |
178 | ||
179 | static int fio_syslet_prep(struct thread_data fio_unused *td, struct io_u *io_u) | |
180 | { | |
181 | struct fio_file *f = io_u->file; | |
182 | ||
183 | if (io_u->ddir == DDIR_SYNC) | |
184 | fio_syslet_prep_sync(io_u, f); | |
185 | else | |
186 | fio_syslet_prep_rw(io_u, f); | |
187 | ||
188 | return 0; | |
189 | } | |
190 | ||
bf0dc8fa IM |
191 | static void cachemiss_thread_start(void) |
192 | { | |
193 | while (1) | |
7756b0d0 | 194 | async_thread(NULL, NULL); |
bf0dc8fa IM |
195 | } |
196 | ||
197 | #define THREAD_STACK_SIZE (16384) | |
198 | ||
199 | static unsigned long thread_stack_alloc() | |
200 | { | |
5b38ee84 | 201 | return (unsigned long) malloc(THREAD_STACK_SIZE) + THREAD_STACK_SIZE; |
bf0dc8fa IM |
202 | } |
203 | ||
a0a930ef JA |
204 | static void fio_syslet_queued(struct thread_data *td, struct syslet_data *sd) |
205 | { | |
206 | struct syslet_uatom *atom; | |
207 | struct timeval now; | |
208 | ||
209 | fio_gettime(&now, NULL); | |
210 | ||
211 | atom = sd->head; | |
212 | while (atom) { | |
67f5b27e | 213 | struct io_u *io_u = ATOM_TO_IOU(atom); |
a0a930ef JA |
214 | |
215 | memcpy(&io_u->issue_time, &now, sizeof(now)); | |
216 | io_u_queued(td, io_u); | |
67f5b27e | 217 | atom = (struct syslet_uatom *) (unsigned long) atom->next; |
a0a930ef JA |
218 | } |
219 | } | |
220 | ||
9ff9de69 | 221 | static int fio_syslet_commit(struct thread_data *td) |
a4f4fdd7 JA |
222 | { |
223 | struct syslet_data *sd = td->io_ops->data; | |
bf0dc8fa | 224 | struct syslet_uatom *done; |
9ff9de69 JA |
225 | |
226 | if (!sd->head) | |
227 | return 0; | |
a4f4fdd7 | 228 | |
5b38ee84 JA |
229 | assert(!sd->tail->next); |
230 | ||
bf0dc8fa IM |
231 | if (!sd->ahu.new_thread_stack) |
232 | sd->ahu.new_thread_stack = thread_stack_alloc(); | |
233 | ||
a0a930ef JA |
234 | fio_syslet_queued(td, sd); |
235 | ||
7d44a745 JA |
236 | /* |
237 | * On sync completion, the atom is returned. So on NULL return | |
238 | * it's queued asynchronously. | |
239 | */ | |
9ff9de69 | 240 | done = async_exec(sd->head, &sd->ahu); |
bf0dc8fa | 241 | |
76f58b92 JA |
242 | if (done == (void *) -1) { |
243 | log_err("fio: syslets don't appear to work\n"); | |
244 | return -1; | |
245 | } | |
246 | ||
9ff9de69 | 247 | sd->head = sd->tail = NULL; |
a4f4fdd7 | 248 | |
9ff9de69 JA |
249 | if (done) |
250 | fio_syslet_complete_atom(td, done); | |
a4f4fdd7 | 251 | |
9ff9de69 JA |
252 | return 0; |
253 | } | |
254 | ||
255 | static int fio_syslet_queue(struct thread_data *td, struct io_u *io_u) | |
256 | { | |
257 | struct syslet_data *sd = td->io_ops->data; | |
bf0dc8fa | 258 | |
7101d9c2 JA |
259 | fio_ro_check(td, io_u); |
260 | ||
9ff9de69 | 261 | if (sd->tail) { |
67f5b27e | 262 | sd->tail->next = (uint64_t) (unsigned long) &io_u->req.atom; |
9ff9de69 JA |
263 | sd->tail = &io_u->req.atom; |
264 | } else | |
1d79a6d1 | 265 | sd->head = sd->tail = (struct syslet_uatom *)&io_u->req.atom; |
a4f4fdd7 | 266 | |
5b38ee84 | 267 | io_u->req.head = sd->head; |
9ff9de69 | 268 | return FIO_Q_QUEUED; |
a4f4fdd7 JA |
269 | } |
270 | ||
db64e9bc | 271 | static int async_head_init(struct syslet_data *sd, unsigned int depth) |
a4f4fdd7 | 272 | { |
a4f4fdd7 JA |
273 | unsigned long ring_size; |
274 | ||
bf0dc8fa | 275 | memset(&sd->ahu, 0, sizeof(struct async_head_user)); |
2ca50be4 | 276 | |
a4f4fdd7 JA |
277 | ring_size = sizeof(struct syslet_uatom *) * depth; |
278 | sd->ring = malloc(ring_size); | |
279 | memset(sd->ring, 0, ring_size); | |
280 | ||
bf0dc8fa | 281 | sd->ahu.user_ring_idx = 0; |
67f5b27e | 282 | sd->ahu.completion_ring_ptr = (uint64_t) (unsigned long) sd->ring; |
bf0dc8fa IM |
283 | sd->ahu.ring_size_bytes = ring_size; |
284 | sd->ahu.head_stack = thread_stack_alloc(); | |
67f5b27e JA |
285 | sd->ahu.head_ip = (uint64_t) (unsigned long) cachemiss_thread_start; |
286 | sd->ahu.new_thread_ip = (uint64_t) (unsigned long) cachemiss_thread_start; | |
6a117e0c | 287 | sd->ahu.new_thread_stack = thread_stack_alloc(); |
db64e9bc JA |
288 | |
289 | return 0; | |
a4f4fdd7 JA |
290 | } |
291 | ||
2ca50be4 | 292 | static void async_head_exit(struct syslet_data *sd) |
a4f4fdd7 | 293 | { |
7f059a76 | 294 | free(sd->ring); |
a4f4fdd7 JA |
295 | } |
296 | ||
76f58b92 JA |
297 | static int check_syslet_support(struct syslet_data *sd) |
298 | { | |
299 | struct syslet_uatom atom; | |
300 | void *ret; | |
301 | ||
302 | init_atom(&atom, __NR_getpid, NULL, NULL, NULL, NULL, NULL, 0, NULL); | |
b4d8ddac | 303 | ret = async_exec(&atom, &sd->ahu); |
76f58b92 JA |
304 | if (ret == (void *) -1) |
305 | return 1; | |
306 | ||
307 | return 0; | |
308 | } | |
309 | ||
a4f4fdd7 JA |
310 | static void fio_syslet_cleanup(struct thread_data *td) |
311 | { | |
312 | struct syslet_data *sd = td->io_ops->data; | |
313 | ||
314 | if (sd) { | |
2ca50be4 | 315 | async_head_exit(sd); |
a4f4fdd7 JA |
316 | free(sd->events); |
317 | free(sd); | |
318 | td->io_ops->data = NULL; | |
319 | } | |
320 | } | |
321 | ||
322 | static int fio_syslet_init(struct thread_data *td) | |
323 | { | |
324 | struct syslet_data *sd; | |
325 | ||
326 | sd = malloc(sizeof(*sd)); | |
327 | memset(sd, 0, sizeof(*sd)); | |
2dc1bbeb JA |
328 | sd->events = malloc(sizeof(struct io_u *) * td->o.iodepth); |
329 | memset(sd->events, 0, sizeof(struct io_u *) * td->o.iodepth); | |
db64e9bc JA |
330 | |
331 | /* | |
332 | * This will handily fail for kernels where syslet isn't available | |
333 | */ | |
2dc1bbeb | 334 | if (async_head_init(sd, td->o.iodepth)) { |
db64e9bc JA |
335 | free(sd->events); |
336 | free(sd); | |
337 | return 1; | |
338 | } | |
339 | ||
76f58b92 JA |
340 | if (check_syslet_support(sd)) { |
341 | log_err("fio: syslets do not appear to work\n"); | |
342 | free(sd->events); | |
343 | free(sd); | |
344 | return 1; | |
345 | } | |
346 | ||
a4f4fdd7 | 347 | td->io_ops->data = sd; |
a4f4fdd7 JA |
348 | return 0; |
349 | } | |
350 | ||
351 | static struct ioengine_ops ioengine = { | |
352 | .name = "syslet-rw", | |
353 | .version = FIO_IOOPS_VERSION, | |
354 | .init = fio_syslet_init, | |
355 | .prep = fio_syslet_prep, | |
356 | .queue = fio_syslet_queue, | |
9ff9de69 | 357 | .commit = fio_syslet_commit, |
a4f4fdd7 JA |
358 | .getevents = fio_syslet_getevents, |
359 | .event = fio_syslet_event, | |
360 | .cleanup = fio_syslet_cleanup, | |
b5af8293 JA |
361 | .open_file = generic_open_file, |
362 | .close_file = generic_close_file, | |
a4f4fdd7 JA |
363 | }; |
364 | ||
365 | #else /* FIO_HAVE_SYSLET */ | |
366 | ||
367 | /* | |
368 | * When we have a proper configure system in place, we simply wont build | |
369 | * and install this io engine. For now install a crippled version that | |
370 | * just complains and fails to load. | |
371 | */ | |
372 | static int fio_syslet_init(struct thread_data fio_unused *td) | |
373 | { | |
374 | fprintf(stderr, "fio: syslet not available\n"); | |
375 | return 1; | |
376 | } | |
377 | ||
378 | static struct ioengine_ops ioengine = { | |
379 | .name = "syslet-rw", | |
380 | .version = FIO_IOOPS_VERSION, | |
381 | .init = fio_syslet_init, | |
382 | }; | |
383 | ||
384 | #endif /* FIO_HAVE_SYSLET */ | |
385 | ||
386 | static void fio_init fio_syslet_register(void) | |
387 | { | |
388 | register_ioengine(&ioengine); | |
389 | } | |
390 | ||
391 | static void fio_exit fio_syslet_unregister(void) | |
392 | { | |
393 | unregister_ioengine(&ioengine); | |
394 | } |