Commit | Line | Data |
---|---|---|
d643a1e2 RJ |
1 | /* |
2 | * NBD engine | |
3 | * | |
4 | * IO engine that talks to an NBD server. | |
5 | * | |
6 | * Copyright (C) 2019 Red Hat Inc. | |
7 | * Written by Richard W.M. Jones <rjones@redhat.com> | |
8 | * | |
9 | */ | |
10 | ||
11 | #include <stdio.h> | |
12 | #include <stdlib.h> | |
13 | #include <stdint.h> | |
14 | #include <errno.h> | |
15 | ||
16 | #include <libnbd.h> | |
17 | ||
18 | #include "../fio.h" | |
19 | #include "../optgroup.h" | |
20 | ||
21 | /* Actually this differs across servers, but for nbdkit ... */ | |
22 | #define NBD_MAX_REQUEST_SIZE (64 * 1024 * 1024) | |
23 | ||
24 | /* Storage for the NBD handle. */ | |
25 | struct nbd_data { | |
26 | struct nbd_handle *nbd; | |
27 | int debug; | |
28 | ||
29 | /* The list of completed io_u structs. */ | |
30 | struct io_u **completed; | |
31 | size_t nr_completed; | |
32 | }; | |
33 | ||
34 | /* Options. */ | |
35 | struct nbd_options { | |
36 | void *padding; | |
37 | char *uri; | |
38 | }; | |
39 | ||
40 | static struct fio_option options[] = { | |
41 | { | |
42 | .name = "uri", | |
43 | .lname = "NBD URI", | |
44 | .help = "Name of NBD URI", | |
45 | .category = FIO_OPT_C_ENGINE, | |
46 | .group = FIO_OPT_G_NBD, | |
47 | .type = FIO_OPT_STR_STORE, | |
48 | .off1 = offsetof(struct nbd_options, uri), | |
49 | }, | |
50 | { | |
51 | .name = NULL, | |
52 | }, | |
53 | }; | |
54 | ||
55 | /* Alocates nbd_data. */ | |
56 | static int nbd_setup(struct thread_data *td) | |
57 | { | |
58 | struct nbd_data *nbd_data; | |
59 | struct nbd_options *o = td->eo; | |
60 | struct fio_file *f; | |
61 | int r; | |
62 | int64_t size; | |
63 | ||
64 | nbd_data = calloc(1, sizeof(*nbd_data)); | |
65 | if (!nbd_data) { | |
66 | td_verror(td, errno, "calloc"); | |
67 | return 1; | |
68 | } | |
69 | td->io_ops_data = nbd_data; | |
70 | ||
71 | /* Pretend to deal with files. See engines/rbd.c */ | |
72 | if (!td->files_index) { | |
73 | add_file(td, "nbd", 0, 0); | |
74 | td->o.nr_files = td->o.nr_files ? : 1; | |
75 | td->o.open_files++; | |
76 | } | |
77 | f = td->files[0]; | |
78 | ||
79 | nbd_data->nbd = nbd_create(); | |
80 | if (!nbd_data->nbd) { | |
81 | log_err("fio: nbd_create: %s\n", nbd_get_error()); | |
82 | return 1; | |
83 | } | |
84 | ||
85 | /* Get the debug flag which can be set through LIBNBD_DEBUG=1. */ | |
86 | nbd_data->debug = nbd_get_debug(nbd_data->nbd); | |
87 | ||
88 | /* Connect synchronously here so we can check for the size and | |
89 | * in future other properties of the server. | |
90 | */ | |
91 | if (!o->uri) { | |
92 | log_err("fio: nbd: uri parameter was not specified\n"); | |
93 | return 1; | |
94 | } | |
95 | r = nbd_connect_uri(nbd_data->nbd, o->uri); | |
96 | if (r == -1) { | |
97 | log_err("fio: nbd_connect_uri: %s\n", nbd_get_error()); | |
98 | return 1; | |
99 | } | |
100 | size = nbd_get_size(nbd_data->nbd); | |
101 | if (size == -1) { | |
102 | log_err("fio: nbd_get_size: %s\n", nbd_get_error()); | |
103 | return 1; | |
104 | } | |
105 | ||
106 | f->real_file_size = size; | |
107 | ||
108 | nbd_close (nbd_data->nbd); | |
109 | nbd_data->nbd = NULL; | |
110 | ||
111 | return 0; | |
112 | } | |
113 | ||
114 | /* Closes socket and frees nbd_data -- the opposite of nbd_setup. */ | |
115 | static void nbd_cleanup(struct thread_data *td) | |
116 | { | |
117 | struct nbd_data *nbd_data = td->io_ops_data; | |
118 | ||
119 | if (nbd_data) { | |
120 | if (nbd_data->nbd) | |
121 | nbd_close(nbd_data->nbd); | |
122 | free(nbd_data); | |
123 | } | |
124 | } | |
125 | ||
126 | /* Connect to the server from each thread. */ | |
127 | static int nbd_init(struct thread_data *td) | |
128 | { | |
129 | struct nbd_options *o = td->eo; | |
130 | struct nbd_data *nbd_data = td->io_ops_data; | |
131 | int r; | |
132 | ||
133 | if (!o->uri) { | |
134 | log_err("fio: nbd: uri parameter was not specified\n"); | |
135 | return 1; | |
136 | } | |
137 | ||
138 | nbd_data->nbd = nbd_create(); | |
139 | if (!nbd_data->nbd) { | |
140 | log_err("fio: nbd_create: %s\n", nbd_get_error()); | |
141 | return 1; | |
142 | } | |
143 | /* This is actually a synchronous connect and handshake. */ | |
144 | r = nbd_connect_uri(nbd_data->nbd, o->uri); | |
145 | if (r == -1) { | |
146 | log_err("fio: nbd_connect_uri: %s\n", nbd_get_error()); | |
147 | return 1; | |
148 | } | |
149 | ||
150 | log_info("fio: connected to NBD server\n"); | |
151 | return 0; | |
152 | } | |
153 | ||
154 | /* A command in flight has been completed. */ | |
155 | static int cmd_completed (unsigned valid_flag, void *vp, int *error) | |
156 | { | |
157 | struct io_u *io_u; | |
158 | struct nbd_data *nbd_data; | |
159 | struct io_u **completed; | |
160 | ||
161 | if (!(valid_flag & LIBNBD_CALLBACK_VALID)) | |
162 | return 0; | |
163 | ||
164 | io_u = vp; | |
165 | nbd_data = io_u->engine_data; | |
166 | ||
167 | if (nbd_data->debug) | |
168 | log_info("fio: nbd: command completed\n"); | |
169 | ||
170 | if (*error != 0) | |
171 | io_u->error = *error; | |
172 | else | |
173 | io_u->error = 0; | |
174 | ||
175 | /* Add this completion to the list so it can be picked up | |
176 | * later by ->event. | |
177 | */ | |
178 | completed = realloc(nbd_data->completed, | |
179 | sizeof(struct io_u *) * | |
180 | (nbd_data->nr_completed+1)); | |
181 | if (completed == NULL) { | |
182 | io_u->error = errno; | |
183 | return 0; | |
184 | } | |
185 | ||
186 | nbd_data->completed = completed; | |
187 | nbd_data->completed[nbd_data->nr_completed] = io_u; | |
188 | nbd_data->nr_completed++; | |
189 | ||
190 | return 0; | |
191 | } | |
192 | ||
193 | /* Begin read or write request. */ | |
194 | static enum fio_q_status nbd_queue(struct thread_data *td, | |
195 | struct io_u *io_u) | |
196 | { | |
197 | struct nbd_data *nbd_data = td->io_ops_data; | |
198 | int r; | |
199 | ||
200 | fio_ro_check(td, io_u); | |
201 | ||
202 | io_u->engine_data = nbd_data; | |
203 | ||
204 | if (io_u->ddir == DDIR_WRITE || io_u->ddir == DDIR_READ) | |
205 | assert(io_u->xfer_buflen <= NBD_MAX_REQUEST_SIZE); | |
206 | ||
207 | switch (io_u->ddir) { | |
208 | case DDIR_READ: | |
209 | r = nbd_aio_pread_callback(nbd_data->nbd, | |
210 | io_u->xfer_buf, io_u->xfer_buflen, | |
211 | io_u->offset, | |
212 | cmd_completed, io_u, | |
213 | 0); | |
214 | break; | |
215 | case DDIR_WRITE: | |
216 | r = nbd_aio_pwrite_callback(nbd_data->nbd, | |
217 | io_u->xfer_buf, io_u->xfer_buflen, | |
218 | io_u->offset, | |
219 | cmd_completed, io_u, | |
220 | 0); | |
221 | break; | |
222 | case DDIR_TRIM: | |
223 | r = nbd_aio_trim_callback(nbd_data->nbd, io_u->xfer_buflen, | |
224 | io_u->offset, | |
225 | cmd_completed, io_u, | |
226 | 0); | |
227 | break; | |
228 | case DDIR_SYNC: | |
229 | /* XXX We could probably also handle | |
230 | * DDIR_SYNC_FILE_RANGE with a bit of effort. | |
231 | */ | |
232 | r = nbd_aio_flush_callback(nbd_data->nbd, | |
233 | cmd_completed, io_u, | |
234 | 0); | |
235 | break; | |
236 | default: | |
237 | io_u->error = EINVAL; | |
238 | return FIO_Q_COMPLETED; | |
239 | } | |
240 | ||
241 | if (r == -1) { | |
242 | /* errno is optional information on libnbd error path; | |
243 | * if it's 0, set it to a default value | |
244 | */ | |
245 | io_u->error = nbd_get_errno(); | |
246 | if (io_u->error == 0) | |
247 | io_u->error = EIO; | |
248 | return FIO_Q_COMPLETED; | |
249 | } | |
250 | ||
251 | if (nbd_data->debug) | |
252 | log_info("fio: nbd: command issued\n"); | |
253 | io_u->error = 0; | |
254 | return FIO_Q_QUEUED; | |
255 | } | |
256 | ||
257 | static unsigned retire_commands(struct nbd_handle *nbd) | |
258 | { | |
259 | int64_t cookie; | |
260 | unsigned r = 0; | |
261 | ||
262 | while ((cookie = nbd_aio_peek_command_completed(nbd)) > 0) { | |
263 | /* Ignore the return value. cmd_completed has already | |
264 | * checked for an error and set io_u->error. We only | |
265 | * have to call this to retire the command. | |
266 | */ | |
267 | nbd_aio_command_completed(nbd, cookie); | |
268 | r++; | |
269 | } | |
270 | ||
271 | if (nbd_get_debug(nbd)) | |
272 | log_info("fio: nbd: %u commands retired\n", r); | |
273 | return r; | |
274 | } | |
275 | ||
276 | static int nbd_getevents(struct thread_data *td, unsigned int min, | |
277 | unsigned int max, const struct timespec *t) | |
278 | { | |
279 | struct nbd_data *nbd_data = td->io_ops_data; | |
280 | int r; | |
281 | unsigned events = 0; | |
282 | int timeout; | |
283 | ||
284 | /* XXX This handling of timeout is wrong because it will wait | |
285 | * for up to loop iterations * timeout. | |
286 | */ | |
287 | timeout = !t ? -1 : t->tv_sec * 1000 + t->tv_nsec / 1000000; | |
288 | ||
289 | while (events < min) { | |
290 | r = nbd_poll(nbd_data->nbd, timeout); | |
291 | if (r == -1) { | |
292 | /* error in poll */ | |
293 | log_err("fio: nbd_poll: %s\n", nbd_get_error()); | |
294 | return -1; | |
295 | } | |
296 | else { | |
297 | /* poll made progress */ | |
298 | events += retire_commands(nbd_data->nbd); | |
299 | } | |
300 | } | |
301 | ||
302 | return events; | |
303 | } | |
304 | ||
305 | static struct io_u *nbd_event(struct thread_data *td, int event) | |
306 | { | |
307 | struct nbd_data *nbd_data = td->io_ops_data; | |
308 | ||
309 | if (nbd_data->nr_completed == 0) | |
310 | return NULL; | |
311 | ||
312 | /* XXX We ignore the event number and assume fio calls us | |
313 | * exactly once for [0..nr_events-1]. | |
314 | */ | |
315 | nbd_data->nr_completed--; | |
316 | return nbd_data->completed[nbd_data->nr_completed]; | |
317 | } | |
318 | ||
319 | static int nbd_io_u_init(struct thread_data *td, struct io_u *io_u) | |
320 | { | |
321 | io_u->engine_data = NULL; | |
322 | return 0; | |
323 | } | |
324 | ||
325 | static void nbd_io_u_free(struct thread_data *td, struct io_u *io_u) | |
326 | { | |
327 | /* Nothing needs to be done. */ | |
328 | } | |
329 | ||
330 | static int nbd_open_file(struct thread_data *td, struct fio_file *f) | |
331 | { | |
332 | return 0; | |
333 | } | |
334 | ||
335 | static int nbd_invalidate(struct thread_data *td, struct fio_file *f) | |
336 | { | |
337 | return 0; | |
338 | } | |
339 | ||
340 | static struct ioengine_ops ioengine = { | |
341 | .name = "nbd", | |
342 | .version = FIO_IOOPS_VERSION, | |
343 | .options = options, | |
344 | .option_struct_size = sizeof(struct nbd_options), | |
345 | .flags = FIO_DISKLESSIO | FIO_NOEXTEND, | |
346 | ||
347 | .setup = nbd_setup, | |
348 | .init = nbd_init, | |
349 | .cleanup = nbd_cleanup, | |
350 | .queue = nbd_queue, | |
351 | .getevents = nbd_getevents, | |
352 | .event = nbd_event, | |
353 | .io_u_init = nbd_io_u_init, | |
354 | .io_u_free = nbd_io_u_free, | |
355 | ||
356 | .open_file = nbd_open_file, | |
357 | .invalidate = nbd_invalidate, | |
358 | }; | |
359 | ||
360 | static void fio_init fio_nbd_register(void) | |
361 | { | |
362 | register_ioengine(&ioengine); | |
363 | } | |
364 | ||
365 | static void fio_exit fio_nbd_unregister(void) | |
366 | { | |
367 | unregister_ioengine(&ioengine); | |
368 | } |