Commit | Line | Data |
---|---|---|
6e7d7dfb | 1 | /* |
2 | * glusterfs engine | |
3 | * | |
cc47f094 | 4 | * common Glusterfs's gfapi interface |
6e7d7dfb | 5 | * |
6 | */ | |
7 | ||
cc47f094 | 8 | #include "gfapi.h" |
d220c761 | 9 | #include "../optgroup.h" |
6e7d7dfb | 10 | |
cc47f094 | 11 | struct fio_option gfapi_options[] = { |
b29c813f JA |
12 | { |
13 | .name = "volume", | |
14 | .lname = "Glusterfs volume", | |
15 | .type = FIO_OPT_STR_STORE, | |
16 | .help = "Name of the Glusterfs volume", | |
17 | .off1 = offsetof(struct gf_options, gf_vol), | |
18 | .category = FIO_OPT_C_ENGINE, | |
19 | .group = FIO_OPT_G_GFAPI, | |
20 | }, | |
21 | { | |
22 | .name = "brick", | |
23 | .lname = "Glusterfs brick name", | |
24 | .type = FIO_OPT_STR_STORE, | |
25 | .help = "Name of the Glusterfs brick to connect", | |
26 | .off1 = offsetof(struct gf_options, gf_brick), | |
27 | .category = FIO_OPT_C_ENGINE, | |
28 | .group = FIO_OPT_G_GFAPI, | |
29 | }, | |
53c508dd ZH |
30 | { |
31 | .name = "single-instance", | |
32 | .lname = "Single glusterfs instance", | |
33 | .type = FIO_OPT_BOOL, | |
34 | .help = "Only one glusterfs instance", | |
35 | .off1 = offsetof(struct gf_options, gf_single_instance), | |
36 | .category = FIO_OPT_C_ENGINE, | |
37 | .group = FIO_OPT_G_GFAPI, | |
38 | }, | |
b29c813f JA |
39 | { |
40 | .name = NULL, | |
41 | }, | |
6e7d7dfb | 42 | }; |
43 | ||
53c508dd ZH |
44 | struct glfs_info { |
45 | struct flist_head list; | |
46 | char *volume; | |
47 | char *brick; | |
48 | glfs_t *fs; | |
49 | int refcount; | |
50 | }; | |
51 | ||
52 | static pthread_mutex_t glfs_lock = PTHREAD_MUTEX_INITIALIZER; | |
53 | static FLIST_HEAD(glfs_list_head); | |
54 | ||
55 | static glfs_t *fio_gf_new_fs(char *volume, char *brick) | |
6e7d7dfb | 56 | { |
57 | int r = 0; | |
53c508dd ZH |
58 | glfs_t *fs; |
59 | struct stat sb = { 0, }; | |
60 | ||
61 | fs = glfs_new(volume); | |
62 | if (!fs) { | |
63 | log_err("glfs_new failed.\n"); | |
64 | goto out; | |
65 | } | |
66 | glfs_set_logging(fs, "/tmp/fio_gfapi.log", 7); | |
67 | /* default to tcp */ | |
68 | r = glfs_set_volfile_server(fs, "tcp", brick, 0); | |
69 | if (r) { | |
70 | log_err("glfs_set_volfile_server failed.\n"); | |
71 | goto out; | |
72 | } | |
73 | r = glfs_init(fs); | |
74 | if (r) { | |
75 | log_err("glfs_init failed. Is glusterd running on brick?\n"); | |
76 | goto out; | |
77 | } | |
78 | sleep(2); | |
79 | r = glfs_lstat(fs, ".", &sb); | |
80 | if (r) { | |
81 | log_err("glfs_lstat failed.\n"); | |
82 | goto out; | |
83 | } | |
84 | ||
85 | out: | |
86 | if (r) { | |
87 | glfs_fini(fs); | |
88 | fs = NULL; | |
89 | } | |
90 | return fs; | |
91 | } | |
92 | ||
93 | static glfs_t *fio_gf_get_glfs(struct gf_options *opt, | |
94 | char *volume, char *brick) | |
95 | { | |
96 | struct glfs_info *glfs = NULL; | |
97 | struct glfs_info *tmp; | |
98 | struct flist_head *entry; | |
99 | ||
100 | if (!opt->gf_single_instance) | |
101 | return fio_gf_new_fs(volume, brick); | |
102 | ||
103 | pthread_mutex_lock (&glfs_lock); | |
104 | ||
105 | flist_for_each(entry, &glfs_list_head) { | |
106 | tmp = flist_entry(entry, struct glfs_info, list); | |
107 | if (!strcmp(volume, tmp->volume) && | |
108 | !strcmp(brick, tmp->brick)) { | |
109 | glfs = tmp; | |
110 | break; | |
111 | } | |
112 | } | |
113 | ||
114 | if (glfs) { | |
115 | glfs->refcount++; | |
116 | } else { | |
117 | glfs = malloc(sizeof(*glfs)); | |
118 | if (!glfs) | |
119 | goto out; | |
120 | INIT_FLIST_HEAD(&glfs->list); | |
121 | glfs->refcount = 0; | |
122 | glfs->volume = strdup(volume); | |
123 | glfs->brick = strdup(brick); | |
124 | glfs->fs = fio_gf_new_fs(volume, brick); | |
125 | if (!glfs->fs) { | |
126 | free(glfs); | |
127 | glfs = NULL; | |
128 | goto out; | |
129 | } | |
130 | ||
131 | flist_add_tail(&glfs->list, &glfs_list_head); | |
132 | glfs->refcount = 1; | |
133 | } | |
134 | ||
135 | out: | |
136 | pthread_mutex_unlock (&glfs_lock); | |
137 | ||
138 | if (glfs) | |
139 | return glfs->fs; | |
140 | return NULL; | |
141 | } | |
142 | ||
143 | static void fio_gf_put_glfs(struct gf_options *opt, glfs_t *fs) | |
144 | { | |
145 | struct glfs_info *glfs = NULL; | |
146 | struct glfs_info *tmp; | |
147 | struct flist_head *entry; | |
148 | ||
149 | if (!opt->gf_single_instance) { | |
150 | glfs_fini(fs); | |
151 | return; | |
152 | } | |
153 | ||
154 | pthread_mutex_lock (&glfs_lock); | |
155 | ||
156 | flist_for_each(entry, &glfs_list_head) { | |
157 | tmp = flist_entry(entry, struct glfs_info, list); | |
158 | if (tmp->fs == fs) { | |
159 | glfs = tmp; | |
160 | break; | |
161 | } | |
162 | } | |
163 | ||
164 | if (!glfs) { | |
165 | log_err("glfs not found to fini.\n"); | |
166 | } else { | |
167 | glfs->refcount--; | |
168 | ||
169 | if (glfs->refcount == 0) { | |
170 | glfs_fini(glfs->fs); | |
171 | free(glfs->volume); | |
172 | free(glfs->brick); | |
173 | flist_del(&glfs->list); | |
174 | } | |
175 | } | |
176 | ||
177 | pthread_mutex_unlock (&glfs_lock); | |
178 | } | |
179 | ||
180 | int fio_gf_setup(struct thread_data *td) | |
181 | { | |
6e7d7dfb | 182 | struct gf_data *g = NULL; |
183 | struct gf_options *opt = td->eo; | |
7cb57176 | 184 | |
185 | dprint(FD_IO, "fio setup\n"); | |
6e7d7dfb | 186 | |
565e784d | 187 | if (td->io_ops_data) |
b29c813f | 188 | return 0; |
6e7d7dfb | 189 | |
190 | g = malloc(sizeof(struct gf_data)); | |
b29c813f JA |
191 | if (!g) { |
192 | log_err("malloc failed.\n"); | |
193 | return -ENOMEM; | |
6e7d7dfb | 194 | } |
b29c813f JA |
195 | g->fd = NULL; |
196 | g->aio_events = NULL; | |
197 | ||
53c508dd ZH |
198 | g->fs = fio_gf_get_glfs(opt, opt->gf_vol, opt->gf_brick); |
199 | if (!g->fs) | |
b29c813f | 200 | goto cleanup; |
53c508dd | 201 | |
7cb57176 | 202 | dprint(FD_FILE, "fio setup %p\n", g->fs); |
565e784d | 203 | td->io_ops_data = g; |
93782cbb | 204 | return 0; |
6e7d7dfb | 205 | cleanup: |
93782cbb | 206 | free(g); |
565e784d | 207 | td->io_ops_data = NULL; |
53c508dd | 208 | return -EIO; |
6e7d7dfb | 209 | } |
210 | ||
cc47f094 | 211 | void fio_gf_cleanup(struct thread_data *td) |
6e7d7dfb | 212 | { |
565e784d | 213 | struct gf_data *g = td->io_ops_data; |
cc47f094 | 214 | |
215 | if (g) { | |
b29c813f JA |
216 | if (g->aio_events) |
217 | free(g->aio_events); | |
218 | if (g->fd) | |
219 | glfs_close(g->fd); | |
220 | if (g->fs) | |
53c508dd | 221 | fio_gf_put_glfs(td->eo, g->fs); |
b29c813f | 222 | free(g); |
565e784d | 223 | td->io_ops_data = NULL; |
cc47f094 | 224 | } |
6e7d7dfb | 225 | } |
226 | ||
cc47f094 | 227 | int fio_gf_get_file_size(struct thread_data *td, struct fio_file *f) |
6e7d7dfb | 228 | { |
b29c813f JA |
229 | struct stat buf; |
230 | int ret; | |
565e784d | 231 | struct gf_data *g = td->io_ops_data; |
6e7d7dfb | 232 | |
b29c813f | 233 | dprint(FD_FILE, "get file size %s\n", f->file_name); |
6aa56500 | 234 | |
b29c813f JA |
235 | if (!g || !g->fs) { |
236 | return 0; | |
237 | } | |
238 | if (fio_file_size_known(f)) | |
239 | return 0; | |
6e7d7dfb | 240 | |
b29c813f JA |
241 | ret = glfs_lstat(g->fs, f->file_name, &buf); |
242 | if (ret < 0) { | |
243 | log_err("glfs_lstat failed.\n"); | |
244 | return ret; | |
245 | } | |
6e7d7dfb | 246 | |
b29c813f JA |
247 | f->real_file_size = buf.st_size; |
248 | fio_file_set_size_known(f); | |
6e7d7dfb | 249 | |
b29c813f | 250 | return 0; |
6e7d7dfb | 251 | |
252 | } | |
253 | ||
cc47f094 | 254 | int fio_gf_open_file(struct thread_data *td, struct fio_file *f) |
6e7d7dfb | 255 | { |
a8a93dee | 256 | |
b29c813f JA |
257 | int flags = 0; |
258 | int ret = 0; | |
565e784d | 259 | struct gf_data *g = td->io_ops_data; |
b29c813f JA |
260 | struct stat sb = { 0, }; |
261 | ||
262 | if (td_write(td)) { | |
263 | if (!read_only) | |
264 | flags = O_RDWR; | |
265 | } else if (td_read(td)) { | |
266 | if (!read_only) | |
267 | flags = O_RDWR; | |
268 | else | |
269 | flags = O_RDONLY; | |
270 | } | |
65bea10f TM |
271 | |
272 | if (td->o.odirect) | |
273 | flags |= OS_O_DIRECT; | |
eb9f8d7f | 274 | flags |= td->o.sync_io; |
65bea10f | 275 | |
b29c813f | 276 | dprint(FD_FILE, "fio file %s open mode %s td rw %s\n", f->file_name, |
65bea10f | 277 | flags & O_RDONLY ? "ro" : "rw", td_read(td) ? "read" : "write"); |
b29c813f JA |
278 | g->fd = glfs_creat(g->fs, f->file_name, flags, 0644); |
279 | if (!g->fd) { | |
b29c813f | 280 | ret = errno; |
1a889967 TM |
281 | log_err("glfs_creat failed.\n"); |
282 | return ret; | |
b29c813f JA |
283 | } |
284 | /* file for read doesn't exist or shorter than required, create/extend it */ | |
285 | if (td_read(td)) { | |
286 | if (glfs_lstat(g->fs, f->file_name, &sb) | |
287 | || sb.st_size < f->real_file_size) { | |
f6149216 SW |
288 | dprint(FD_FILE, "fio extend file %s from %jd to %" PRIu64 "\n", |
289 | f->file_name, (intmax_t) sb.st_size, f->real_file_size); | |
ce4d13ca JA |
290 | #if defined(CONFIG_GF_NEW_API) |
291 | ret = glfs_ftruncate(g->fd, f->real_file_size, NULL, NULL); | |
292 | #else | |
b29c813f | 293 | ret = glfs_ftruncate(g->fd, f->real_file_size); |
ce4d13ca | 294 | #endif |
b29c813f | 295 | if (ret) { |
f6149216 | 296 | log_err("failed fio extend file %s to %" PRIu64 "\n", |
b29c813f JA |
297 | f->file_name, f->real_file_size); |
298 | } else { | |
299 | unsigned long long left; | |
300 | unsigned int bs; | |
301 | char *b; | |
302 | int r; | |
303 | ||
304 | /* fill the file, copied from extend_file */ | |
305 | b = malloc(td->o.max_bs[DDIR_WRITE]); | |
306 | ||
307 | left = f->real_file_size; | |
308 | while (left && !td->terminate) { | |
309 | bs = td->o.max_bs[DDIR_WRITE]; | |
310 | if (bs > left) | |
311 | bs = left; | |
312 | ||
313 | fill_io_buffer(td, b, bs, bs); | |
314 | ||
315 | r = glfs_write(g->fd, b, bs, 0); | |
316 | dprint(FD_IO, | |
f6149216 | 317 | "fio write %d of %" PRIu64 " file %s\n", |
b29c813f JA |
318 | r, f->real_file_size, |
319 | f->file_name); | |
320 | ||
321 | if (r > 0) { | |
322 | left -= r; | |
323 | continue; | |
324 | } else { | |
325 | if (r < 0) { | |
326 | int __e = errno; | |
327 | ||
328 | if (__e == ENOSPC) { | |
329 | if (td->o. | |
330 | fill_device) | |
331 | break; | |
332 | log_info | |
333 | ("fio: ENOSPC on laying out " | |
334 | "file, stopping\n"); | |
335 | break; | |
336 | } | |
337 | td_verror(td, errno, | |
338 | "write"); | |
339 | } else | |
340 | td_verror(td, EIO, | |
341 | "write"); | |
342 | ||
343 | break; | |
344 | } | |
345 | } | |
346 | ||
347 | if (b) | |
348 | free(b); | |
349 | glfs_lseek(g->fd, 0, SEEK_SET); | |
350 | ||
38ef9c90 | 351 | if (td->terminate && td->o.unlink) { |
b29c813f JA |
352 | dprint(FD_FILE, "terminate unlink %s\n", |
353 | f->file_name); | |
38ef9c90 | 354 | glfs_unlink(g->fs, f->file_name); |
b29c813f | 355 | } else if (td->o.create_fsync) { |
ce4d13ca JA |
356 | #if defined(CONFIG_GF_NEW_API) |
357 | if (glfs_fsync(g->fd, NULL, NULL) < 0) { | |
358 | #else | |
b29c813f | 359 | if (glfs_fsync(g->fd) < 0) { |
ce4d13ca | 360 | #endif |
b29c813f JA |
361 | dprint(FD_FILE, |
362 | "failed to sync, close %s\n", | |
363 | f->file_name); | |
364 | td_verror(td, errno, "fsync"); | |
365 | glfs_close(g->fd); | |
366 | g->fd = NULL; | |
367 | return 1; | |
368 | } | |
369 | } | |
370 | } | |
371 | } | |
372 | } | |
6fa14b99 | 373 | #if defined(GFAPI_USE_FADVISE) |
b29c813f JA |
374 | { |
375 | int r = 0; | |
376 | if (td_random(td)) { | |
377 | r = glfs_fadvise(g->fd, 0, f->real_file_size, | |
378 | POSIX_FADV_RANDOM); | |
379 | } else { | |
380 | r = glfs_fadvise(g->fd, 0, f->real_file_size, | |
381 | POSIX_FADV_SEQUENTIAL); | |
382 | } | |
383 | if (r) { | |
384 | dprint(FD_FILE, "fio %p fadvise %s status %d\n", g->fs, | |
385 | f->file_name, r); | |
386 | } | |
387 | } | |
6fa14b99 | 388 | #endif |
b29c813f JA |
389 | dprint(FD_FILE, "fio %p created %s\n", g->fs, f->file_name); |
390 | f->fd = -1; | |
391 | f->shadow_fd = -1; | |
e6590c12 | 392 | td->o.open_files ++; |
b29c813f | 393 | return ret; |
6e7d7dfb | 394 | } |
395 | ||
cc47f094 | 396 | int fio_gf_close_file(struct thread_data *td, struct fio_file *f) |
6e7d7dfb | 397 | { |
398 | int ret = 0; | |
565e784d | 399 | struct gf_data *g = td->io_ops_data; |
6e7d7dfb | 400 | |
401 | dprint(FD_FILE, "fd close %s\n", f->file_name); | |
402 | ||
b29c813f JA |
403 | if (g) { |
404 | if (g->fd && glfs_close(g->fd) < 0) | |
405 | ret = errno; | |
38ef9c90 CF |
406 | g->fd = NULL; |
407 | } | |
408 | ||
409 | return ret; | |
410 | } | |
411 | ||
412 | int fio_gf_unlink_file(struct thread_data *td, struct fio_file *f) | |
413 | { | |
414 | int ret = 0; | |
565e784d | 415 | struct gf_data *g = td->io_ops_data; |
38ef9c90 CF |
416 | |
417 | dprint(FD_FILE, "fd unlink %s\n", f->file_name); | |
418 | ||
419 | if (g) { | |
420 | if (g->fd && glfs_close(g->fd) < 0) | |
421 | ret = errno; | |
422 | ||
423 | glfs_unlink(g->fs, f->file_name); | |
7cb57176 | 424 | |
b29c813f JA |
425 | if (g->fs) |
426 | glfs_fini(g->fs); | |
6e7d7dfb | 427 | |
b29c813f JA |
428 | g->fd = NULL; |
429 | free(g); | |
430 | } | |
565e784d | 431 | td->io_ops_data = NULL; |
6e7d7dfb | 432 | |
433 | return ret; | |
434 | } |