extend --sync to allow {sync,dsync,0,1}, to support O_DSYNC
[fio.git] / engines / glusterfs.c
CommitLineData
6e7d7dfb 1/*
2 * glusterfs engine
3 *
cc47f094 4 * common Glusterfs's gfapi interface
6e7d7dfb 5 *
6 */
7
cc47f094 8#include "gfapi.h"
d220c761 9#include "../optgroup.h"
6e7d7dfb 10
cc47f094 11struct fio_option gfapi_options[] = {
b29c813f
JA
12 {
13 .name = "volume",
14 .lname = "Glusterfs volume",
15 .type = FIO_OPT_STR_STORE,
16 .help = "Name of the Glusterfs volume",
17 .off1 = offsetof(struct gf_options, gf_vol),
18 .category = FIO_OPT_C_ENGINE,
19 .group = FIO_OPT_G_GFAPI,
20 },
21 {
22 .name = "brick",
23 .lname = "Glusterfs brick name",
24 .type = FIO_OPT_STR_STORE,
25 .help = "Name of the Glusterfs brick to connect",
26 .off1 = offsetof(struct gf_options, gf_brick),
27 .category = FIO_OPT_C_ENGINE,
28 .group = FIO_OPT_G_GFAPI,
29 },
53c508dd
ZH
30 {
31 .name = "single-instance",
32 .lname = "Single glusterfs instance",
33 .type = FIO_OPT_BOOL,
34 .help = "Only one glusterfs instance",
35 .off1 = offsetof(struct gf_options, gf_single_instance),
36 .category = FIO_OPT_C_ENGINE,
37 .group = FIO_OPT_G_GFAPI,
38 },
b29c813f
JA
39 {
40 .name = NULL,
41 },
6e7d7dfb 42};
43
53c508dd
ZH
44struct glfs_info {
45 struct flist_head list;
46 char *volume;
47 char *brick;
48 glfs_t *fs;
49 int refcount;
50};
51
52static pthread_mutex_t glfs_lock = PTHREAD_MUTEX_INITIALIZER;
53static FLIST_HEAD(glfs_list_head);
54
55static glfs_t *fio_gf_new_fs(char *volume, char *brick)
6e7d7dfb 56{
57 int r = 0;
53c508dd
ZH
58 glfs_t *fs;
59 struct stat sb = { 0, };
60
61 fs = glfs_new(volume);
62 if (!fs) {
63 log_err("glfs_new failed.\n");
64 goto out;
65 }
66 glfs_set_logging(fs, "/tmp/fio_gfapi.log", 7);
67 /* default to tcp */
68 r = glfs_set_volfile_server(fs, "tcp", brick, 0);
69 if (r) {
70 log_err("glfs_set_volfile_server failed.\n");
71 goto out;
72 }
73 r = glfs_init(fs);
74 if (r) {
75 log_err("glfs_init failed. Is glusterd running on brick?\n");
76 goto out;
77 }
78 sleep(2);
79 r = glfs_lstat(fs, ".", &sb);
80 if (r) {
81 log_err("glfs_lstat failed.\n");
82 goto out;
83 }
84
85out:
86 if (r) {
87 glfs_fini(fs);
88 fs = NULL;
89 }
90 return fs;
91}
92
93static glfs_t *fio_gf_get_glfs(struct gf_options *opt,
94 char *volume, char *brick)
95{
96 struct glfs_info *glfs = NULL;
97 struct glfs_info *tmp;
98 struct flist_head *entry;
99
100 if (!opt->gf_single_instance)
101 return fio_gf_new_fs(volume, brick);
102
103 pthread_mutex_lock (&glfs_lock);
104
105 flist_for_each(entry, &glfs_list_head) {
106 tmp = flist_entry(entry, struct glfs_info, list);
107 if (!strcmp(volume, tmp->volume) &&
108 !strcmp(brick, tmp->brick)) {
109 glfs = tmp;
110 break;
111 }
112 }
113
114 if (glfs) {
115 glfs->refcount++;
116 } else {
117 glfs = malloc(sizeof(*glfs));
118 if (!glfs)
119 goto out;
120 INIT_FLIST_HEAD(&glfs->list);
121 glfs->refcount = 0;
122 glfs->volume = strdup(volume);
123 glfs->brick = strdup(brick);
124 glfs->fs = fio_gf_new_fs(volume, brick);
125 if (!glfs->fs) {
126 free(glfs);
127 glfs = NULL;
128 goto out;
129 }
130
131 flist_add_tail(&glfs->list, &glfs_list_head);
132 glfs->refcount = 1;
133 }
134
135out:
136 pthread_mutex_unlock (&glfs_lock);
137
138 if (glfs)
139 return glfs->fs;
140 return NULL;
141}
142
143static void fio_gf_put_glfs(struct gf_options *opt, glfs_t *fs)
144{
145 struct glfs_info *glfs = NULL;
146 struct glfs_info *tmp;
147 struct flist_head *entry;
148
149 if (!opt->gf_single_instance) {
150 glfs_fini(fs);
151 return;
152 }
153
154 pthread_mutex_lock (&glfs_lock);
155
156 flist_for_each(entry, &glfs_list_head) {
157 tmp = flist_entry(entry, struct glfs_info, list);
158 if (tmp->fs == fs) {
159 glfs = tmp;
160 break;
161 }
162 }
163
164 if (!glfs) {
165 log_err("glfs not found to fini.\n");
166 } else {
167 glfs->refcount--;
168
169 if (glfs->refcount == 0) {
170 glfs_fini(glfs->fs);
171 free(glfs->volume);
172 free(glfs->brick);
173 flist_del(&glfs->list);
174 }
175 }
176
177 pthread_mutex_unlock (&glfs_lock);
178}
179
180int fio_gf_setup(struct thread_data *td)
181{
6e7d7dfb 182 struct gf_data *g = NULL;
183 struct gf_options *opt = td->eo;
7cb57176 184
185 dprint(FD_IO, "fio setup\n");
6e7d7dfb 186
565e784d 187 if (td->io_ops_data)
b29c813f 188 return 0;
6e7d7dfb 189
190 g = malloc(sizeof(struct gf_data));
b29c813f
JA
191 if (!g) {
192 log_err("malloc failed.\n");
193 return -ENOMEM;
6e7d7dfb 194 }
b29c813f
JA
195 g->fd = NULL;
196 g->aio_events = NULL;
197
53c508dd
ZH
198 g->fs = fio_gf_get_glfs(opt, opt->gf_vol, opt->gf_brick);
199 if (!g->fs)
b29c813f 200 goto cleanup;
53c508dd 201
7cb57176 202 dprint(FD_FILE, "fio setup %p\n", g->fs);
565e784d 203 td->io_ops_data = g;
93782cbb 204 return 0;
6e7d7dfb 205cleanup:
93782cbb 206 free(g);
565e784d 207 td->io_ops_data = NULL;
53c508dd 208 return -EIO;
6e7d7dfb 209}
210
cc47f094 211void fio_gf_cleanup(struct thread_data *td)
6e7d7dfb 212{
565e784d 213 struct gf_data *g = td->io_ops_data;
cc47f094 214
215 if (g) {
b29c813f
JA
216 if (g->aio_events)
217 free(g->aio_events);
218 if (g->fd)
219 glfs_close(g->fd);
220 if (g->fs)
53c508dd 221 fio_gf_put_glfs(td->eo, g->fs);
b29c813f 222 free(g);
565e784d 223 td->io_ops_data = NULL;
cc47f094 224 }
6e7d7dfb 225}
226
cc47f094 227int fio_gf_get_file_size(struct thread_data *td, struct fio_file *f)
6e7d7dfb 228{
b29c813f
JA
229 struct stat buf;
230 int ret;
565e784d 231 struct gf_data *g = td->io_ops_data;
6e7d7dfb 232
b29c813f 233 dprint(FD_FILE, "get file size %s\n", f->file_name);
6aa56500 234
b29c813f
JA
235 if (!g || !g->fs) {
236 return 0;
237 }
238 if (fio_file_size_known(f))
239 return 0;
6e7d7dfb 240
b29c813f
JA
241 ret = glfs_lstat(g->fs, f->file_name, &buf);
242 if (ret < 0) {
243 log_err("glfs_lstat failed.\n");
244 return ret;
245 }
6e7d7dfb 246
b29c813f
JA
247 f->real_file_size = buf.st_size;
248 fio_file_set_size_known(f);
6e7d7dfb 249
b29c813f 250 return 0;
6e7d7dfb 251
252}
253
cc47f094 254int fio_gf_open_file(struct thread_data *td, struct fio_file *f)
6e7d7dfb 255{
a8a93dee 256
b29c813f
JA
257 int flags = 0;
258 int ret = 0;
565e784d 259 struct gf_data *g = td->io_ops_data;
b29c813f
JA
260 struct stat sb = { 0, };
261
262 if (td_write(td)) {
263 if (!read_only)
264 flags = O_RDWR;
265 } else if (td_read(td)) {
266 if (!read_only)
267 flags = O_RDWR;
268 else
269 flags = O_RDONLY;
270 }
65bea10f
TM
271
272 if (td->o.odirect)
273 flags |= OS_O_DIRECT;
eb9f8d7f 274 flags |= td->o.sync_io;
65bea10f 275
b29c813f 276 dprint(FD_FILE, "fio file %s open mode %s td rw %s\n", f->file_name,
65bea10f 277 flags & O_RDONLY ? "ro" : "rw", td_read(td) ? "read" : "write");
b29c813f
JA
278 g->fd = glfs_creat(g->fs, f->file_name, flags, 0644);
279 if (!g->fd) {
b29c813f 280 ret = errno;
1a889967
TM
281 log_err("glfs_creat failed.\n");
282 return ret;
b29c813f
JA
283 }
284 /* file for read doesn't exist or shorter than required, create/extend it */
285 if (td_read(td)) {
286 if (glfs_lstat(g->fs, f->file_name, &sb)
287 || sb.st_size < f->real_file_size) {
f6149216
SW
288 dprint(FD_FILE, "fio extend file %s from %jd to %" PRIu64 "\n",
289 f->file_name, (intmax_t) sb.st_size, f->real_file_size);
ce4d13ca
JA
290#if defined(CONFIG_GF_NEW_API)
291 ret = glfs_ftruncate(g->fd, f->real_file_size, NULL, NULL);
292#else
b29c813f 293 ret = glfs_ftruncate(g->fd, f->real_file_size);
ce4d13ca 294#endif
b29c813f 295 if (ret) {
f6149216 296 log_err("failed fio extend file %s to %" PRIu64 "\n",
b29c813f
JA
297 f->file_name, f->real_file_size);
298 } else {
299 unsigned long long left;
300 unsigned int bs;
301 char *b;
302 int r;
303
304 /* fill the file, copied from extend_file */
305 b = malloc(td->o.max_bs[DDIR_WRITE]);
306
307 left = f->real_file_size;
308 while (left && !td->terminate) {
309 bs = td->o.max_bs[DDIR_WRITE];
310 if (bs > left)
311 bs = left;
312
313 fill_io_buffer(td, b, bs, bs);
314
315 r = glfs_write(g->fd, b, bs, 0);
316 dprint(FD_IO,
f6149216 317 "fio write %d of %" PRIu64 " file %s\n",
b29c813f
JA
318 r, f->real_file_size,
319 f->file_name);
320
321 if (r > 0) {
322 left -= r;
323 continue;
324 } else {
325 if (r < 0) {
326 int __e = errno;
327
328 if (__e == ENOSPC) {
329 if (td->o.
330 fill_device)
331 break;
332 log_info
333 ("fio: ENOSPC on laying out "
334 "file, stopping\n");
335 break;
336 }
337 td_verror(td, errno,
338 "write");
339 } else
340 td_verror(td, EIO,
341 "write");
342
343 break;
344 }
345 }
346
347 if (b)
348 free(b);
349 glfs_lseek(g->fd, 0, SEEK_SET);
350
38ef9c90 351 if (td->terminate && td->o.unlink) {
b29c813f
JA
352 dprint(FD_FILE, "terminate unlink %s\n",
353 f->file_name);
38ef9c90 354 glfs_unlink(g->fs, f->file_name);
b29c813f 355 } else if (td->o.create_fsync) {
ce4d13ca
JA
356#if defined(CONFIG_GF_NEW_API)
357 if (glfs_fsync(g->fd, NULL, NULL) < 0) {
358#else
b29c813f 359 if (glfs_fsync(g->fd) < 0) {
ce4d13ca 360#endif
b29c813f
JA
361 dprint(FD_FILE,
362 "failed to sync, close %s\n",
363 f->file_name);
364 td_verror(td, errno, "fsync");
365 glfs_close(g->fd);
366 g->fd = NULL;
367 return 1;
368 }
369 }
370 }
371 }
372 }
6fa14b99 373#if defined(GFAPI_USE_FADVISE)
b29c813f
JA
374 {
375 int r = 0;
376 if (td_random(td)) {
377 r = glfs_fadvise(g->fd, 0, f->real_file_size,
378 POSIX_FADV_RANDOM);
379 } else {
380 r = glfs_fadvise(g->fd, 0, f->real_file_size,
381 POSIX_FADV_SEQUENTIAL);
382 }
383 if (r) {
384 dprint(FD_FILE, "fio %p fadvise %s status %d\n", g->fs,
385 f->file_name, r);
386 }
387 }
6fa14b99 388#endif
b29c813f
JA
389 dprint(FD_FILE, "fio %p created %s\n", g->fs, f->file_name);
390 f->fd = -1;
391 f->shadow_fd = -1;
e6590c12 392 td->o.open_files ++;
b29c813f 393 return ret;
6e7d7dfb 394}
395
cc47f094 396int fio_gf_close_file(struct thread_data *td, struct fio_file *f)
6e7d7dfb 397{
398 int ret = 0;
565e784d 399 struct gf_data *g = td->io_ops_data;
6e7d7dfb 400
401 dprint(FD_FILE, "fd close %s\n", f->file_name);
402
b29c813f
JA
403 if (g) {
404 if (g->fd && glfs_close(g->fd) < 0)
405 ret = errno;
38ef9c90
CF
406 g->fd = NULL;
407 }
408
409 return ret;
410}
411
412int fio_gf_unlink_file(struct thread_data *td, struct fio_file *f)
413{
414 int ret = 0;
565e784d 415 struct gf_data *g = td->io_ops_data;
38ef9c90
CF
416
417 dprint(FD_FILE, "fd unlink %s\n", f->file_name);
418
419 if (g) {
420 if (g->fd && glfs_close(g->fd) < 0)
421 ret = errno;
422
423 glfs_unlink(g->fs, f->file_name);
7cb57176 424
b29c813f
JA
425 if (g->fs)
426 glfs_fini(g->fs);
6e7d7dfb 427
b29c813f
JA
428 g->fd = NULL;
429 free(g);
430 }
565e784d 431 td->io_ops_data = NULL;
6e7d7dfb 432
433 return ret;
434}