Commit | Line | Data |
---|---|---|
c2f6a13d LMB |
1 | /* |
2 | * HTTP GET/PUT IO engine | |
3 | * | |
4 | * IO engine to perform HTTP(S) GET/PUT requests via libcurl-easy. | |
5 | * | |
6 | * Copyright (C) 2018 SUSE LLC | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU General Public License, | |
10 | * version 2 as published by the Free Software Foundation.. | |
11 | * | |
12 | * This program is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | * GNU General Public License for more details. | |
16 | * | |
17 | * You should have received a copy of the GNU General Public | |
18 | * License along with this program; if not, write to the Free | |
19 | * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, | |
20 | * Boston, MA 02110-1301, USA. | |
21 | */ | |
22 | ||
23 | #include <pthread.h> | |
24 | #include <time.h> | |
25 | #include <curl/curl.h> | |
26 | #include <openssl/hmac.h> | |
27 | #include <openssl/sha.h> | |
09fd2966 | 28 | #include <openssl/md5.h> |
c2f6a13d LMB |
29 | #include "fio.h" |
30 | #include "../optgroup.h" | |
31 | ||
32 | ||
09fd2966 LMB |
33 | enum { |
34 | FIO_HTTP_WEBDAV = 0, | |
35 | FIO_HTTP_S3 = 1, | |
36 | FIO_HTTP_SWIFT = 2, | |
37 | ||
38 | FIO_HTTPS_OFF = 0, | |
39 | FIO_HTTPS_ON = 1, | |
40 | FIO_HTTPS_INSECURE = 2, | |
41 | }; | |
42 | ||
c2f6a13d LMB |
43 | struct http_data { |
44 | CURL *curl; | |
45 | }; | |
46 | ||
47 | struct http_options { | |
48 | void *pad; | |
09fd2966 | 49 | unsigned int https; |
c2f6a13d LMB |
50 | char *host; |
51 | char *user; | |
52 | char *pass; | |
53 | char *s3_key; | |
54 | char *s3_keyid; | |
55 | char *s3_region; | |
09fd2966 | 56 | char *swift_auth_token; |
c2f6a13d | 57 | int verbose; |
09fd2966 | 58 | unsigned int mode; |
c2f6a13d LMB |
59 | }; |
60 | ||
61 | struct http_curl_stream { | |
62 | char *buf; | |
63 | size_t pos; | |
64 | size_t max; | |
65 | }; | |
66 | ||
67 | static struct fio_option options[] = { | |
68 | { | |
69 | .name = "https", | |
70 | .lname = "https", | |
09fd2966 | 71 | .type = FIO_OPT_STR, |
c2f6a13d LMB |
72 | .help = "Enable https", |
73 | .off1 = offsetof(struct http_options, https), | |
09fd2966 LMB |
74 | .def = "off", |
75 | .posval = { | |
76 | { .ival = "off", | |
77 | .oval = FIO_HTTPS_OFF, | |
78 | .help = "No HTTPS", | |
79 | }, | |
80 | { .ival = "on", | |
81 | .oval = FIO_HTTPS_ON, | |
82 | .help = "Enable HTTPS", | |
83 | }, | |
84 | { .ival = "insecure", | |
85 | .oval = FIO_HTTPS_INSECURE, | |
86 | .help = "Enable HTTPS, disable peer verification", | |
87 | }, | |
88 | }, | |
c2f6a13d LMB |
89 | .category = FIO_OPT_C_ENGINE, |
90 | .group = FIO_OPT_G_HTTP, | |
91 | }, | |
92 | { | |
93 | .name = "http_host", | |
94 | .lname = "http_host", | |
95 | .type = FIO_OPT_STR_STORE, | |
96 | .help = "Hostname (S3 bucket)", | |
97 | .off1 = offsetof(struct http_options, host), | |
98 | .def = "localhost", | |
99 | .category = FIO_OPT_C_ENGINE, | |
100 | .group = FIO_OPT_G_HTTP, | |
101 | }, | |
102 | { | |
103 | .name = "http_user", | |
104 | .lname = "http_user", | |
105 | .type = FIO_OPT_STR_STORE, | |
106 | .help = "HTTP user name", | |
107 | .off1 = offsetof(struct http_options, user), | |
108 | .category = FIO_OPT_C_ENGINE, | |
109 | .group = FIO_OPT_G_HTTP, | |
110 | }, | |
111 | { | |
112 | .name = "http_pass", | |
113 | .lname = "http_pass", | |
114 | .type = FIO_OPT_STR_STORE, | |
115 | .help = "HTTP password", | |
116 | .off1 = offsetof(struct http_options, pass), | |
117 | .category = FIO_OPT_C_ENGINE, | |
118 | .group = FIO_OPT_G_HTTP, | |
119 | }, | |
120 | { | |
121 | .name = "http_s3_key", | |
122 | .lname = "S3 secret key", | |
123 | .type = FIO_OPT_STR_STORE, | |
124 | .help = "S3 secret key", | |
125 | .off1 = offsetof(struct http_options, s3_key), | |
126 | .def = "", | |
127 | .category = FIO_OPT_C_ENGINE, | |
128 | .group = FIO_OPT_G_HTTP, | |
129 | }, | |
130 | { | |
131 | .name = "http_s3_keyid", | |
132 | .lname = "S3 key id", | |
133 | .type = FIO_OPT_STR_STORE, | |
134 | .help = "S3 key id", | |
135 | .off1 = offsetof(struct http_options, s3_keyid), | |
136 | .def = "", | |
137 | .category = FIO_OPT_C_ENGINE, | |
138 | .group = FIO_OPT_G_HTTP, | |
139 | }, | |
09fd2966 LMB |
140 | { |
141 | .name = "http_swift_auth_token", | |
142 | .lname = "Swift auth token", | |
143 | .type = FIO_OPT_STR_STORE, | |
144 | .help = "OpenStack Swift auth token", | |
145 | .off1 = offsetof(struct http_options, swift_auth_token), | |
146 | .def = "", | |
147 | .category = FIO_OPT_C_ENGINE, | |
148 | .group = FIO_OPT_G_HTTP, | |
149 | }, | |
c2f6a13d LMB |
150 | { |
151 | .name = "http_s3_region", | |
152 | .lname = "S3 region", | |
153 | .type = FIO_OPT_STR_STORE, | |
154 | .help = "S3 region", | |
155 | .off1 = offsetof(struct http_options, s3_region), | |
156 | .def = "us-east-1", | |
157 | .category = FIO_OPT_C_ENGINE, | |
158 | .group = FIO_OPT_G_HTTP, | |
159 | }, | |
160 | { | |
09fd2966 LMB |
161 | .name = "http_mode", |
162 | .lname = "Request mode to use", | |
163 | .type = FIO_OPT_STR, | |
164 | .help = "Whether to use WebDAV, Swift, or S3", | |
165 | .off1 = offsetof(struct http_options, mode), | |
166 | .def = "webdav", | |
167 | .posval = { | |
168 | { .ival = "webdav", | |
169 | .oval = FIO_HTTP_WEBDAV, | |
170 | .help = "WebDAV server", | |
171 | }, | |
172 | { .ival = "s3", | |
173 | .oval = FIO_HTTP_S3, | |
174 | .help = "S3 storage backend", | |
175 | }, | |
176 | { .ival = "swift", | |
177 | .oval = FIO_HTTP_SWIFT, | |
178 | .help = "OpenStack Swift storage", | |
179 | }, | |
180 | }, | |
c2f6a13d LMB |
181 | .category = FIO_OPT_C_ENGINE, |
182 | .group = FIO_OPT_G_HTTP, | |
183 | }, | |
184 | { | |
185 | .name = "http_verbose", | |
09fd2966 | 186 | .lname = "HTTP verbosity level", |
c2f6a13d LMB |
187 | .type = FIO_OPT_INT, |
188 | .help = "increase http engine verbosity", | |
189 | .off1 = offsetof(struct http_options, verbose), | |
190 | .def = "0", | |
191 | .category = FIO_OPT_C_ENGINE, | |
192 | .group = FIO_OPT_G_HTTP, | |
193 | }, | |
194 | { | |
195 | .name = NULL, | |
196 | }, | |
197 | }; | |
198 | ||
199 | static char *_aws_uriencode(const char *uri) | |
200 | { | |
201 | size_t bufsize = 1024; | |
202 | char *r = malloc(bufsize); | |
203 | char c; | |
204 | int i, n; | |
205 | const char *hex = "0123456789ABCDEF"; | |
206 | ||
207 | if (!r) { | |
208 | log_err("malloc failed\n"); | |
209 | return NULL; | |
210 | } | |
211 | ||
212 | n = 0; | |
213 | for (i = 0; (c = uri[i]); i++) { | |
214 | if (n > bufsize-5) { | |
215 | log_err("encoding the URL failed\n"); | |
216 | return NULL; | |
217 | } | |
218 | ||
219 | if ( (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') | |
220 | || (c >= '0' && c <= '9') || c == '_' || c == '-' | |
221 | || c == '~' || c == '.' || c == '/') | |
222 | r[n++] = c; | |
223 | else { | |
224 | r[n++] = '%'; | |
225 | r[n++] = hex[(c >> 4 ) & 0xF]; | |
226 | r[n++] = hex[c & 0xF]; | |
227 | } | |
228 | } | |
229 | r[n++] = 0; | |
230 | return r; | |
231 | } | |
232 | ||
233 | static char *_conv_hex(const unsigned char *p, size_t len) | |
234 | { | |
235 | char *r; | |
236 | int i,n; | |
237 | const char *hex = "0123456789abcdef"; | |
238 | r = malloc(len * 2 + 1); | |
239 | n = 0; | |
240 | for (i = 0; i < len; i++) { | |
241 | r[n++] = hex[(p[i] >> 4 ) & 0xF]; | |
242 | r[n++] = hex[p[i] & 0xF]; | |
243 | } | |
244 | r[n] = 0; | |
245 | ||
246 | return r; | |
247 | } | |
248 | ||
249 | static char *_gen_hex_sha256(const char *p, size_t len) | |
250 | { | |
251 | unsigned char hash[SHA256_DIGEST_LENGTH]; | |
252 | ||
253 | SHA256((unsigned char*)p, len, hash); | |
254 | return _conv_hex(hash, SHA256_DIGEST_LENGTH); | |
255 | } | |
256 | ||
09fd2966 LMB |
257 | static char *_gen_hex_md5(const char *p, size_t len) |
258 | { | |
259 | unsigned char hash[MD5_DIGEST_LENGTH]; | |
260 | ||
261 | MD5((unsigned char*)p, len, hash); | |
262 | return _conv_hex(hash, MD5_DIGEST_LENGTH); | |
263 | } | |
264 | ||
c2f6a13d | 265 | static void _hmac(unsigned char *md, void *key, int key_len, char *data) { |
b61a5f46 DD |
266 | #ifndef CONFIG_HAVE_OPAQUE_HMAC_CTX |
267 | HMAC_CTX _ctx; | |
268 | #endif | |
c2f6a13d LMB |
269 | HMAC_CTX *ctx; |
270 | unsigned int hmac_len; | |
271 | ||
b61a5f46 | 272 | #ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX |
c2f6a13d | 273 | ctx = HMAC_CTX_new(); |
b61a5f46 DD |
274 | #else |
275 | ctx = &_ctx; | |
e8aaa776 JA |
276 | /* work-around crash in certain versions of libssl */ |
277 | HMAC_CTX_init(ctx); | |
b61a5f46 | 278 | #endif |
c2f6a13d LMB |
279 | HMAC_Init_ex(ctx, key, key_len, EVP_sha256(), NULL); |
280 | HMAC_Update(ctx, (unsigned char*)data, strlen(data)); | |
281 | HMAC_Final(ctx, md, &hmac_len); | |
b61a5f46 | 282 | #ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX |
c2f6a13d | 283 | HMAC_CTX_free(ctx); |
b61a5f46 DD |
284 | #else |
285 | HMAC_CTX_cleanup(ctx); | |
286 | #endif | |
c2f6a13d LMB |
287 | } |
288 | ||
289 | static int _curl_trace(CURL *handle, curl_infotype type, | |
290 | char *data, size_t size, | |
291 | void *userp) | |
292 | { | |
293 | const char *text; | |
294 | (void)handle; /* prevent compiler warning */ | |
295 | (void)userp; | |
296 | ||
297 | switch (type) { | |
298 | case CURLINFO_TEXT: | |
24010223 JA |
299 | fprintf(stderr, "== Info: %s", data); |
300 | /* fall through */ | |
c2f6a13d LMB |
301 | default: |
302 | case CURLINFO_SSL_DATA_OUT: | |
24010223 | 303 | /* fall through */ |
c2f6a13d LMB |
304 | case CURLINFO_SSL_DATA_IN: |
305 | return 0; | |
306 | ||
307 | case CURLINFO_HEADER_OUT: | |
308 | text = "=> Send header"; | |
309 | break; | |
310 | case CURLINFO_DATA_OUT: | |
311 | text = "=> Send data"; | |
312 | break; | |
313 | case CURLINFO_HEADER_IN: | |
314 | text = "<= Recv header"; | |
315 | break; | |
316 | case CURLINFO_DATA_IN: | |
317 | text = "<= Recv data"; | |
318 | break; | |
319 | } | |
320 | ||
321 | log_info("%s: %s", text, data); | |
322 | return 0; | |
323 | } | |
324 | ||
325 | /* https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html | |
326 | * https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro | |
327 | */ | |
328 | static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct http_options *o, | |
329 | int op, const char *uri, char *buf, size_t len) | |
330 | { | |
331 | char date_short[16]; | |
332 | char date_iso[32]; | |
333 | char method[8]; | |
334 | char dkey[128]; | |
335 | char creq[512]; | |
336 | char sts[256]; | |
337 | char s[512]; | |
338 | char *uri_encoded = NULL; | |
339 | char *dsha = NULL; | |
340 | char *csha = NULL; | |
341 | char *signature = NULL; | |
342 | const char *service = "s3"; | |
343 | const char *aws = "aws4_request"; | |
344 | unsigned char md[SHA256_DIGEST_LENGTH]; | |
345 | ||
346 | time_t t = time(NULL); | |
347 | struct tm *gtm = gmtime(&t); | |
348 | ||
349 | strftime (date_short, sizeof(date_short), "%Y%m%d", gtm); | |
350 | strftime (date_iso, sizeof(date_iso), "%Y%m%dT%H%M%SZ", gtm); | |
351 | uri_encoded = _aws_uriencode(uri); | |
352 | ||
353 | if (op == DDIR_WRITE) { | |
354 | dsha = _gen_hex_sha256(buf, len); | |
355 | sprintf(method, "PUT"); | |
356 | } else { | |
357 | /* DDIR_READ && DDIR_TRIM supply an empty body */ | |
358 | if (op == DDIR_READ) | |
359 | sprintf(method, "GET"); | |
360 | else | |
361 | sprintf(method, "DELETE"); | |
362 | dsha = _gen_hex_sha256("", 0); | |
363 | } | |
364 | ||
365 | /* Create the canonical request first */ | |
366 | snprintf(creq, sizeof(creq), | |
367 | "%s\n" | |
368 | "%s\n" | |
369 | "\n" | |
370 | "host:%s\n" | |
371 | "x-amz-content-sha256:%s\n" | |
372 | "x-amz-date:%s\n" | |
373 | "\n" | |
374 | "host;x-amz-content-sha256;x-amz-date\n" | |
375 | "%s" | |
376 | , method | |
377 | , uri_encoded, o->host, dsha, date_iso, dsha); | |
378 | ||
379 | csha = _gen_hex_sha256(creq, strlen(creq)); | |
380 | snprintf(sts, sizeof(sts), "AWS4-HMAC-SHA256\n%s\n%s/%s/%s/%s\n%s", | |
381 | date_iso, date_short, o->s3_region, service, aws, csha); | |
382 | ||
383 | snprintf((char *)dkey, sizeof(dkey), "AWS4%s", o->s3_key); | |
384 | _hmac(md, dkey, strlen(dkey), date_short); | |
385 | _hmac(md, md, SHA256_DIGEST_LENGTH, o->s3_region); | |
386 | _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) service); | |
387 | _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) aws); | |
388 | _hmac(md, md, SHA256_DIGEST_LENGTH, sts); | |
389 | ||
390 | signature = _conv_hex(md, SHA256_DIGEST_LENGTH); | |
391 | ||
392 | /* Surpress automatic Accept: header */ | |
393 | slist = curl_slist_append(slist, "Accept:"); | |
394 | ||
395 | snprintf(s, sizeof(s), "x-amz-content-sha256: %s", dsha); | |
396 | slist = curl_slist_append(slist, s); | |
397 | ||
398 | snprintf(s, sizeof(s), "x-amz-date: %s", date_iso); | |
399 | slist = curl_slist_append(slist, s); | |
400 | ||
401 | snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request," | |
402 | "SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=%s", | |
403 | o->s3_keyid, date_short, o->s3_region, signature); | |
404 | slist = curl_slist_append(slist, s); | |
405 | ||
406 | curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); | |
407 | ||
408 | free(uri_encoded); | |
409 | free(csha); | |
410 | free(dsha); | |
411 | free(signature); | |
412 | } | |
413 | ||
09fd2966 LMB |
414 | static void _add_swift_header(CURL *curl, struct curl_slist *slist, struct http_options *o, |
415 | int op, const char *uri, char *buf, size_t len) | |
416 | { | |
417 | char *dsha = NULL; | |
418 | char s[512]; | |
419 | ||
420 | if (op == DDIR_WRITE) { | |
421 | dsha = _gen_hex_md5(buf, len); | |
422 | } | |
423 | /* Surpress automatic Accept: header */ | |
424 | slist = curl_slist_append(slist, "Accept:"); | |
425 | ||
426 | snprintf(s, sizeof(s), "etag: %s", dsha); | |
427 | slist = curl_slist_append(slist, s); | |
428 | ||
429 | snprintf(s, sizeof(s), "x-auth-token: %s", o->swift_auth_token); | |
430 | slist = curl_slist_append(slist, s); | |
431 | ||
432 | curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); | |
433 | ||
434 | free(dsha); | |
435 | } | |
436 | ||
c2f6a13d LMB |
437 | static void fio_http_cleanup(struct thread_data *td) |
438 | { | |
439 | struct http_data *http = td->io_ops_data; | |
440 | ||
441 | if (http) { | |
442 | curl_easy_cleanup(http->curl); | |
443 | free(http); | |
444 | } | |
445 | } | |
446 | ||
447 | static size_t _http_read(void *ptr, size_t size, size_t nmemb, void *stream) | |
448 | { | |
449 | struct http_curl_stream *state = stream; | |
450 | size_t len = size * nmemb; | |
451 | /* We're retrieving; nothing is supposed to be read locally */ | |
452 | if (!stream) | |
453 | return 0; | |
454 | if (len+state->pos > state->max) | |
455 | len = state->max - state->pos; | |
456 | memcpy(ptr, &state->buf[state->pos], len); | |
457 | state->pos += len; | |
458 | return len; | |
459 | } | |
460 | ||
461 | static size_t _http_write(void *ptr, size_t size, size_t nmemb, void *stream) | |
462 | { | |
463 | struct http_curl_stream *state = stream; | |
464 | /* We're just discarding the returned body after a PUT */ | |
465 | if (!stream) | |
466 | return nmemb; | |
467 | if (size != 1) | |
468 | return CURLE_WRITE_ERROR; | |
469 | if (nmemb + state->pos > state->max) | |
470 | return CURLE_WRITE_ERROR; | |
471 | memcpy(&state->buf[state->pos], ptr, nmemb); | |
472 | state->pos += nmemb; | |
473 | return nmemb; | |
474 | } | |
475 | ||
476 | static int _http_seek(void *stream, curl_off_t offset, int origin) | |
477 | { | |
478 | struct http_curl_stream *state = stream; | |
479 | if (offset < state->max && origin == SEEK_SET) { | |
480 | state->pos = offset; | |
481 | return CURL_SEEKFUNC_OK; | |
482 | } else | |
483 | return CURL_SEEKFUNC_FAIL; | |
484 | } | |
485 | ||
486 | static enum fio_q_status fio_http_queue(struct thread_data *td, | |
487 | struct io_u *io_u) | |
488 | { | |
489 | struct http_data *http = td->io_ops_data; | |
490 | struct http_options *o = td->eo; | |
491 | struct http_curl_stream _curl_stream; | |
492 | struct curl_slist *slist = NULL; | |
493 | char object[512]; | |
494 | char url[1024]; | |
495 | long status; | |
496 | CURLcode res; | |
497 | int r = -1; | |
498 | ||
499 | fio_ro_check(td, io_u); | |
500 | memset(&_curl_stream, 0, sizeof(_curl_stream)); | |
09fd2966 LMB |
501 | snprintf(object, sizeof(object), "%s_%llu_%llu", td->files[0]->file_name, |
502 | io_u->offset, io_u->xfer_buflen); | |
503 | if (o->https == FIO_HTTPS_OFF) | |
504 | snprintf(url, sizeof(url), "http://%s%s", o->host, object); | |
505 | else | |
506 | snprintf(url, sizeof(url), "https://%s%s", o->host, object); | |
c2f6a13d LMB |
507 | curl_easy_setopt(http->curl, CURLOPT_URL, url); |
508 | _curl_stream.buf = io_u->xfer_buf; | |
509 | _curl_stream.max = io_u->xfer_buflen; | |
510 | curl_easy_setopt(http->curl, CURLOPT_SEEKDATA, &_curl_stream); | |
511 | curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)io_u->xfer_buflen); | |
512 | ||
09fd2966 | 513 | if (o->mode == FIO_HTTP_S3) |
c2f6a13d LMB |
514 | _add_aws_auth_header(http->curl, slist, o, io_u->ddir, object, |
515 | io_u->xfer_buf, io_u->xfer_buflen); | |
09fd2966 LMB |
516 | else if (o->mode == FIO_HTTP_SWIFT) |
517 | _add_swift_header(http->curl, slist, o, io_u->ddir, object, | |
518 | io_u->xfer_buf, io_u->xfer_buflen); | |
c2f6a13d LMB |
519 | |
520 | if (io_u->ddir == DDIR_WRITE) { | |
521 | curl_easy_setopt(http->curl, CURLOPT_READDATA, &_curl_stream); | |
522 | curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL); | |
523 | curl_easy_setopt(http->curl, CURLOPT_UPLOAD, 1L); | |
524 | res = curl_easy_perform(http->curl); | |
525 | if (res == CURLE_OK) { | |
526 | curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status); | |
527 | if (status == 100 || (status >= 200 && status <= 204)) | |
528 | goto out; | |
529 | log_err("DDIR_WRITE failed with HTTP status code %ld\n", status); | |
530 | goto err; | |
531 | } | |
532 | } else if (io_u->ddir == DDIR_READ) { | |
533 | curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL); | |
534 | curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, &_curl_stream); | |
535 | curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L); | |
536 | res = curl_easy_perform(http->curl); | |
537 | if (res == CURLE_OK) { | |
538 | curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status); | |
539 | if (status == 200) | |
540 | goto out; | |
541 | else if (status == 404) { | |
542 | /* Object doesn't exist. Pretend we read | |
543 | * zeroes */ | |
544 | memset(io_u->xfer_buf, 0, io_u->xfer_buflen); | |
545 | goto out; | |
546 | } | |
547 | log_err("DDIR_READ failed with HTTP status code %ld\n", status); | |
548 | } | |
549 | goto err; | |
550 | } else if (io_u->ddir == DDIR_TRIM) { | |
551 | curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L); | |
552 | curl_easy_setopt(http->curl, CURLOPT_CUSTOMREQUEST, "DELETE"); | |
bd915a6b | 553 | curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)0); |
c2f6a13d LMB |
554 | curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL); |
555 | curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL); | |
556 | res = curl_easy_perform(http->curl); | |
557 | if (res == CURLE_OK) { | |
558 | curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status); | |
559 | if (status == 200 || status == 202 || status == 204 || status == 404) | |
560 | goto out; | |
561 | log_err("DDIR_TRIM failed with HTTP status code %ld\n", status); | |
562 | } | |
563 | goto err; | |
564 | } | |
565 | ||
566 | log_err("WARNING: Only DDIR_READ/DDIR_WRITE/DDIR_TRIM are supported!\n"); | |
567 | ||
568 | err: | |
569 | io_u->error = r; | |
570 | td_verror(td, io_u->error, "transfer"); | |
571 | out: | |
572 | curl_slist_free_all(slist); | |
573 | return FIO_Q_COMPLETED; | |
574 | } | |
575 | ||
576 | static struct io_u *fio_http_event(struct thread_data *td, int event) | |
577 | { | |
578 | /* sync IO engine - never any outstanding events */ | |
579 | return NULL; | |
580 | } | |
581 | ||
582 | int fio_http_getevents(struct thread_data *td, unsigned int min, | |
583 | unsigned int max, const struct timespec *t) | |
584 | { | |
585 | /* sync IO engine - never any outstanding events */ | |
586 | return 0; | |
587 | } | |
588 | ||
589 | static int fio_http_setup(struct thread_data *td) | |
590 | { | |
591 | struct http_data *http = NULL; | |
592 | struct http_options *o = td->eo; | |
c65f3435 | 593 | |
c2f6a13d LMB |
594 | /* allocate engine specific structure to deal with libhttp. */ |
595 | http = calloc(1, sizeof(*http)); | |
596 | if (!http) { | |
597 | log_err("calloc failed.\n"); | |
598 | goto cleanup; | |
599 | } | |
600 | ||
601 | http->curl = curl_easy_init(); | |
602 | if (o->verbose) | |
603 | curl_easy_setopt(http->curl, CURLOPT_VERBOSE, 1L); | |
604 | if (o->verbose > 1) | |
605 | curl_easy_setopt(http->curl, CURLOPT_DEBUGFUNCTION, &_curl_trace); | |
606 | curl_easy_setopt(http->curl, CURLOPT_NOPROGRESS, 1L); | |
607 | curl_easy_setopt(http->curl, CURLOPT_FOLLOWLOCATION, 1L); | |
608 | curl_easy_setopt(http->curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP|CURLPROTO_HTTPS); | |
09fd2966 LMB |
609 | if (o->https == FIO_HTTPS_INSECURE) { |
610 | curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYPEER, 0L); | |
611 | curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYHOST, 0L); | |
612 | } | |
c2f6a13d LMB |
613 | curl_easy_setopt(http->curl, CURLOPT_READFUNCTION, _http_read); |
614 | curl_easy_setopt(http->curl, CURLOPT_WRITEFUNCTION, _http_write); | |
bd915a6b | 615 | curl_easy_setopt(http->curl, CURLOPT_SEEKFUNCTION, &_http_seek); |
c2f6a13d LMB |
616 | if (o->user && o->pass) { |
617 | curl_easy_setopt(http->curl, CURLOPT_USERNAME, o->user); | |
618 | curl_easy_setopt(http->curl, CURLOPT_PASSWORD, o->pass); | |
619 | curl_easy_setopt(http->curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY); | |
620 | } | |
621 | ||
622 | td->io_ops_data = http; | |
623 | ||
624 | /* Force single process mode. */ | |
625 | td->o.use_thread = 1; | |
626 | ||
627 | return 0; | |
628 | cleanup: | |
629 | fio_http_cleanup(td); | |
c65f3435 | 630 | return 1; |
c2f6a13d LMB |
631 | } |
632 | ||
633 | static int fio_http_open(struct thread_data *td, struct fio_file *f) | |
634 | { | |
635 | return 0; | |
636 | } | |
637 | static int fio_http_invalidate(struct thread_data *td, struct fio_file *f) | |
638 | { | |
639 | return 0; | |
640 | } | |
641 | ||
5a8a6a03 | 642 | FIO_STATIC struct ioengine_ops ioengine = { |
c2f6a13d LMB |
643 | .name = "http", |
644 | .version = FIO_IOOPS_VERSION, | |
f32a30d4 | 645 | .flags = FIO_DISKLESSIO | FIO_SYNCIO, |
c2f6a13d LMB |
646 | .setup = fio_http_setup, |
647 | .queue = fio_http_queue, | |
648 | .getevents = fio_http_getevents, | |
649 | .event = fio_http_event, | |
650 | .cleanup = fio_http_cleanup, | |
651 | .open_file = fio_http_open, | |
652 | .invalidate = fio_http_invalidate, | |
653 | .options = options, | |
654 | .option_struct_size = sizeof(struct http_options), | |
655 | }; | |
656 | ||
657 | static void fio_init fio_http_register(void) | |
658 | { | |
659 | register_ioengine(&ioengine); | |
660 | } | |
661 | ||
662 | static void fio_exit fio_http_unregister(void) | |
663 | { | |
664 | unregister_ioengine(&ioengine); | |
665 | } |