2 * HTTP GET/PUT IO engine
4 * IO engine to perform HTTP(S) GET/PUT requests via libcurl-easy.
6 * Copyright (C) 2018 SUSE LLC
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License,
10 * version 2 as published by the Free Software Foundation..
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public
18 * License along with this program; if not, write to the Free
19 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
20 * Boston, MA 02110-1301, USA.
25 #include <curl/curl.h>
26 #include <openssl/hmac.h>
27 #include <openssl/sha.h>
28 #include <openssl/md5.h>
30 #include "../optgroup.h"
40 FIO_HTTPS_INSECURE = 2,
56 char *swift_auth_token;
61 struct http_curl_stream {
67 static struct fio_option options[] = {
72 .help = "Enable https",
73 .off1 = offsetof(struct http_options, https),
77 .oval = FIO_HTTPS_OFF,
82 .help = "Enable HTTPS",
85 .oval = FIO_HTTPS_INSECURE,
86 .help = "Enable HTTPS, disable peer verification",
89 .category = FIO_OPT_C_ENGINE,
90 .group = FIO_OPT_G_HTTP,
95 .type = FIO_OPT_STR_STORE,
96 .help = "Hostname (S3 bucket)",
97 .off1 = offsetof(struct http_options, host),
99 .category = FIO_OPT_C_ENGINE,
100 .group = FIO_OPT_G_HTTP,
104 .lname = "http_user",
105 .type = FIO_OPT_STR_STORE,
106 .help = "HTTP user name",
107 .off1 = offsetof(struct http_options, user),
108 .category = FIO_OPT_C_ENGINE,
109 .group = FIO_OPT_G_HTTP,
113 .lname = "http_pass",
114 .type = FIO_OPT_STR_STORE,
115 .help = "HTTP password",
116 .off1 = offsetof(struct http_options, pass),
117 .category = FIO_OPT_C_ENGINE,
118 .group = FIO_OPT_G_HTTP,
121 .name = "http_s3_key",
122 .lname = "S3 secret key",
123 .type = FIO_OPT_STR_STORE,
124 .help = "S3 secret key",
125 .off1 = offsetof(struct http_options, s3_key),
127 .category = FIO_OPT_C_ENGINE,
128 .group = FIO_OPT_G_HTTP,
131 .name = "http_s3_keyid",
132 .lname = "S3 key id",
133 .type = FIO_OPT_STR_STORE,
135 .off1 = offsetof(struct http_options, s3_keyid),
137 .category = FIO_OPT_C_ENGINE,
138 .group = FIO_OPT_G_HTTP,
141 .name = "http_swift_auth_token",
142 .lname = "Swift auth token",
143 .type = FIO_OPT_STR_STORE,
144 .help = "OpenStack Swift auth token",
145 .off1 = offsetof(struct http_options, swift_auth_token),
147 .category = FIO_OPT_C_ENGINE,
148 .group = FIO_OPT_G_HTTP,
151 .name = "http_s3_region",
152 .lname = "S3 region",
153 .type = FIO_OPT_STR_STORE,
155 .off1 = offsetof(struct http_options, s3_region),
157 .category = FIO_OPT_C_ENGINE,
158 .group = FIO_OPT_G_HTTP,
162 .lname = "Request mode to use",
164 .help = "Whether to use WebDAV, Swift, or S3",
165 .off1 = offsetof(struct http_options, mode),
169 .oval = FIO_HTTP_WEBDAV,
170 .help = "WebDAV server",
174 .help = "S3 storage backend",
177 .oval = FIO_HTTP_SWIFT,
178 .help = "OpenStack Swift storage",
181 .category = FIO_OPT_C_ENGINE,
182 .group = FIO_OPT_G_HTTP,
185 .name = "http_verbose",
186 .lname = "HTTP verbosity level",
188 .help = "increase http engine verbosity",
189 .off1 = offsetof(struct http_options, verbose),
191 .category = FIO_OPT_C_ENGINE,
192 .group = FIO_OPT_G_HTTP,
199 static char *_aws_uriencode(const char *uri)
201 size_t bufsize = 1024;
202 char *r = malloc(bufsize);
205 const char *hex = "0123456789ABCDEF";
208 log_err("malloc failed\n");
213 for (i = 0; (c = uri[i]); i++) {
215 log_err("encoding the URL failed\n");
219 if ( (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
220 || (c >= '0' && c <= '9') || c == '_' || c == '-'
221 || c == '~' || c == '.' || c == '/')
225 r[n++] = hex[(c >> 4 ) & 0xF];
226 r[n++] = hex[c & 0xF];
233 static char *_conv_hex(const unsigned char *p, size_t len)
237 const char *hex = "0123456789abcdef";
238 r = malloc(len * 2 + 1);
240 for (i = 0; i < len; i++) {
241 r[n++] = hex[(p[i] >> 4 ) & 0xF];
242 r[n++] = hex[p[i] & 0xF];
249 static char *_gen_hex_sha256(const char *p, size_t len)
251 unsigned char hash[SHA256_DIGEST_LENGTH];
253 SHA256((unsigned char*)p, len, hash);
254 return _conv_hex(hash, SHA256_DIGEST_LENGTH);
257 static char *_gen_hex_md5(const char *p, size_t len)
259 unsigned char hash[MD5_DIGEST_LENGTH];
261 MD5((unsigned char*)p, len, hash);
262 return _conv_hex(hash, MD5_DIGEST_LENGTH);
265 static void _hmac(unsigned char *md, void *key, int key_len, char *data) {
266 #ifndef CONFIG_HAVE_OPAQUE_HMAC_CTX
270 unsigned int hmac_len;
272 #ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX
273 ctx = HMAC_CTX_new();
276 /* work-around crash in certain versions of libssl */
279 HMAC_Init_ex(ctx, key, key_len, EVP_sha256(), NULL);
280 HMAC_Update(ctx, (unsigned char*)data, strlen(data));
281 HMAC_Final(ctx, md, &hmac_len);
282 #ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX
285 HMAC_CTX_cleanup(ctx);
289 static int _curl_trace(CURL *handle, curl_infotype type,
290 char *data, size_t size,
294 (void)handle; /* prevent compiler warning */
299 fprintf(stderr, "== Info: %s", data);
302 case CURLINFO_SSL_DATA_OUT:
303 case CURLINFO_SSL_DATA_IN:
306 case CURLINFO_HEADER_OUT:
307 text = "=> Send header";
309 case CURLINFO_DATA_OUT:
310 text = "=> Send data";
312 case CURLINFO_HEADER_IN:
313 text = "<= Recv header";
315 case CURLINFO_DATA_IN:
316 text = "<= Recv data";
320 log_info("%s: %s", text, data);
324 /* https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
325 * https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
327 static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct http_options *o,
328 int op, const char *uri, char *buf, size_t len)
337 char *uri_encoded = NULL;
340 char *signature = NULL;
341 const char *service = "s3";
342 const char *aws = "aws4_request";
343 unsigned char md[SHA256_DIGEST_LENGTH];
345 time_t t = time(NULL);
346 struct tm *gtm = gmtime(&t);
348 strftime (date_short, sizeof(date_short), "%Y%m%d", gtm);
349 strftime (date_iso, sizeof(date_iso), "%Y%m%dT%H%M%SZ", gtm);
350 uri_encoded = _aws_uriencode(uri);
352 if (op == DDIR_WRITE) {
353 dsha = _gen_hex_sha256(buf, len);
354 sprintf(method, "PUT");
356 /* DDIR_READ && DDIR_TRIM supply an empty body */
358 sprintf(method, "GET");
360 sprintf(method, "DELETE");
361 dsha = _gen_hex_sha256("", 0);
364 /* Create the canonical request first */
365 snprintf(creq, sizeof(creq),
370 "x-amz-content-sha256:%s\n"
373 "host;x-amz-content-sha256;x-amz-date\n"
376 , uri_encoded, o->host, dsha, date_iso, dsha);
378 csha = _gen_hex_sha256(creq, strlen(creq));
379 snprintf(sts, sizeof(sts), "AWS4-HMAC-SHA256\n%s\n%s/%s/%s/%s\n%s",
380 date_iso, date_short, o->s3_region, service, aws, csha);
382 snprintf((char *)dkey, sizeof(dkey), "AWS4%s", o->s3_key);
383 _hmac(md, dkey, strlen(dkey), date_short);
384 _hmac(md, md, SHA256_DIGEST_LENGTH, o->s3_region);
385 _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) service);
386 _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) aws);
387 _hmac(md, md, SHA256_DIGEST_LENGTH, sts);
389 signature = _conv_hex(md, SHA256_DIGEST_LENGTH);
391 /* Suppress automatic Accept: header */
392 slist = curl_slist_append(slist, "Accept:");
394 snprintf(s, sizeof(s), "x-amz-content-sha256: %s", dsha);
395 slist = curl_slist_append(slist, s);
397 snprintf(s, sizeof(s), "x-amz-date: %s", date_iso);
398 slist = curl_slist_append(slist, s);
400 snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
401 "SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=%s",
402 o->s3_keyid, date_short, o->s3_region, signature);
403 slist = curl_slist_append(slist, s);
405 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
413 static void _add_swift_header(CURL *curl, struct curl_slist *slist, struct http_options *o,
414 int op, const char *uri, char *buf, size_t len)
419 if (op == DDIR_WRITE) {
420 dsha = _gen_hex_md5(buf, len);
422 /* Suppress automatic Accept: header */
423 slist = curl_slist_append(slist, "Accept:");
425 snprintf(s, sizeof(s), "etag: %s", dsha);
426 slist = curl_slist_append(slist, s);
428 snprintf(s, sizeof(s), "x-auth-token: %s", o->swift_auth_token);
429 slist = curl_slist_append(slist, s);
431 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
436 static void fio_http_cleanup(struct thread_data *td)
438 struct http_data *http = td->io_ops_data;
441 curl_easy_cleanup(http->curl);
446 static size_t _http_read(void *ptr, size_t size, size_t nmemb, void *stream)
448 struct http_curl_stream *state = stream;
449 size_t len = size * nmemb;
450 /* We're retrieving; nothing is supposed to be read locally */
453 if (len+state->pos > state->max)
454 len = state->max - state->pos;
455 memcpy(ptr, &state->buf[state->pos], len);
460 static size_t _http_write(void *ptr, size_t size, size_t nmemb, void *stream)
462 struct http_curl_stream *state = stream;
463 /* We're just discarding the returned body after a PUT */
467 return CURLE_WRITE_ERROR;
468 if (nmemb + state->pos > state->max)
469 return CURLE_WRITE_ERROR;
470 memcpy(&state->buf[state->pos], ptr, nmemb);
475 static int _http_seek(void *stream, curl_off_t offset, int origin)
477 struct http_curl_stream *state = stream;
478 if (offset < state->max && origin == SEEK_SET) {
480 return CURL_SEEKFUNC_OK;
482 return CURL_SEEKFUNC_FAIL;
485 static enum fio_q_status fio_http_queue(struct thread_data *td,
488 struct http_data *http = td->io_ops_data;
489 struct http_options *o = td->eo;
490 struct http_curl_stream _curl_stream;
491 struct curl_slist *slist = NULL;
498 fio_ro_check(td, io_u);
499 memset(&_curl_stream, 0, sizeof(_curl_stream));
500 snprintf(object, sizeof(object), "%s_%llu_%llu", td->files[0]->file_name,
501 io_u->offset, io_u->xfer_buflen);
502 if (o->https == FIO_HTTPS_OFF)
503 snprintf(url, sizeof(url), "http://%s%s", o->host, object);
505 snprintf(url, sizeof(url), "https://%s%s", o->host, object);
506 curl_easy_setopt(http->curl, CURLOPT_URL, url);
507 _curl_stream.buf = io_u->xfer_buf;
508 _curl_stream.max = io_u->xfer_buflen;
509 curl_easy_setopt(http->curl, CURLOPT_SEEKDATA, &_curl_stream);
510 curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)io_u->xfer_buflen);
512 if (o->mode == FIO_HTTP_S3)
513 _add_aws_auth_header(http->curl, slist, o, io_u->ddir, object,
514 io_u->xfer_buf, io_u->xfer_buflen);
515 else if (o->mode == FIO_HTTP_SWIFT)
516 _add_swift_header(http->curl, slist, o, io_u->ddir, object,
517 io_u->xfer_buf, io_u->xfer_buflen);
519 if (io_u->ddir == DDIR_WRITE) {
520 curl_easy_setopt(http->curl, CURLOPT_READDATA, &_curl_stream);
521 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
522 curl_easy_setopt(http->curl, CURLOPT_UPLOAD, 1L);
523 res = curl_easy_perform(http->curl);
524 if (res == CURLE_OK) {
525 curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
526 if (status == 100 || (status >= 200 && status <= 204))
528 log_err("DDIR_WRITE failed with HTTP status code %ld\n", status);
531 } else if (io_u->ddir == DDIR_READ) {
532 curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
533 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, &_curl_stream);
534 curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
535 res = curl_easy_perform(http->curl);
536 if (res == CURLE_OK) {
537 curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
540 else if (status == 404) {
541 /* Object doesn't exist. Pretend we read
543 memset(io_u->xfer_buf, 0, io_u->xfer_buflen);
546 log_err("DDIR_READ failed with HTTP status code %ld\n", status);
549 } else if (io_u->ddir == DDIR_TRIM) {
550 curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
551 curl_easy_setopt(http->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
552 curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)0);
553 curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
554 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
555 res = curl_easy_perform(http->curl);
556 if (res == CURLE_OK) {
557 curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
558 if (status == 200 || status == 202 || status == 204 || status == 404)
560 log_err("DDIR_TRIM failed with HTTP status code %ld\n", status);
565 log_err("WARNING: Only DDIR_READ/DDIR_WRITE/DDIR_TRIM are supported!\n");
569 td_verror(td, io_u->error, "transfer");
571 curl_slist_free_all(slist);
572 return FIO_Q_COMPLETED;
575 static struct io_u *fio_http_event(struct thread_data *td, int event)
577 /* sync IO engine - never any outstanding events */
581 int fio_http_getevents(struct thread_data *td, unsigned int min,
582 unsigned int max, const struct timespec *t)
584 /* sync IO engine - never any outstanding events */
588 static int fio_http_setup(struct thread_data *td)
590 struct http_data *http = NULL;
591 struct http_options *o = td->eo;
593 /* allocate engine specific structure to deal with libhttp. */
594 http = calloc(1, sizeof(*http));
596 log_err("calloc failed.\n");
600 http->curl = curl_easy_init();
602 curl_easy_setopt(http->curl, CURLOPT_VERBOSE, 1L);
604 curl_easy_setopt(http->curl, CURLOPT_DEBUGFUNCTION, &_curl_trace);
605 curl_easy_setopt(http->curl, CURLOPT_NOPROGRESS, 1L);
606 curl_easy_setopt(http->curl, CURLOPT_FOLLOWLOCATION, 1L);
607 curl_easy_setopt(http->curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP|CURLPROTO_HTTPS);
608 if (o->https == FIO_HTTPS_INSECURE) {
609 curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYPEER, 0L);
610 curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYHOST, 0L);
612 curl_easy_setopt(http->curl, CURLOPT_READFUNCTION, _http_read);
613 curl_easy_setopt(http->curl, CURLOPT_WRITEFUNCTION, _http_write);
614 curl_easy_setopt(http->curl, CURLOPT_SEEKFUNCTION, &_http_seek);
615 if (o->user && o->pass) {
616 curl_easy_setopt(http->curl, CURLOPT_USERNAME, o->user);
617 curl_easy_setopt(http->curl, CURLOPT_PASSWORD, o->pass);
618 curl_easy_setopt(http->curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
621 td->io_ops_data = http;
623 /* Force single process mode. */
624 td->o.use_thread = 1;
628 fio_http_cleanup(td);
632 static int fio_http_open(struct thread_data *td, struct fio_file *f)
636 static int fio_http_invalidate(struct thread_data *td, struct fio_file *f)
641 FIO_STATIC struct ioengine_ops ioengine = {
643 .version = FIO_IOOPS_VERSION,
644 .flags = FIO_DISKLESSIO | FIO_SYNCIO,
645 .setup = fio_http_setup,
646 .queue = fio_http_queue,
647 .getevents = fio_http_getevents,
648 .event = fio_http_event,
649 .cleanup = fio_http_cleanup,
650 .open_file = fio_http_open,
651 .invalidate = fio_http_invalidate,
653 .option_struct_size = sizeof(struct http_options),
656 static void fio_init fio_http_register(void)
658 register_ioengine(&ioengine);
661 static void fio_exit fio_http_unregister(void)
663 unregister_ioengine(&ioengine);