engines/http.c

   1 /*
   2  * HTTP GET/PUT IO engine
   3  *
   4  * IO engine to perform HTTP(S) GET/PUT requests via libcurl-easy.
   5  *
   6  * Copyright (C) 2018 SUSE LLC
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License,
  10  * version 2 as published by the Free Software Foundation..
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public
  18  * License along with this program; if not, write to the Free
  19  * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  20  * Boston, MA 02110-1301, USA.
  21  */
  22
  23 #include <pthread.h>
  24 #include <time.h>
  25 #include <curl/curl.h>
  26 #include <openssl/hmac.h>
  27 #include <openssl/sha.h>
  28 #include "fio.h"
  29 #include "../optgroup.h"
  30
  31
  32 struct http_data {
  33         CURL *curl;
  34 };
  35
  36 struct http_options {
  37         void *pad;
  38         int  https;
  39         char *host;
  40         char *user;
  41         char *pass;
  42         char *s3_key;
  43         char *s3_keyid;
  44         char *s3_region;
  45         int verbose;
  46         int s3;
  47 };
  48
  49 struct http_curl_stream {
  50         char *buf;
  51         size_t pos;
  52         size_t max;
  53 };
  54
  55 static struct fio_option options[] = {
  56         {
  57                 .name     = "https",
  58                 .lname    = "https",
  59                 .type     = FIO_OPT_BOOL,
  60                 .help     = "Enable https",
  61                 .off1     = offsetof(struct http_options, https),
  62                 .def      = "0",
  63                 .category = FIO_OPT_C_ENGINE,
  64                 .group    = FIO_OPT_G_HTTP,
  65         },
  66         {
  67                 .name     = "http_host",
  68                 .lname    = "http_host",
  69                 .type     = FIO_OPT_STR_STORE,
  70                 .help     = "Hostname (S3 bucket)",
  71                 .off1     = offsetof(struct http_options, host),
  72                 .def      = "localhost",
  73                 .category = FIO_OPT_C_ENGINE,
  74                 .group    = FIO_OPT_G_HTTP,
  75         },
  76         {
  77                 .name     = "http_user",
  78                 .lname    = "http_user",
  79                 .type     = FIO_OPT_STR_STORE,
  80                 .help     = "HTTP user name",
  81                 .off1     = offsetof(struct http_options, user),
  82                 .category = FIO_OPT_C_ENGINE,
  83                 .group    = FIO_OPT_G_HTTP,
  84         },
  85         {
  86                 .name     = "http_pass",
  87                 .lname    = "http_pass",
  88                 .type     = FIO_OPT_STR_STORE,
  89                 .help     = "HTTP password",
  90                 .off1     = offsetof(struct http_options, pass),
  91                 .category = FIO_OPT_C_ENGINE,
  92                 .group    = FIO_OPT_G_HTTP,
  93         },
  94         {
  95                 .name     = "http_s3_key",
  96                 .lname    = "S3 secret key",
  97                 .type     = FIO_OPT_STR_STORE,
  98                 .help     = "S3 secret key",
  99                 .off1     = offsetof(struct http_options, s3_key),
 100                 .def      = "",
 101                 .category = FIO_OPT_C_ENGINE,
 102                 .group    = FIO_OPT_G_HTTP,
 103         },
 104         {
 105                 .name     = "http_s3_keyid",
 106                 .lname    = "S3 key id",
 107                 .type     = FIO_OPT_STR_STORE,
 108                 .help     = "S3 key id",
 109                 .off1     = offsetof(struct http_options, s3_keyid),
 110                 .def      = "",
 111                 .category = FIO_OPT_C_ENGINE,
 112                 .group    = FIO_OPT_G_HTTP,
 113         },
 114         {
 115                 .name     = "http_s3_region",
 116                 .lname    = "S3 region",
 117                 .type     = FIO_OPT_STR_STORE,
 118                 .help     = "S3 region",
 119                 .off1     = offsetof(struct http_options, s3_region),
 120                 .def      = "us-east-1",
 121                 .category = FIO_OPT_C_ENGINE,
 122                 .group    = FIO_OPT_G_HTTP,
 123         },
 124         {
 125                 .name     = "http_s3",
 126                 .lname    = "S3 extensions",
 127                 .type     = FIO_OPT_BOOL,
 128                 .help     = "Whether to enable S3 specific headers",
 129                 .off1     = offsetof(struct http_options, s3),
 130                 .def      = "0",
 131                 .category = FIO_OPT_C_ENGINE,
 132                 .group    = FIO_OPT_G_HTTP,
 133         },
 134         {
 135                 .name     = "http_verbose",
 136                 .lname    = "CURL verbosity",
 137                 .type     = FIO_OPT_INT,
 138                 .help     = "increase http engine verbosity",
 139                 .off1     = offsetof(struct http_options, verbose),
 140                 .def      = "0",
 141                 .category = FIO_OPT_C_ENGINE,
 142                 .group    = FIO_OPT_G_HTTP,
 143         },
 144         {
 145                 .name     = NULL,
 146         },
 147 };
 148
 149 static char *_aws_uriencode(const char *uri)
 150 {
 151         size_t bufsize = 1024;
 152         char *r = malloc(bufsize);
 153         char c;
 154         int i, n;
 155         const char *hex = "0123456789ABCDEF";
 156
 157         if (!r) {
 158                 log_err("malloc failed\n");
 159                 return NULL;
 160         }
 161
 162         n = 0;
 163         for (i = 0; (c = uri[i]); i++) {
 164                 if (n > bufsize-5) {
 165                         log_err("encoding the URL failed\n");
 166                         return NULL;
 167                 }
 168
 169                 if ( (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
 170                 || (c >= '0' && c <= '9') || c == '_' || c == '-'
 171                 || c == '~' || c == '.' || c == '/')
 172                         r[n++] = c;
 173                 else {
 174                         r[n++] = '%';
 175                         r[n++] = hex[(c >> 4 ) & 0xF];
 176                         r[n++] = hex[c & 0xF];
 177                 }
 178         }
 179         r[n++] = 0;
 180         return r;
 181 }
 182
 183 static char *_conv_hex(const unsigned char *p, size_t len)
 184 {
 185         char *r;
 186         int i,n;
 187         const char *hex = "0123456789abcdef";
 188         r = malloc(len * 2 + 1);
 189         n = 0;
 190         for (i = 0; i < len; i++) {
 191                 r[n++] = hex[(p[i] >> 4 ) & 0xF];
 192                 r[n++] = hex[p[i] & 0xF];
 193         }
 194         r[n] = 0;
 195
 196         return r;
 197 }
 198
 199 static char *_gen_hex_sha256(const char *p, size_t len)
 200 {
 201         unsigned char hash[SHA256_DIGEST_LENGTH];
 202
 203         SHA256((unsigned char*)p, len, hash);
 204         return _conv_hex(hash, SHA256_DIGEST_LENGTH);
 205 }
 206
 207 static void _hmac(unsigned char *md, void *key, int key_len, char *data) {
 208 #ifndef CONFIG_HAVE_OPAQUE_HMAC_CTX
 209         HMAC_CTX _ctx;
 210 #endif
 211         HMAC_CTX *ctx;
 212         unsigned int hmac_len;
 213
 214 #ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX
 215         ctx = HMAC_CTX_new();
 216 #else
 217         ctx = &_ctx;
 218 #endif
 219         HMAC_Init_ex(ctx, key, key_len, EVP_sha256(), NULL);
 220         HMAC_Update(ctx, (unsigned char*)data, strlen(data));
 221         HMAC_Final(ctx, md, &hmac_len);
 222 #ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX
 223         HMAC_CTX_free(ctx);
 224 #else
 225         HMAC_CTX_cleanup(ctx);
 226 #endif
 227 }
 228
 229 static int _curl_trace(CURL *handle, curl_infotype type,
 230              char *data, size_t size,
 231              void *userp)
 232 {
 233         const char *text;
 234         (void)handle; /* prevent compiler warning */
 235         (void)userp;
 236
 237         switch (type) {
 238         case CURLINFO_TEXT:
 239         fprintf(stderr, "== Info: %s", data);
 240         default:
 241         case CURLINFO_SSL_DATA_OUT:
 242         case CURLINFO_SSL_DATA_IN:
 243                 return 0;
 244
 245         case CURLINFO_HEADER_OUT:
 246                 text = "=> Send header";
 247                 break;
 248         case CURLINFO_DATA_OUT:
 249                 text = "=> Send data";
 250                 break;
 251         case CURLINFO_HEADER_IN:
 252                 text = "<= Recv header";
 253                 break;
 254         case CURLINFO_DATA_IN:
 255                 text = "<= Recv data";
 256                 break;
 257         }
 258
 259         log_info("%s: %s", text, data);
 260         return 0;
 261 }
 262
 263 /* https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
 264  * https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
 265  */
 266 static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct http_options *o,
 267                 int op, const char *uri, char *buf, size_t len)
 268 {
 269         char date_short[16];
 270         char date_iso[32];
 271         char method[8];
 272         char dkey[128];
 273         char creq[512];
 274         char sts[256];
 275         char s[512];
 276         char *uri_encoded = NULL;
 277         char *dsha = NULL;
 278         char *csha = NULL;
 279         char *signature = NULL;
 280         const char *service = "s3";
 281         const char *aws = "aws4_request";
 282         unsigned char md[SHA256_DIGEST_LENGTH];
 283
 284         time_t t = time(NULL);
 285         struct tm *gtm = gmtime(&t);
 286
 287         strftime (date_short, sizeof(date_short), "%Y%m%d", gtm);
 288         strftime (date_iso, sizeof(date_iso), "%Y%m%dT%H%M%SZ", gtm);
 289         uri_encoded = _aws_uriencode(uri);
 290
 291         if (op == DDIR_WRITE) {
 292                 dsha = _gen_hex_sha256(buf, len);
 293                 sprintf(method, "PUT");
 294         } else {
 295                 /* DDIR_READ && DDIR_TRIM supply an empty body */
 296                 if (op == DDIR_READ)
 297                         sprintf(method, "GET");
 298                 else
 299                         sprintf(method, "DELETE");
 300                 dsha = _gen_hex_sha256("", 0);
 301         }
 302
 303         /* Create the canonical request first */
 304         snprintf(creq, sizeof(creq),
 305         "%s\n"
 306         "%s\n"
 307         "\n"
 308         "host:%s\n"
 309         "x-amz-content-sha256:%s\n"
 310         "x-amz-date:%s\n"
 311         "\n"
 312         "host;x-amz-content-sha256;x-amz-date\n"
 313         "%s"
 314         , method
 315         , uri_encoded, o->host, dsha, date_iso, dsha);
 316
 317         csha = _gen_hex_sha256(creq, strlen(creq));
 318         snprintf(sts, sizeof(sts), "AWS4-HMAC-SHA256\n%s\n%s/%s/%s/%s\n%s",
 319                 date_iso, date_short, o->s3_region, service, aws, csha);
 320
 321         snprintf((char *)dkey, sizeof(dkey), "AWS4%s", o->s3_key);
 322         _hmac(md, dkey, strlen(dkey), date_short);
 323         _hmac(md, md, SHA256_DIGEST_LENGTH, o->s3_region);
 324         _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) service);
 325         _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) aws);
 326         _hmac(md, md, SHA256_DIGEST_LENGTH, sts);
 327
 328         signature = _conv_hex(md, SHA256_DIGEST_LENGTH);
 329
 330         /* Surpress automatic Accept: header */
 331         slist = curl_slist_append(slist, "Accept:");
 332
 333         snprintf(s, sizeof(s), "x-amz-content-sha256: %s", dsha);
 334         slist = curl_slist_append(slist, s);
 335
 336         snprintf(s, sizeof(s), "x-amz-date: %s", date_iso);
 337         slist = curl_slist_append(slist, s);
 338
 339         snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
 340         "SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=%s",
 341         o->s3_keyid, date_short, o->s3_region, signature);
 342         slist = curl_slist_append(slist, s);
 343
 344         curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
 345
 346         free(uri_encoded);
 347         free(csha);
 348         free(dsha);
 349         free(signature);
 350 }
 351
 352 static void fio_http_cleanup(struct thread_data *td)
 353 {
 354         struct http_data *http = td->io_ops_data;
 355
 356         if (http) {
 357                 curl_easy_cleanup(http->curl);
 358                 free(http);
 359         }
 360 }
 361
 362 static size_t _http_read(void *ptr, size_t size, size_t nmemb, void *stream)
 363 {
 364         struct http_curl_stream *state = stream;
 365         size_t len = size * nmemb;
 366         /* We're retrieving; nothing is supposed to be read locally */
 367         if (!stream)
 368                 return 0;
 369         if (len+state->pos > state->max)
 370                 len = state->max - state->pos;
 371         memcpy(ptr, &state->buf[state->pos], len);
 372         state->pos += len;
 373         return len;
 374 }
 375
 376 static size_t _http_write(void *ptr, size_t size, size_t nmemb, void *stream)
 377 {
 378         struct http_curl_stream *state = stream;
 379         /* We're just discarding the returned body after a PUT */
 380         if (!stream)
 381                 return nmemb;
 382         if (size != 1)
 383                 return CURLE_WRITE_ERROR;
 384         if (nmemb + state->pos > state->max)
 385                 return CURLE_WRITE_ERROR;
 386         memcpy(&state->buf[state->pos], ptr, nmemb);
 387         state->pos += nmemb;
 388         return nmemb;
 389 }
 390
 391 static int _http_seek(void *stream, curl_off_t offset, int origin)
 392 {
 393         struct http_curl_stream *state = stream;
 394         if (offset < state->max && origin == SEEK_SET) {
 395                 state->pos = offset;
 396                 return CURL_SEEKFUNC_OK;
 397         } else
 398                 return CURL_SEEKFUNC_FAIL;
 399 }
 400
 401 static enum fio_q_status fio_http_queue(struct thread_data *td,
 402                                          struct io_u *io_u)
 403 {
 404         struct http_data *http = td->io_ops_data;
 405         struct http_options *o = td->eo;
 406         struct http_curl_stream _curl_stream;
 407         struct curl_slist *slist = NULL;
 408         char object[512];
 409         char url[1024];
 410         long status;
 411         CURLcode res;
 412         int r = -1;
 413
 414         fio_ro_check(td, io_u);
 415         memset(&_curl_stream, 0, sizeof(_curl_stream));
 416         snprintf(object, sizeof(object), "%s_%llu_%llu", td->files[0]->file_name, io_u->offset, io_u->xfer_buflen);
 417         snprintf(url, sizeof(url), "%s://%s%s", o->https ? "https" : "http", o->host, object);
 418         curl_easy_setopt(http->curl, CURLOPT_URL, url);
 419         _curl_stream.buf = io_u->xfer_buf;
 420         _curl_stream.max = io_u->xfer_buflen;
 421         curl_easy_setopt(http->curl, CURLOPT_SEEKDATA, &_curl_stream);
 422         curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)io_u->xfer_buflen);
 423
 424         if (o->s3)
 425                 _add_aws_auth_header(http->curl, slist, o, io_u->ddir, object,
 426                         io_u->xfer_buf, io_u->xfer_buflen);
 427
 428         if (io_u->ddir == DDIR_WRITE) {
 429                 curl_easy_setopt(http->curl, CURLOPT_READDATA, &_curl_stream);
 430                 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
 431                 curl_easy_setopt(http->curl, CURLOPT_UPLOAD, 1L);
 432                 res = curl_easy_perform(http->curl);
 433                 if (res == CURLE_OK) {
 434                         curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
 435                         if (status == 100 || (status >= 200 && status <= 204))
 436                                 goto out;
 437                         log_err("DDIR_WRITE failed with HTTP status code %ld\n", status);
 438                         goto err;
 439                 }
 440         } else if (io_u->ddir == DDIR_READ) {
 441                 curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
 442                 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, &_curl_stream);
 443                 curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
 444                 res = curl_easy_perform(http->curl);
 445                 if (res == CURLE_OK) {
 446                         curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
 447                         if (status == 200)
 448                                 goto out;
 449                         else if (status == 404) {
 450                                 /* Object doesn't exist. Pretend we read
 451                                  * zeroes */
 452                                 memset(io_u->xfer_buf, 0, io_u->xfer_buflen);
 453                                 goto out;
 454                         }
 455                         log_err("DDIR_READ failed with HTTP status code %ld\n", status);
 456                 }
 457                 goto err;
 458         } else if (io_u->ddir == DDIR_TRIM) {
 459                 curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
 460                 curl_easy_setopt(http->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
 461                 curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, 0);
 462                 curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
 463                 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
 464                 res = curl_easy_perform(http->curl);
 465                 if (res == CURLE_OK) {
 466                         curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
 467                         if (status == 200 || status == 202 || status == 204 || status == 404)
 468                                 goto out;
 469                         log_err("DDIR_TRIM failed with HTTP status code %ld\n", status);
 470                 }
 471                 goto err;
 472         }
 473
 474         log_err("WARNING: Only DDIR_READ/DDIR_WRITE/DDIR_TRIM are supported!\n");
 475
 476 err:
 477         io_u->error = r;
 478         td_verror(td, io_u->error, "transfer");
 479 out:
 480         curl_slist_free_all(slist);
 481         return FIO_Q_COMPLETED;
 482 }
 483
 484 static struct io_u *fio_http_event(struct thread_data *td, int event)
 485 {
 486         /* sync IO engine - never any outstanding events */
 487         return NULL;
 488 }
 489
 490 int fio_http_getevents(struct thread_data *td, unsigned int min,
 491         unsigned int max, const struct timespec *t)
 492 {
 493         /* sync IO engine - never any outstanding events */
 494         return 0;
 495 }
 496
 497 static int fio_http_setup(struct thread_data *td)
 498 {
 499         struct http_data *http = NULL;
 500         struct http_options *o = td->eo;
 501         int r;
 502         /* allocate engine specific structure to deal with libhttp. */
 503         http = calloc(1, sizeof(*http));
 504         if (!http) {
 505                 log_err("calloc failed.\n");
 506                 goto cleanup;
 507         }
 508
 509         http->curl = curl_easy_init();
 510         if (o->verbose)
 511                 curl_easy_setopt(http->curl, CURLOPT_VERBOSE, 1L);
 512         if (o->verbose > 1)
 513                 curl_easy_setopt(http->curl, CURLOPT_DEBUGFUNCTION, &_curl_trace);
 514         curl_easy_setopt(http->curl, CURLOPT_NOPROGRESS, 1L);
 515         curl_easy_setopt(http->curl, CURLOPT_FOLLOWLOCATION, 1L);
 516         curl_easy_setopt(http->curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP|CURLPROTO_HTTPS);
 517         curl_easy_setopt(http->curl, CURLOPT_READFUNCTION, _http_read);
 518         curl_easy_setopt(http->curl, CURLOPT_WRITEFUNCTION, _http_write);
 519         curl_easy_setopt(http->curl, CURLOPT_SEEKFUNCTION, _http_seek);
 520         if (o->user && o->pass) {
 521                 curl_easy_setopt(http->curl, CURLOPT_USERNAME, o->user);
 522                 curl_easy_setopt(http->curl, CURLOPT_PASSWORD, o->pass);
 523                 curl_easy_setopt(http->curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
 524         }
 525
 526         td->io_ops_data = http;
 527
 528         /* Force single process mode. */
 529         td->o.use_thread = 1;
 530
 531         return 0;
 532 cleanup:
 533         fio_http_cleanup(td);
 534         return r;
 535 }
 536
 537 static int fio_http_open(struct thread_data *td, struct fio_file *f)
 538 {
 539         return 0;
 540 }
 541 static int fio_http_invalidate(struct thread_data *td, struct fio_file *f)
 542 {
 543         return 0;
 544 }
 545
 546 static struct ioengine_ops ioengine = {
 547         .name = "http",
 548         .version                = FIO_IOOPS_VERSION,
 549         .flags                  = FIO_DISKLESSIO,
 550         .setup                  = fio_http_setup,
 551         .queue                  = fio_http_queue,
 552         .getevents              = fio_http_getevents,
 553         .event                  = fio_http_event,
 554         .cleanup                = fio_http_cleanup,
 555         .open_file              = fio_http_open,
 556         .invalidate             = fio_http_invalidate,
 557         .options                = options,
 558         .option_struct_size     = sizeof(struct http_options),
 559 };
 560
 561 static void fio_init fio_http_register(void)
 562 {
 563         register_ioengine(&ioengine);
 564 }
 565
 566 static void fio_exit fio_http_unregister(void)
 567 {
 568         unregister_ioengine(&ioengine);
 569 }