engines/http.c

   1 /*
   2  * HTTP GET/PUT IO engine
   3  *
   4  * IO engine to perform HTTP(S) GET/PUT requests via libcurl-easy.
   5  *
   6  * Copyright (C) 2018 SUSE LLC
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License,
  10  * version 2 as published by the Free Software Foundation..
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public
  18  * License along with this program; if not, write to the Free
  19  * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  20  * Boston, MA 02110-1301, USA.
  21  */
  22
  23 #include <pthread.h>
  24 #include <time.h>
  25 #include <curl/curl.h>
  26 #include <openssl/hmac.h>
  27 #include <openssl/sha.h>
  28 #include <openssl/md5.h>
  29 #include "fio.h"
  30 #include "../optgroup.h"
  31
  32 /*
  33  * Silence OpenSSL 3.0 deprecated function warnings
  34  */
  35 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
  36
  37 enum {
  38         FIO_HTTP_WEBDAV     = 0,
  39         FIO_HTTP_S3         = 1,
  40         FIO_HTTP_SWIFT      = 2,
  41
  42         FIO_HTTPS_OFF       = 0,
  43         FIO_HTTPS_ON        = 1,
  44         FIO_HTTPS_INSECURE  = 2,
  45 };
  46
  47 struct http_data {
  48         CURL *curl;
  49 };
  50
  51 struct http_options {
  52         void *pad;
  53         unsigned int https;
  54         char *host;
  55         char *user;
  56         char *pass;
  57         char *s3_key;
  58         char *s3_keyid;
  59         char *s3_region;
  60         char *s3_sse_customer_key;
  61         char *s3_sse_customer_algorithm;
  62         char *s3_storage_class;
  63         char *swift_auth_token;
  64         int verbose;
  65         unsigned int mode;
  66 };
  67
  68 struct http_curl_stream {
  69         char *buf;
  70         size_t pos;
  71         size_t max;
  72 };
  73
  74 static struct fio_option options[] = {
  75         {
  76                 .name     = "https",
  77                 .lname    = "https",
  78                 .type     = FIO_OPT_STR,
  79                 .help     = "Enable https",
  80                 .off1     = offsetof(struct http_options, https),
  81                 .def      = "off",
  82                 .posval = {
  83                           { .ival = "off",
  84                             .oval = FIO_HTTPS_OFF,
  85                             .help = "No HTTPS",
  86                           },
  87                           { .ival = "on",
  88                             .oval = FIO_HTTPS_ON,
  89                             .help = "Enable HTTPS",
  90                           },
  91                           { .ival = "insecure",
  92                             .oval = FIO_HTTPS_INSECURE,
  93                             .help = "Enable HTTPS, disable peer verification",
  94                           },
  95                 },
  96                 .category = FIO_OPT_C_ENGINE,
  97                 .group    = FIO_OPT_G_HTTP,
  98         },
  99         {
 100                 .name     = "http_host",
 101                 .lname    = "http_host",
 102                 .type     = FIO_OPT_STR_STORE,
 103                 .help     = "Hostname (S3 bucket)",
 104                 .off1     = offsetof(struct http_options, host),
 105                 .def      = "localhost",
 106                 .category = FIO_OPT_C_ENGINE,
 107                 .group    = FIO_OPT_G_HTTP,
 108         },
 109         {
 110                 .name     = "http_user",
 111                 .lname    = "http_user",
 112                 .type     = FIO_OPT_STR_STORE,
 113                 .help     = "HTTP user name",
 114                 .off1     = offsetof(struct http_options, user),
 115                 .category = FIO_OPT_C_ENGINE,
 116                 .group    = FIO_OPT_G_HTTP,
 117         },
 118         {
 119                 .name     = "http_pass",
 120                 .lname    = "http_pass",
 121                 .type     = FIO_OPT_STR_STORE,
 122                 .help     = "HTTP password",
 123                 .off1     = offsetof(struct http_options, pass),
 124                 .category = FIO_OPT_C_ENGINE,
 125                 .group    = FIO_OPT_G_HTTP,
 126         },
 127         {
 128                 .name     = "http_s3_key",
 129                 .lname    = "S3 secret key",
 130                 .type     = FIO_OPT_STR_STORE,
 131                 .help     = "S3 secret key",
 132                 .off1     = offsetof(struct http_options, s3_key),
 133                 .def      = "",
 134                 .category = FIO_OPT_C_ENGINE,
 135                 .group    = FIO_OPT_G_HTTP,
 136         },
 137         {
 138                 .name     = "http_s3_keyid",
 139                 .lname    = "S3 key id",
 140                 .type     = FIO_OPT_STR_STORE,
 141                 .help     = "S3 key id",
 142                 .off1     = offsetof(struct http_options, s3_keyid),
 143                 .def      = "",
 144                 .category = FIO_OPT_C_ENGINE,
 145                 .group    = FIO_OPT_G_HTTP,
 146         },
 147         {
 148                 .name     = "http_swift_auth_token",
 149                 .lname    = "Swift auth token",
 150                 .type     = FIO_OPT_STR_STORE,
 151                 .help     = "OpenStack Swift auth token",
 152                 .off1     = offsetof(struct http_options, swift_auth_token),
 153                 .def      = "",
 154                 .category = FIO_OPT_C_ENGINE,
 155                 .group    = FIO_OPT_G_HTTP,
 156         },
 157         {
 158                 .name     = "http_s3_region",
 159                 .lname    = "S3 region",
 160                 .type     = FIO_OPT_STR_STORE,
 161                 .help     = "S3 region",
 162                 .off1     = offsetof(struct http_options, s3_region),
 163                 .def      = "us-east-1",
 164                 .category = FIO_OPT_C_ENGINE,
 165                 .group    = FIO_OPT_G_HTTP,
 166         },
 167         {
 168                 .name     = "http_s3_sse_customer_key",
 169                 .lname    = "SSE Customer Key",
 170                 .type     = FIO_OPT_STR_STORE,
 171                 .help     = "S3 SSE Customer Key",
 172                 .off1     = offsetof(struct http_options, s3_sse_customer_key),
 173                 .def      = "",
 174                 .category = FIO_OPT_C_ENGINE,
 175                 .group    = FIO_OPT_G_HTTP,
 176         },
 177         {
 178                 .name     = "http_s3_sse_customer_algorithm",
 179                 .lname    = "SSE Customer Algorithm",
 180                 .type     = FIO_OPT_STR_STORE,
 181                 .help     = "S3 SSE Customer Algorithm",
 182                 .off1     = offsetof(struct http_options, s3_sse_customer_algorithm),
 183                 .def      = "AES256",
 184                 .category = FIO_OPT_C_ENGINE,
 185                 .group    = FIO_OPT_G_HTTP,
 186         },
 187         {
 188                 .name     = "http_s3_storage_class",
 189                 .lname    = "S3 Storage class",
 190                 .type     = FIO_OPT_STR_STORE,
 191                 .help     = "S3 Storage Class",
 192                 .off1     = offsetof(struct http_options, s3_storage_class),
 193                 .def      = "STANDARD",
 194                 .category = FIO_OPT_C_ENGINE,
 195                 .group    = FIO_OPT_G_HTTP,
 196         },
 197         {
 198                 .name     = "http_mode",
 199                 .lname    = "Request mode to use",
 200                 .type     = FIO_OPT_STR,
 201                 .help     = "Whether to use WebDAV, Swift, or S3",
 202                 .off1     = offsetof(struct http_options, mode),
 203                 .def      = "webdav",
 204                 .posval = {
 205                           { .ival = "webdav",
 206                             .oval = FIO_HTTP_WEBDAV,
 207                             .help = "WebDAV server",
 208                           },
 209                           { .ival = "s3",
 210                             .oval = FIO_HTTP_S3,
 211                             .help = "S3 storage backend",
 212                           },
 213                           { .ival = "swift",
 214                             .oval = FIO_HTTP_SWIFT,
 215                             .help = "OpenStack Swift storage",
 216                           },
 217                 },
 218                 .category = FIO_OPT_C_ENGINE,
 219                 .group    = FIO_OPT_G_HTTP,
 220         },
 221         {
 222                 .name     = "http_verbose",
 223                 .lname    = "HTTP verbosity level",
 224                 .type     = FIO_OPT_INT,
 225                 .help     = "increase http engine verbosity",
 226                 .off1     = offsetof(struct http_options, verbose),
 227                 .def      = "0",
 228                 .category = FIO_OPT_C_ENGINE,
 229                 .group    = FIO_OPT_G_HTTP,
 230         },
 231         {
 232                 .name     = NULL,
 233         },
 234 };
 235
 236 static char *_aws_uriencode(const char *uri)
 237 {
 238         size_t bufsize = 1024;
 239         char *r = malloc(bufsize);
 240         char c;
 241         int i, n;
 242         const char *hex = "0123456789ABCDEF";
 243
 244         if (!r) {
 245                 log_err("malloc failed\n");
 246                 return NULL;
 247         }
 248
 249         n = 0;
 250         for (i = 0; (c = uri[i]); i++) {
 251                 if (n > bufsize-5) {
 252                         log_err("encoding the URL failed\n");
 253                         return NULL;
 254                 }
 255
 256                 if ( (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
 257                 || (c >= '0' && c <= '9') || c == '_' || c == '-'
 258                 || c == '~' || c == '.' || c == '/')
 259                         r[n++] = c;
 260                 else {
 261                         r[n++] = '%';
 262                         r[n++] = hex[(c >> 4 ) & 0xF];
 263                         r[n++] = hex[c & 0xF];
 264                 }
 265         }
 266         r[n++] = 0;
 267         return r;
 268 }
 269
 270 static char *_conv_hex(const unsigned char *p, size_t len)
 271 {
 272         char *r;
 273         int i,n;
 274         const char *hex = "0123456789abcdef";
 275         r = malloc(len * 2 + 1);
 276         n = 0;
 277         for (i = 0; i < len; i++) {
 278                 r[n++] = hex[(p[i] >> 4 ) & 0xF];
 279                 r[n++] = hex[p[i] & 0xF];
 280         }
 281         r[n] = 0;
 282
 283         return r;
 284 }
 285
 286 static char *_gen_hex_sha256(const char *p, size_t len)
 287 {
 288         unsigned char hash[SHA256_DIGEST_LENGTH];
 289
 290         SHA256((unsigned char*)p, len, hash);
 291         return _conv_hex(hash, SHA256_DIGEST_LENGTH);
 292 }
 293
 294 static char *_gen_hex_md5(const char *p, size_t len)
 295 {
 296         unsigned char hash[MD5_DIGEST_LENGTH];
 297
 298         MD5((unsigned char*)p, len, hash);
 299         return _conv_hex(hash, MD5_DIGEST_LENGTH);
 300 }
 301
 302 static char *_conv_base64_encode(const unsigned char *p, size_t len)
 303 {
 304         char *r, *ret;
 305         int i;
 306         static const char sEncodingTable[] = {
 307                 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
 308                 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
 309                 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
 310                 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
 311                 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
 312                 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
 313                 'w', 'x', 'y', 'z', '0', '1', '2', '3',
 314                 '4', '5', '6', '7', '8', '9', '+', '/'
 315         };
 316
 317         size_t out_len = 4 * ((len + 2) / 3);
 318         ret = r = malloc(out_len + 1);
 319
 320         for (i = 0; i < len - 2; i += 3) {
 321                 *r++ = sEncodingTable[(p[i] >> 2) & 0x3F];
 322                 *r++ = sEncodingTable[((p[i] & 0x3) << 4) | ((int) (p[i + 1] & 0xF0) >> 4)];
 323                 *r++ = sEncodingTable[((p[i + 1] & 0xF) << 2) | ((int) (p[i + 2] & 0xC0) >> 6)];
 324                 *r++ = sEncodingTable[p[i + 2] & 0x3F];
 325         }
 326
 327         if (i < len) {
 328                 *r++ = sEncodingTable[(p[i] >> 2) & 0x3F];
 329                 if (i == (len - 1)) {
 330                         *r++ = sEncodingTable[((p[i] & 0x3) << 4)];
 331                         *r++ = '=';
 332                 } else {
 333                         *r++ = sEncodingTable[((p[i] & 0x3) << 4) | ((int) (p[i + 1] & 0xF0) >> 4)];
 334                         *r++ = sEncodingTable[((p[i + 1] & 0xF) << 2)];
 335                 }
 336                 *r++ = '=';
 337         }
 338
 339         ret[out_len]=0;
 340         return ret;
 341 }
 342
 343 static char *_gen_base64_md5(const unsigned char *p, size_t len)
 344 {
 345         unsigned char hash[MD5_DIGEST_LENGTH];
 346         MD5((unsigned char*)p, len, hash);
 347         return _conv_base64_encode(hash, MD5_DIGEST_LENGTH);
 348 }
 349
 350 static void _hmac(unsigned char *md, void *key, int key_len, char *data) {
 351 #ifndef CONFIG_HAVE_OPAQUE_HMAC_CTX
 352         HMAC_CTX _ctx;
 353 #endif
 354         HMAC_CTX *ctx;
 355         unsigned int hmac_len;
 356
 357 #ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX
 358         ctx = HMAC_CTX_new();
 359 #else
 360         ctx = &_ctx;
 361         /* work-around crash in certain versions of libssl */
 362         HMAC_CTX_init(ctx);
 363 #endif
 364         HMAC_Init_ex(ctx, key, key_len, EVP_sha256(), NULL);
 365         HMAC_Update(ctx, (unsigned char*)data, strlen(data));
 366         HMAC_Final(ctx, md, &hmac_len);
 367 #ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX
 368         HMAC_CTX_free(ctx);
 369 #else
 370         HMAC_CTX_cleanup(ctx);
 371 #endif
 372 }
 373
 374 static int _curl_trace(CURL *handle, curl_infotype type,
 375              char *data, size_t size,
 376              void *userp)
 377 {
 378         const char *text;
 379         (void)handle; /* prevent compiler warning */
 380         (void)userp;
 381
 382         switch (type) {
 383         case CURLINFO_TEXT:
 384                 fprintf(stderr, "== Info: %s", data);
 385                 fio_fallthrough;
 386         default:
 387         case CURLINFO_SSL_DATA_OUT:
 388         case CURLINFO_SSL_DATA_IN:
 389                 return 0;
 390
 391         case CURLINFO_HEADER_OUT:
 392                 text = "=> Send header";
 393                 break;
 394         case CURLINFO_DATA_OUT:
 395                 text = "=> Send data";
 396                 break;
 397         case CURLINFO_HEADER_IN:
 398                 text = "<= Recv header";
 399                 break;
 400         case CURLINFO_DATA_IN:
 401                 text = "<= Recv data";
 402                 break;
 403         }
 404
 405         log_info("%s: %s", text, data);
 406         return 0;
 407 }
 408
 409 /* https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
 410  * https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
 411  */
 412 static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct http_options *o,
 413                 int op, const char *uri, char *buf, size_t len)
 414 {
 415         char date_short[16];
 416         char date_iso[32];
 417         char method[8];
 418         char dkey[128];
 419         char creq[4096];
 420         char sts[512];
 421         char s[512];
 422         char *uri_encoded = NULL;
 423         char *dsha = NULL;
 424         char *csha = NULL;
 425         char *signature = NULL;
 426         const char *service = "s3";
 427         const char *aws = "aws4_request";
 428         unsigned char md[SHA256_DIGEST_LENGTH];
 429         unsigned char sse_key[33] = {0};
 430         char *sse_key_base64 = NULL;
 431         char *sse_key_md5_base64 = NULL;
 432
 433         time_t t = time(NULL);
 434         struct tm *gtm = gmtime(&t);
 435
 436         strftime (date_short, sizeof(date_short), "%Y%m%d", gtm);
 437         strftime (date_iso, sizeof(date_iso), "%Y%m%dT%H%M%SZ", gtm);
 438         uri_encoded = _aws_uriencode(uri);
 439
 440         if (o->s3_sse_customer_key != NULL)
 441                 strncpy((char*)sse_key, o->s3_sse_customer_key, sizeof(sse_key) - 1);
 442
 443         if (op == DDIR_WRITE) {
 444                 dsha = _gen_hex_sha256(buf, len);
 445                 sprintf(method, "PUT");
 446         } else {
 447                 /* DDIR_READ && DDIR_TRIM supply an empty body */
 448                 if (op == DDIR_READ)
 449                         sprintf(method, "GET");
 450                 else
 451                         sprintf(method, "DELETE");
 452                 dsha = _gen_hex_sha256("", 0);
 453         }
 454
 455         /* Create the canonical request first */
 456         if (sse_key[0] != '\0') {
 457                 sse_key_base64 = _conv_base64_encode(sse_key, sizeof(sse_key) - 1);
 458                 sse_key_md5_base64 = _gen_base64_md5(sse_key, sizeof(sse_key) - 1);
 459                 snprintf(creq, sizeof(creq),
 460                         "%s\n"
 461                         "%s\n"
 462                         "\n"
 463                         "host:%s\n"
 464                         "x-amz-content-sha256:%s\n"
 465                         "x-amz-date:%s\n"
 466                         "x-amz-server-side-encryption-customer-algorithm:%s\n"
 467                         "x-amz-server-side-encryption-customer-key:%s\n"
 468                         "x-amz-server-side-encryption-customer-key-md5:%s\n"
 469                         "x-amz-storage-class:%s\n"
 470                         "\n"
 471                         "host;x-amz-content-sha256;x-amz-date;"
 472                         "x-amz-server-side-encryption-customer-algorithm;"
 473                         "x-amz-server-side-encryption-customer-key;"
 474                         "x-amz-server-side-encryption-customer-key-md5;"
 475                         "x-amz-storage-class\n"
 476                         "%s"
 477                         , method
 478                         , uri_encoded, o->host, dsha, date_iso
 479                         , o->s3_sse_customer_algorithm, sse_key_base64
 480                         , sse_key_md5_base64, o->s3_storage_class, dsha);
 481         } else {
 482                 snprintf(creq, sizeof(creq),
 483                         "%s\n"
 484                         "%s\n"
 485                         "\n"
 486                         "host:%s\n"
 487                         "x-amz-content-sha256:%s\n"
 488                         "x-amz-date:%s\n"
 489                         "x-amz-storage-class:%s\n"
 490                         "\n"
 491                         "host;x-amz-content-sha256;x-amz-date;x-amz-storage-class\n"
 492                         "%s"
 493                         , method
 494                         , uri_encoded, o->host, dsha, date_iso, o->s3_storage_class, dsha);
 495         }
 496
 497         csha = _gen_hex_sha256(creq, strlen(creq));
 498         snprintf(sts, sizeof(sts), "AWS4-HMAC-SHA256\n%s\n%s/%s/%s/%s\n%s",
 499                         date_iso, date_short, o->s3_region, service, aws, csha);
 500
 501         snprintf((char *)dkey, sizeof(dkey), "AWS4%s", o->s3_key);
 502         _hmac(md, dkey, strlen(dkey), date_short);
 503         _hmac(md, md, SHA256_DIGEST_LENGTH, o->s3_region);
 504         _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) service);
 505         _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) aws);
 506         _hmac(md, md, SHA256_DIGEST_LENGTH, sts);
 507
 508         signature = _conv_hex(md, SHA256_DIGEST_LENGTH);
 509
 510         /* Suppress automatic Accept: header */
 511         slist = curl_slist_append(slist, "Accept:");
 512
 513         snprintf(s, sizeof(s), "x-amz-content-sha256: %s", dsha);
 514         slist = curl_slist_append(slist, s);
 515
 516         snprintf(s, sizeof(s), "x-amz-date: %s", date_iso);
 517         slist = curl_slist_append(slist, s);
 518
 519         if (sse_key[0] != '\0') {
 520                 snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-algorithm: %s", o->s3_sse_customer_algorithm);
 521                 slist = curl_slist_append(slist, s);
 522                 snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-key: %s", sse_key_base64);
 523                 slist = curl_slist_append(slist, s);
 524                 snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-key-md5: %s", sse_key_md5_base64);
 525                 slist = curl_slist_append(slist, s);
 526         }
 527
 528         snprintf(s, sizeof(s), "x-amz-storage-class: %s", o->s3_storage_class);
 529         slist = curl_slist_append(slist, s);
 530
 531         if (sse_key[0] != '\0') {
 532                 snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
 533                         "SignedHeaders=host;x-amz-content-sha256;"
 534                         "x-amz-date;x-amz-server-side-encryption-customer-algorithm;"
 535                         "x-amz-server-side-encryption-customer-key;"
 536                         "x-amz-server-side-encryption-customer-key-md5;"
 537                         "x-amz-storage-class,"
 538                         "Signature=%s",
 539                 o->s3_keyid, date_short, o->s3_region, signature);
 540         } else {
 541                 snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
 542                         "SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=%s",
 543                         o->s3_keyid, date_short, o->s3_region, signature);
 544         }
 545         slist = curl_slist_append(slist, s);
 546
 547         curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
 548
 549         free(uri_encoded);
 550         free(csha);
 551         free(dsha);
 552         free(signature);
 553         if (sse_key_base64 != NULL) {
 554                 free(sse_key_base64);
 555                 free(sse_key_md5_base64);
 556         }
 557 }
 558
 559 static void _add_swift_header(CURL *curl, struct curl_slist *slist, struct http_options *o,
 560                 int op, const char *uri, char *buf, size_t len)
 561 {
 562         char *dsha = NULL;
 563         char s[512];
 564
 565         if (op == DDIR_WRITE) {
 566                 dsha = _gen_hex_md5(buf, len);
 567         }
 568         /* Suppress automatic Accept: header */
 569         slist = curl_slist_append(slist, "Accept:");
 570
 571         snprintf(s, sizeof(s), "etag: %s", dsha);
 572         slist = curl_slist_append(slist, s);
 573
 574         snprintf(s, sizeof(s), "x-auth-token: %s", o->swift_auth_token);
 575         slist = curl_slist_append(slist, s);
 576
 577         curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
 578
 579         free(dsha);
 580 }
 581
 582 static void fio_http_cleanup(struct thread_data *td)
 583 {
 584         struct http_data *http = td->io_ops_data;
 585
 586         if (http) {
 587                 curl_easy_cleanup(http->curl);
 588                 free(http);
 589         }
 590 }
 591
 592 static size_t _http_read(void *ptr, size_t size, size_t nmemb, void *stream)
 593 {
 594         struct http_curl_stream *state = stream;
 595         size_t len = size * nmemb;
 596         /* We're retrieving; nothing is supposed to be read locally */
 597         if (!stream)
 598                 return 0;
 599         if (len+state->pos > state->max)
 600                 len = state->max - state->pos;
 601         memcpy(ptr, &state->buf[state->pos], len);
 602         state->pos += len;
 603         return len;
 604 }
 605
 606 static size_t _http_write(void *ptr, size_t size, size_t nmemb, void *stream)
 607 {
 608         struct http_curl_stream *state = stream;
 609         /* We're just discarding the returned body after a PUT */
 610         if (!stream)
 611                 return nmemb;
 612         if (size != 1)
 613                 return CURLE_WRITE_ERROR;
 614         if (nmemb + state->pos > state->max)
 615                 return CURLE_WRITE_ERROR;
 616         memcpy(&state->buf[state->pos], ptr, nmemb);
 617         state->pos += nmemb;
 618         return nmemb;
 619 }
 620
 621 static int _http_seek(void *stream, curl_off_t offset, int origin)
 622 {
 623         struct http_curl_stream *state = stream;
 624         if (offset < state->max && origin == SEEK_SET) {
 625                 state->pos = offset;
 626                 return CURL_SEEKFUNC_OK;
 627         } else
 628                 return CURL_SEEKFUNC_FAIL;
 629 }
 630
 631 static enum fio_q_status fio_http_queue(struct thread_data *td,
 632                                          struct io_u *io_u)
 633 {
 634         struct http_data *http = td->io_ops_data;
 635         struct http_options *o = td->eo;
 636         struct http_curl_stream _curl_stream;
 637         struct curl_slist *slist = NULL;
 638         char object[512];
 639         char url[1024];
 640         long status;
 641         CURLcode res;
 642         int r = -1;
 643
 644         fio_ro_check(td, io_u);
 645         memset(&_curl_stream, 0, sizeof(_curl_stream));
 646         snprintf(object, sizeof(object), "%s_%llu_%llu", td->files[0]->file_name,
 647                 io_u->offset, io_u->xfer_buflen);
 648         if (o->https == FIO_HTTPS_OFF)
 649                 snprintf(url, sizeof(url), "http://%s%s", o->host, object);
 650         else
 651                 snprintf(url, sizeof(url), "https://%s%s", o->host, object);
 652         curl_easy_setopt(http->curl, CURLOPT_URL, url);
 653         _curl_stream.buf = io_u->xfer_buf;
 654         _curl_stream.max = io_u->xfer_buflen;
 655         curl_easy_setopt(http->curl, CURLOPT_SEEKDATA, &_curl_stream);
 656         curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)io_u->xfer_buflen);
 657
 658         if (o->mode == FIO_HTTP_S3)
 659                 _add_aws_auth_header(http->curl, slist, o, io_u->ddir, object,
 660                         io_u->xfer_buf, io_u->xfer_buflen);
 661         else if (o->mode == FIO_HTTP_SWIFT)
 662                 _add_swift_header(http->curl, slist, o, io_u->ddir, object,
 663                         io_u->xfer_buf, io_u->xfer_buflen);
 664
 665         if (io_u->ddir == DDIR_WRITE) {
 666                 curl_easy_setopt(http->curl, CURLOPT_READDATA, &_curl_stream);
 667                 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
 668                 curl_easy_setopt(http->curl, CURLOPT_UPLOAD, 1L);
 669                 res = curl_easy_perform(http->curl);
 670                 if (res == CURLE_OK) {
 671                         curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
 672                         if (status == 100 || (status >= 200 && status <= 204))
 673                                 goto out;
 674                         log_err("DDIR_WRITE failed with HTTP status code %ld\n", status);
 675                 }
 676                 goto err;
 677         } else if (io_u->ddir == DDIR_READ) {
 678                 curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
 679                 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, &_curl_stream);
 680                 curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
 681                 res = curl_easy_perform(http->curl);
 682                 if (res == CURLE_OK) {
 683                         curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
 684                         if (status == 200)
 685                                 goto out;
 686                         else if (status == 404) {
 687                                 /* Object doesn't exist. Pretend we read
 688                                  * zeroes */
 689                                 memset(io_u->xfer_buf, 0, io_u->xfer_buflen);
 690                                 goto out;
 691                         }
 692                         log_err("DDIR_READ failed with HTTP status code %ld\n", status);
 693                 }
 694                 goto err;
 695         } else if (io_u->ddir == DDIR_TRIM) {
 696                 curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
 697                 curl_easy_setopt(http->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
 698                 curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)0);
 699                 curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
 700                 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
 701                 res = curl_easy_perform(http->curl);
 702                 if (res == CURLE_OK) {
 703                         curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
 704                         if (status == 200 || status == 202 || status == 204 || status == 404)
 705                                 goto out;
 706                         log_err("DDIR_TRIM failed with HTTP status code %ld\n", status);
 707                 }
 708                 goto err;
 709         }
 710
 711         log_err("WARNING: Only DDIR_READ/DDIR_WRITE/DDIR_TRIM are supported!\n");
 712
 713 err:
 714         io_u->error = r;
 715         td_verror(td, io_u->error, "transfer");
 716 out:
 717         curl_slist_free_all(slist);
 718         return FIO_Q_COMPLETED;
 719 }
 720
 721 static struct io_u *fio_http_event(struct thread_data *td, int event)
 722 {
 723         /* sync IO engine - never any outstanding events */
 724         return NULL;
 725 }
 726
 727 int fio_http_getevents(struct thread_data *td, unsigned int min,
 728         unsigned int max, const struct timespec *t)
 729 {
 730         /* sync IO engine - never any outstanding events */
 731         return 0;
 732 }
 733
 734 static int fio_http_setup(struct thread_data *td)
 735 {
 736         struct http_data *http = NULL;
 737         struct http_options *o = td->eo;
 738
 739         /* allocate engine specific structure to deal with libhttp. */
 740         http = calloc(1, sizeof(*http));
 741         if (!http) {
 742                 log_err("calloc failed.\n");
 743                 goto cleanup;
 744         }
 745
 746         http->curl = curl_easy_init();
 747         if (o->verbose)
 748                 curl_easy_setopt(http->curl, CURLOPT_VERBOSE, 1L);
 749         if (o->verbose > 1)
 750                 curl_easy_setopt(http->curl, CURLOPT_DEBUGFUNCTION, &_curl_trace);
 751         curl_easy_setopt(http->curl, CURLOPT_NOPROGRESS, 1L);
 752         curl_easy_setopt(http->curl, CURLOPT_FOLLOWLOCATION, 1L);
 753         curl_easy_setopt(http->curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP|CURLPROTO_HTTPS);
 754         if (o->https == FIO_HTTPS_INSECURE) {
 755                 curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYPEER, 0L);
 756                 curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYHOST, 0L);
 757         }
 758         curl_easy_setopt(http->curl, CURLOPT_READFUNCTION, _http_read);
 759         curl_easy_setopt(http->curl, CURLOPT_WRITEFUNCTION, _http_write);
 760         curl_easy_setopt(http->curl, CURLOPT_SEEKFUNCTION, &_http_seek);
 761         if (o->user && o->pass) {
 762                 curl_easy_setopt(http->curl, CURLOPT_USERNAME, o->user);
 763                 curl_easy_setopt(http->curl, CURLOPT_PASSWORD, o->pass);
 764                 curl_easy_setopt(http->curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
 765         }
 766
 767         td->io_ops_data = http;
 768
 769         /* Force single process mode. */
 770         td->o.use_thread = 1;
 771
 772         return 0;
 773 cleanup:
 774         fio_http_cleanup(td);
 775         return 1;
 776 }
 777
 778 static int fio_http_open(struct thread_data *td, struct fio_file *f)
 779 {
 780         return 0;
 781 }
 782 static int fio_http_invalidate(struct thread_data *td, struct fio_file *f)
 783 {
 784         return 0;
 785 }
 786
 787 FIO_STATIC struct ioengine_ops ioengine = {
 788         .name = "http",
 789         .version                = FIO_IOOPS_VERSION,
 790         .flags                  = FIO_DISKLESSIO | FIO_SYNCIO,
 791         .setup                  = fio_http_setup,
 792         .queue                  = fio_http_queue,
 793         .getevents              = fio_http_getevents,
 794         .event                  = fio_http_event,
 795         .cleanup                = fio_http_cleanup,
 796         .open_file              = fio_http_open,
 797         .invalidate             = fio_http_invalidate,
 798         .options                = options,
 799         .option_struct_size     = sizeof(struct http_options),
 800 };
 801
 802 static void fio_init fio_http_register(void)
 803 {
 804         register_ioengine(&ioengine);
 805 }
 806
 807 static void fio_exit fio_http_unregister(void)
 808 {
 809         unregister_ioengine(&ioengine);
 810 }