engines/http.c

   1 /*
   2  * HTTP GET/PUT IO engine
   3  *
   4  * IO engine to perform HTTP(S) GET/PUT requests via libcurl-easy.
   5  *
   6  * Copyright (C) 2018 SUSE LLC
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License,
  10  * version 2 as published by the Free Software Foundation..
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public
  18  * License along with this program; if not, write to the Free
  19  * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  20  * Boston, MA 02110-1301, USA.
  21  */
  22
  23 #include <pthread.h>
  24 #include <time.h>
  25 #include <curl/curl.h>
  26 #include <openssl/hmac.h>
  27 #include <openssl/sha.h>
  28 #include <openssl/md5.h>
  29 #include "fio.h"
  30 #include "../optgroup.h"
  31
  32 /*
  33  * Silence OpenSSL 3.0 deprecated function warnings
  34  */
  35 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
  36
  37 enum {
  38         FIO_HTTP_WEBDAV     = 0,
  39         FIO_HTTP_S3         = 1,
  40         FIO_HTTP_SWIFT      = 2,
  41
  42         FIO_HTTPS_OFF       = 0,
  43         FIO_HTTPS_ON        = 1,
  44         FIO_HTTPS_INSECURE  = 2,
  45 };
  46
  47 struct http_data {
  48         CURL *curl;
  49 };
  50
  51 struct http_options {
  52         void *pad;
  53         unsigned int https;
  54         char *host;
  55         char *user;
  56         char *pass;
  57         char *s3_key;
  58         char *s3_keyid;
  59         char *s3_region;
  60         char *s3_sse_customer_key;
  61         char *s3_sse_customer_algorithm;
  62         char *s3_storage_class;
  63         char *swift_auth_token;
  64         int verbose;
  65         unsigned int mode;
  66 };
  67
  68 struct http_curl_stream {
  69         char *buf;
  70         size_t pos;
  71         size_t max;
  72 };
  73
  74 static struct fio_option options[] = {
  75         {
  76                 .name     = "https",
  77                 .lname    = "https",
  78                 .type     = FIO_OPT_STR,
  79                 .help     = "Enable https",
  80                 .off1     = offsetof(struct http_options, https),
  81                 .def      = "off",
  82                 .posval = {
  83                           { .ival = "off",
  84                             .oval = FIO_HTTPS_OFF,
  85                             .help = "No HTTPS",
  86                           },
  87                           { .ival = "on",
  88                             .oval = FIO_HTTPS_ON,
  89                             .help = "Enable HTTPS",
  90                           },
  91                           { .ival = "insecure",
  92                             .oval = FIO_HTTPS_INSECURE,
  93                             .help = "Enable HTTPS, disable peer verification",
  94                           },
  95                 },
  96                 .category = FIO_OPT_C_ENGINE,
  97                 .group    = FIO_OPT_G_HTTP,
  98         },
  99         {
 100                 .name     = "http_host",
 101                 .lname    = "http_host",
 102                 .type     = FIO_OPT_STR_STORE,
 103                 .help     = "Hostname (S3 bucket)",
 104                 .off1     = offsetof(struct http_options, host),
 105                 .def      = "localhost",
 106                 .category = FIO_OPT_C_ENGINE,
 107                 .group    = FIO_OPT_G_HTTP,
 108         },
 109         {
 110                 .name     = "http_user",
 111                 .lname    = "http_user",
 112                 .type     = FIO_OPT_STR_STORE,
 113                 .help     = "HTTP user name",
 114                 .off1     = offsetof(struct http_options, user),
 115                 .category = FIO_OPT_C_ENGINE,
 116                 .group    = FIO_OPT_G_HTTP,
 117         },
 118         {
 119                 .name     = "http_pass",
 120                 .lname    = "http_pass",
 121                 .type     = FIO_OPT_STR_STORE,
 122                 .help     = "HTTP password",
 123                 .off1     = offsetof(struct http_options, pass),
 124                 .category = FIO_OPT_C_ENGINE,
 125                 .group    = FIO_OPT_G_HTTP,
 126         },
 127         {
 128                 .name     = "http_s3_key",
 129                 .lname    = "S3 secret key",
 130                 .type     = FIO_OPT_STR_STORE,
 131                 .help     = "S3 secret key",
 132                 .off1     = offsetof(struct http_options, s3_key),
 133                 .def      = "",
 134                 .category = FIO_OPT_C_ENGINE,
 135                 .group    = FIO_OPT_G_HTTP,
 136         },
 137         {
 138                 .name     = "http_s3_keyid",
 139                 .lname    = "S3 key id",
 140                 .type     = FIO_OPT_STR_STORE,
 141                 .help     = "S3 key id",
 142                 .off1     = offsetof(struct http_options, s3_keyid),
 143                 .def      = "",
 144                 .category = FIO_OPT_C_ENGINE,
 145                 .group    = FIO_OPT_G_HTTP,
 146         },
 147         {
 148                 .name     = "http_swift_auth_token",
 149                 .lname    = "Swift auth token",
 150                 .type     = FIO_OPT_STR_STORE,
 151                 .help     = "OpenStack Swift auth token",
 152                 .off1     = offsetof(struct http_options, swift_auth_token),
 153                 .def      = "",
 154                 .category = FIO_OPT_C_ENGINE,
 155                 .group    = FIO_OPT_G_HTTP,
 156         },
 157         {
 158                 .name     = "http_s3_region",
 159                 .lname    = "S3 region",
 160                 .type     = FIO_OPT_STR_STORE,
 161                 .help     = "S3 region",
 162                 .off1     = offsetof(struct http_options, s3_region),
 163                 .def      = "us-east-1",
 164                 .category = FIO_OPT_C_ENGINE,
 165                 .group    = FIO_OPT_G_HTTP,
 166         },
 167         {
 168                 .name     = "http_s3_sse_customer_key",
 169                 .lname    = "SSE Customer Key",
 170                 .type     = FIO_OPT_STR_STORE,
 171                 .help     = "S3 SSE Customer Key",
 172                 .off1     = offsetof(struct http_options, s3_sse_customer_key),
 173                 .def      = "",
 174                 .category = FIO_OPT_C_ENGINE,
 175                 .group    = FIO_OPT_G_HTTP,
 176         },
 177         {
 178                 .name     = "http_s3_sse_customer_algorithm",
 179                 .lname    = "SSE Customer Algorithm",
 180                 .type     = FIO_OPT_STR_STORE,
 181                 .help     = "S3 SSE Customer Algorithm",
 182                 .off1     = offsetof(struct http_options, s3_sse_customer_algorithm),
 183                 .def      = "AES256",
 184                 .category = FIO_OPT_C_ENGINE,
 185                 .group    = FIO_OPT_G_HTTP,
 186         },
 187         {
 188                 .name     = "http_s3_storage_class",
 189                 .lname    = "S3 Storage class",
 190                 .type     = FIO_OPT_STR_STORE,
 191                 .help     = "S3 Storage Class",
 192                 .off1     = offsetof(struct http_options, s3_storage_class),
 193                 .def      = "STANDARD",
 194                 .category = FIO_OPT_C_ENGINE,
 195                 .group    = FIO_OPT_G_HTTP,
 196         },
 197         {
 198                 .name     = "http_mode",
 199                 .lname    = "Request mode to use",
 200                 .type     = FIO_OPT_STR,
 201                 .help     = "Whether to use WebDAV, Swift, or S3",
 202                 .off1     = offsetof(struct http_options, mode),
 203                 .def      = "webdav",
 204                 .posval = {
 205                           { .ival = "webdav",
 206                             .oval = FIO_HTTP_WEBDAV,
 207                             .help = "WebDAV server",
 208                           },
 209                           { .ival = "s3",
 210                             .oval = FIO_HTTP_S3,
 211                             .help = "S3 storage backend",
 212                           },
 213                           { .ival = "swift",
 214                             .oval = FIO_HTTP_SWIFT,
 215                             .help = "OpenStack Swift storage",
 216                           },
 217                 },
 218                 .category = FIO_OPT_C_ENGINE,
 219                 .group    = FIO_OPT_G_HTTP,
 220         },
 221         {
 222                 .name     = "http_verbose",
 223                 .lname    = "HTTP verbosity level",
 224                 .type     = FIO_OPT_INT,
 225                 .help     = "increase http engine verbosity",
 226                 .off1     = offsetof(struct http_options, verbose),
 227                 .def      = "0",
 228                 .category = FIO_OPT_C_ENGINE,
 229                 .group    = FIO_OPT_G_HTTP,
 230         },
 231         {
 232                 .name     = NULL,
 233         },
 234 };
 235
 236 static char *_aws_uriencode(const char *uri)
 237 {
 238         size_t bufsize = 1024;
 239         char *r = malloc(bufsize);
 240         char c;
 241         int i, n;
 242         const char *hex = "0123456789ABCDEF";
 243
 244         if (!r) {
 245                 log_err("malloc failed\n");
 246                 return NULL;
 247         }
 248
 249         n = 0;
 250         for (i = 0; (c = uri[i]); i++) {
 251                 if (n > bufsize-5) {
 252                         log_err("encoding the URL failed\n");
 253                         free(r);
 254                         return NULL;
 255                 }
 256
 257                 if ( (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
 258                 || (c >= '0' && c <= '9') || c == '_' || c == '-'
 259                 || c == '~' || c == '.' || c == '/')
 260                         r[n++] = c;
 261                 else {
 262                         r[n++] = '%';
 263                         r[n++] = hex[(c >> 4 ) & 0xF];
 264                         r[n++] = hex[c & 0xF];
 265                 }
 266         }
 267         r[n++] = 0;
 268         return r;
 269 }
 270
 271 static char *_conv_hex(const unsigned char *p, size_t len)
 272 {
 273         char *r;
 274         int i,n;
 275         const char *hex = "0123456789abcdef";
 276         r = malloc(len * 2 + 1);
 277         n = 0;
 278         for (i = 0; i < len; i++) {
 279                 r[n++] = hex[(p[i] >> 4 ) & 0xF];
 280                 r[n++] = hex[p[i] & 0xF];
 281         }
 282         r[n] = 0;
 283
 284         return r;
 285 }
 286
 287 static char *_gen_hex_sha256(const char *p, size_t len)
 288 {
 289         unsigned char hash[SHA256_DIGEST_LENGTH];
 290
 291         SHA256((unsigned char*)p, len, hash);
 292         return _conv_hex(hash, SHA256_DIGEST_LENGTH);
 293 }
 294
 295 static char *_gen_hex_md5(const char *p, size_t len)
 296 {
 297         unsigned char hash[MD5_DIGEST_LENGTH];
 298
 299         MD5((unsigned char*)p, len, hash);
 300         return _conv_hex(hash, MD5_DIGEST_LENGTH);
 301 }
 302
 303 static char *_conv_base64_encode(const unsigned char *p, size_t len)
 304 {
 305         char *r, *ret;
 306         int i;
 307         static const char sEncodingTable[] = {
 308                 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
 309                 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
 310                 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
 311                 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
 312                 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
 313                 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
 314                 'w', 'x', 'y', 'z', '0', '1', '2', '3',
 315                 '4', '5', '6', '7', '8', '9', '+', '/'
 316         };
 317
 318         size_t out_len = 4 * ((len + 2) / 3);
 319         ret = r = malloc(out_len + 1);
 320
 321         for (i = 0; i < len - 2; i += 3) {
 322                 *r++ = sEncodingTable[(p[i] >> 2) & 0x3F];
 323                 *r++ = sEncodingTable[((p[i] & 0x3) << 4) | ((int) (p[i + 1] & 0xF0) >> 4)];
 324                 *r++ = sEncodingTable[((p[i + 1] & 0xF) << 2) | ((int) (p[i + 2] & 0xC0) >> 6)];
 325                 *r++ = sEncodingTable[p[i + 2] & 0x3F];
 326         }
 327
 328         if (i < len) {
 329                 *r++ = sEncodingTable[(p[i] >> 2) & 0x3F];
 330                 if (i == (len - 1)) {
 331                         *r++ = sEncodingTable[((p[i] & 0x3) << 4)];
 332                         *r++ = '=';
 333                 } else {
 334                         *r++ = sEncodingTable[((p[i] & 0x3) << 4) | ((int) (p[i + 1] & 0xF0) >> 4)];
 335                         *r++ = sEncodingTable[((p[i + 1] & 0xF) << 2)];
 336                 }
 337                 *r++ = '=';
 338         }
 339
 340         ret[out_len]=0;
 341         return ret;
 342 }
 343
 344 static char *_gen_base64_md5(const unsigned char *p, size_t len)
 345 {
 346         unsigned char hash[MD5_DIGEST_LENGTH];
 347         MD5((unsigned char*)p, len, hash);
 348         return _conv_base64_encode(hash, MD5_DIGEST_LENGTH);
 349 }
 350
 351 static void _hmac(unsigned char *md, void *key, int key_len, char *data) {
 352 #ifndef CONFIG_HAVE_OPAQUE_HMAC_CTX
 353         HMAC_CTX _ctx;
 354 #endif
 355         HMAC_CTX *ctx;
 356         unsigned int hmac_len;
 357
 358 #ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX
 359         ctx = HMAC_CTX_new();
 360 #else
 361         ctx = &_ctx;
 362         /* work-around crash in certain versions of libssl */
 363         HMAC_CTX_init(ctx);
 364 #endif
 365         HMAC_Init_ex(ctx, key, key_len, EVP_sha256(), NULL);
 366         HMAC_Update(ctx, (unsigned char*)data, strlen(data));
 367         HMAC_Final(ctx, md, &hmac_len);
 368 #ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX
 369         HMAC_CTX_free(ctx);
 370 #else
 371         HMAC_CTX_cleanup(ctx);
 372 #endif
 373 }
 374
 375 static int _curl_trace(CURL *handle, curl_infotype type,
 376              char *data, size_t size,
 377              void *userp)
 378 {
 379         const char *text;
 380         (void)handle; /* prevent compiler warning */
 381         (void)userp;
 382
 383         switch (type) {
 384         case CURLINFO_TEXT:
 385                 fprintf(stderr, "== Info: %s", data);
 386                 fio_fallthrough;
 387         default:
 388         case CURLINFO_SSL_DATA_OUT:
 389         case CURLINFO_SSL_DATA_IN:
 390                 return 0;
 391
 392         case CURLINFO_HEADER_OUT:
 393                 text = "=> Send header";
 394                 break;
 395         case CURLINFO_DATA_OUT:
 396                 text = "=> Send data";
 397                 break;
 398         case CURLINFO_HEADER_IN:
 399                 text = "<= Recv header";
 400                 break;
 401         case CURLINFO_DATA_IN:
 402                 text = "<= Recv data";
 403                 break;
 404         }
 405
 406         log_info("%s: %s", text, data);
 407         return 0;
 408 }
 409
 410 /* https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
 411  * https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
 412  */
 413 static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct http_options *o,
 414                 int op, const char *uri, char *buf, size_t len)
 415 {
 416         char date_short[16];
 417         char date_iso[32];
 418         char method[8];
 419         char dkey[128];
 420         char creq[4096];
 421         char sts[512];
 422         char s[512];
 423         char *uri_encoded = NULL;
 424         char *dsha = NULL;
 425         char *csha = NULL;
 426         char *signature = NULL;
 427         const char *service = "s3";
 428         const char *aws = "aws4_request";
 429         unsigned char md[SHA256_DIGEST_LENGTH];
 430         unsigned char sse_key[33] = {0};
 431         char *sse_key_base64 = NULL;
 432         char *sse_key_md5_base64 = NULL;
 433
 434         time_t t = time(NULL);
 435         struct tm *gtm = gmtime(&t);
 436
 437         strftime (date_short, sizeof(date_short), "%Y%m%d", gtm);
 438         strftime (date_iso, sizeof(date_iso), "%Y%m%dT%H%M%SZ", gtm);
 439         uri_encoded = _aws_uriencode(uri);
 440
 441         if (o->s3_sse_customer_key != NULL)
 442                 strncpy((char*)sse_key, o->s3_sse_customer_key, sizeof(sse_key) - 1);
 443
 444         if (op == DDIR_WRITE) {
 445                 dsha = _gen_hex_sha256(buf, len);
 446                 sprintf(method, "PUT");
 447         } else {
 448                 /* DDIR_READ && DDIR_TRIM supply an empty body */
 449                 if (op == DDIR_READ)
 450                         sprintf(method, "GET");
 451                 else
 452                         sprintf(method, "DELETE");
 453                 dsha = _gen_hex_sha256("", 0);
 454         }
 455
 456         /* Create the canonical request first */
 457         if (sse_key[0] != '\0') {
 458                 sse_key_base64 = _conv_base64_encode(sse_key, sizeof(sse_key) - 1);
 459                 sse_key_md5_base64 = _gen_base64_md5(sse_key, sizeof(sse_key) - 1);
 460                 snprintf(creq, sizeof(creq),
 461                         "%s\n"
 462                         "%s\n"
 463                         "\n"
 464                         "host:%s\n"
 465                         "x-amz-content-sha256:%s\n"
 466                         "x-amz-date:%s\n"
 467                         "x-amz-server-side-encryption-customer-algorithm:%s\n"
 468                         "x-amz-server-side-encryption-customer-key:%s\n"
 469                         "x-amz-server-side-encryption-customer-key-md5:%s\n"
 470                         "x-amz-storage-class:%s\n"
 471                         "\n"
 472                         "host;x-amz-content-sha256;x-amz-date;"
 473                         "x-amz-server-side-encryption-customer-algorithm;"
 474                         "x-amz-server-side-encryption-customer-key;"
 475                         "x-amz-server-side-encryption-customer-key-md5;"
 476                         "x-amz-storage-class\n"
 477                         "%s"
 478                         , method
 479                         , uri_encoded, o->host, dsha, date_iso
 480                         , o->s3_sse_customer_algorithm, sse_key_base64
 481                         , sse_key_md5_base64, o->s3_storage_class, dsha);
 482         } else {
 483                 snprintf(creq, sizeof(creq),
 484                         "%s\n"
 485                         "%s\n"
 486                         "\n"
 487                         "host:%s\n"
 488                         "x-amz-content-sha256:%s\n"
 489                         "x-amz-date:%s\n"
 490                         "x-amz-storage-class:%s\n"
 491                         "\n"
 492                         "host;x-amz-content-sha256;x-amz-date;x-amz-storage-class\n"
 493                         "%s"
 494                         , method
 495                         , uri_encoded, o->host, dsha, date_iso, o->s3_storage_class, dsha);
 496         }
 497
 498         csha = _gen_hex_sha256(creq, strlen(creq));
 499         snprintf(sts, sizeof(sts), "AWS4-HMAC-SHA256\n%s\n%s/%s/%s/%s\n%s",
 500                         date_iso, date_short, o->s3_region, service, aws, csha);
 501
 502         snprintf((char *)dkey, sizeof(dkey), "AWS4%s", o->s3_key);
 503         _hmac(md, dkey, strlen(dkey), date_short);
 504         _hmac(md, md, SHA256_DIGEST_LENGTH, o->s3_region);
 505         _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) service);
 506         _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) aws);
 507         _hmac(md, md, SHA256_DIGEST_LENGTH, sts);
 508
 509         signature = _conv_hex(md, SHA256_DIGEST_LENGTH);
 510
 511         /* Suppress automatic Accept: header */
 512         slist = curl_slist_append(slist, "Accept:");
 513
 514         snprintf(s, sizeof(s), "x-amz-content-sha256: %s", dsha);
 515         slist = curl_slist_append(slist, s);
 516
 517         snprintf(s, sizeof(s), "x-amz-date: %s", date_iso);
 518         slist = curl_slist_append(slist, s);
 519
 520         if (sse_key[0] != '\0') {
 521                 snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-algorithm: %s", o->s3_sse_customer_algorithm);
 522                 slist = curl_slist_append(slist, s);
 523                 snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-key: %s", sse_key_base64);
 524                 slist = curl_slist_append(slist, s);
 525                 snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-key-md5: %s", sse_key_md5_base64);
 526                 slist = curl_slist_append(slist, s);
 527         }
 528
 529         snprintf(s, sizeof(s), "x-amz-storage-class: %s", o->s3_storage_class);
 530         slist = curl_slist_append(slist, s);
 531
 532         if (sse_key[0] != '\0') {
 533                 snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
 534                         "SignedHeaders=host;x-amz-content-sha256;"
 535                         "x-amz-date;x-amz-server-side-encryption-customer-algorithm;"
 536                         "x-amz-server-side-encryption-customer-key;"
 537                         "x-amz-server-side-encryption-customer-key-md5;"
 538                         "x-amz-storage-class,"
 539                         "Signature=%s",
 540                 o->s3_keyid, date_short, o->s3_region, signature);
 541         } else {
 542                 snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
 543                         "SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=%s",
 544                         o->s3_keyid, date_short, o->s3_region, signature);
 545         }
 546         slist = curl_slist_append(slist, s);
 547
 548         curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
 549
 550         free(uri_encoded);
 551         free(csha);
 552         free(dsha);
 553         free(signature);
 554         if (sse_key_base64 != NULL) {
 555                 free(sse_key_base64);
 556                 free(sse_key_md5_base64);
 557         }
 558 }
 559
 560 static void _add_swift_header(CURL *curl, struct curl_slist *slist, struct http_options *o,
 561                 int op, const char *uri, char *buf, size_t len)
 562 {
 563         char *dsha = NULL;
 564         char s[512];
 565
 566         if (op == DDIR_WRITE) {
 567                 dsha = _gen_hex_md5(buf, len);
 568         }
 569         /* Suppress automatic Accept: header */
 570         slist = curl_slist_append(slist, "Accept:");
 571
 572         snprintf(s, sizeof(s), "etag: %s", dsha);
 573         slist = curl_slist_append(slist, s);
 574
 575         snprintf(s, sizeof(s), "x-auth-token: %s", o->swift_auth_token);
 576         slist = curl_slist_append(slist, s);
 577
 578         curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
 579
 580         free(dsha);
 581 }
 582
 583 static void fio_http_cleanup(struct thread_data *td)
 584 {
 585         struct http_data *http = td->io_ops_data;
 586
 587         if (http) {
 588                 curl_easy_cleanup(http->curl);
 589                 free(http);
 590         }
 591 }
 592
 593 static size_t _http_read(void *ptr, size_t size, size_t nmemb, void *stream)
 594 {
 595         struct http_curl_stream *state = stream;
 596         size_t len = size * nmemb;
 597         /* We're retrieving; nothing is supposed to be read locally */
 598         if (!stream)
 599                 return 0;
 600         if (len+state->pos > state->max)
 601                 len = state->max - state->pos;
 602         memcpy(ptr, &state->buf[state->pos], len);
 603         state->pos += len;
 604         return len;
 605 }
 606
 607 static size_t _http_write(void *ptr, size_t size, size_t nmemb, void *stream)
 608 {
 609         struct http_curl_stream *state = stream;
 610         /* We're just discarding the returned body after a PUT */
 611         if (!stream)
 612                 return nmemb;
 613         if (size != 1)
 614                 return CURLE_WRITE_ERROR;
 615         if (nmemb + state->pos > state->max)
 616                 return CURLE_WRITE_ERROR;
 617         memcpy(&state->buf[state->pos], ptr, nmemb);
 618         state->pos += nmemb;
 619         return nmemb;
 620 }
 621
 622 static int _http_seek(void *stream, curl_off_t offset, int origin)
 623 {
 624         struct http_curl_stream *state = stream;
 625         if (offset < state->max && origin == SEEK_SET) {
 626                 state->pos = offset;
 627                 return CURL_SEEKFUNC_OK;
 628         } else
 629                 return CURL_SEEKFUNC_FAIL;
 630 }
 631
 632 static enum fio_q_status fio_http_queue(struct thread_data *td,
 633                                          struct io_u *io_u)
 634 {
 635         struct http_data *http = td->io_ops_data;
 636         struct http_options *o = td->eo;
 637         struct http_curl_stream _curl_stream;
 638         struct curl_slist *slist = NULL;
 639         char object[512];
 640         char url[1024];
 641         long status;
 642         CURLcode res;
 643         int r = -1;
 644
 645         fio_ro_check(td, io_u);
 646         memset(&_curl_stream, 0, sizeof(_curl_stream));
 647         snprintf(object, sizeof(object), "%s_%llu_%llu", td->files[0]->file_name,
 648                 io_u->offset, io_u->xfer_buflen);
 649         if (o->https == FIO_HTTPS_OFF)
 650                 snprintf(url, sizeof(url), "http://%s%s", o->host, object);
 651         else
 652                 snprintf(url, sizeof(url), "https://%s%s", o->host, object);
 653         curl_easy_setopt(http->curl, CURLOPT_URL, url);
 654         _curl_stream.buf = io_u->xfer_buf;
 655         _curl_stream.max = io_u->xfer_buflen;
 656         curl_easy_setopt(http->curl, CURLOPT_SEEKDATA, &_curl_stream);
 657         curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)io_u->xfer_buflen);
 658
 659         if (o->mode == FIO_HTTP_S3)
 660                 _add_aws_auth_header(http->curl, slist, o, io_u->ddir, object,
 661                         io_u->xfer_buf, io_u->xfer_buflen);
 662         else if (o->mode == FIO_HTTP_SWIFT)
 663                 _add_swift_header(http->curl, slist, o, io_u->ddir, object,
 664                         io_u->xfer_buf, io_u->xfer_buflen);
 665
 666         if (io_u->ddir == DDIR_WRITE) {
 667                 curl_easy_setopt(http->curl, CURLOPT_READDATA, &_curl_stream);
 668                 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
 669                 curl_easy_setopt(http->curl, CURLOPT_UPLOAD, 1L);
 670                 res = curl_easy_perform(http->curl);
 671                 if (res == CURLE_OK) {
 672                         curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
 673                         if (status == 100 || (status >= 200 && status <= 204))
 674                                 goto out;
 675                         log_err("DDIR_WRITE failed with HTTP status code %ld\n", status);
 676                 }
 677                 goto err;
 678         } else if (io_u->ddir == DDIR_READ) {
 679                 curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
 680                 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, &_curl_stream);
 681                 curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
 682                 res = curl_easy_perform(http->curl);
 683                 if (res == CURLE_OK) {
 684                         curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
 685                         if (status == 200)
 686                                 goto out;
 687                         else if (status == 404) {
 688                                 /* Object doesn't exist. Pretend we read
 689                                  * zeroes */
 690                                 memset(io_u->xfer_buf, 0, io_u->xfer_buflen);
 691                                 goto out;
 692                         }
 693                         log_err("DDIR_READ failed with HTTP status code %ld\n", status);
 694                 }
 695                 goto err;
 696         } else if (io_u->ddir == DDIR_TRIM) {
 697                 curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
 698                 curl_easy_setopt(http->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
 699                 curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)0);
 700                 curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
 701                 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
 702                 res = curl_easy_perform(http->curl);
 703                 if (res == CURLE_OK) {
 704                         curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
 705                         if (status == 200 || status == 202 || status == 204 || status == 404)
 706                                 goto out;
 707                         log_err("DDIR_TRIM failed with HTTP status code %ld\n", status);
 708                 }
 709                 goto err;
 710         }
 711
 712         log_err("WARNING: Only DDIR_READ/DDIR_WRITE/DDIR_TRIM are supported!\n");
 713
 714 err:
 715         io_u->error = r;
 716         td_verror(td, io_u->error, "transfer");
 717 out:
 718         curl_slist_free_all(slist);
 719         return FIO_Q_COMPLETED;
 720 }
 721
 722 static struct io_u *fio_http_event(struct thread_data *td, int event)
 723 {
 724         /* sync IO engine - never any outstanding events */
 725         return NULL;
 726 }
 727
 728 int fio_http_getevents(struct thread_data *td, unsigned int min,
 729         unsigned int max, const struct timespec *t)
 730 {
 731         /* sync IO engine - never any outstanding events */
 732         return 0;
 733 }
 734
 735 static int fio_http_setup(struct thread_data *td)
 736 {
 737         struct http_data *http = NULL;
 738         struct http_options *o = td->eo;
 739
 740         /* allocate engine specific structure to deal with libhttp. */
 741         http = calloc(1, sizeof(*http));
 742         if (!http) {
 743                 log_err("calloc failed.\n");
 744                 goto cleanup;
 745         }
 746
 747         http->curl = curl_easy_init();
 748         if (o->verbose)
 749                 curl_easy_setopt(http->curl, CURLOPT_VERBOSE, 1L);
 750         if (o->verbose > 1)
 751                 curl_easy_setopt(http->curl, CURLOPT_DEBUGFUNCTION, &_curl_trace);
 752         curl_easy_setopt(http->curl, CURLOPT_NOPROGRESS, 1L);
 753         curl_easy_setopt(http->curl, CURLOPT_FOLLOWLOCATION, 1L);
 754         curl_easy_setopt(http->curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP|CURLPROTO_HTTPS);
 755         if (o->https == FIO_HTTPS_INSECURE) {
 756                 curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYPEER, 0L);
 757                 curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYHOST, 0L);
 758         }
 759         curl_easy_setopt(http->curl, CURLOPT_READFUNCTION, _http_read);
 760         curl_easy_setopt(http->curl, CURLOPT_WRITEFUNCTION, _http_write);
 761         curl_easy_setopt(http->curl, CURLOPT_SEEKFUNCTION, &_http_seek);
 762         if (o->user && o->pass) {
 763                 curl_easy_setopt(http->curl, CURLOPT_USERNAME, o->user);
 764                 curl_easy_setopt(http->curl, CURLOPT_PASSWORD, o->pass);
 765                 curl_easy_setopt(http->curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
 766         }
 767
 768         td->io_ops_data = http;
 769
 770         /* Force single process mode. */
 771         td->o.use_thread = 1;
 772
 773         return 0;
 774 cleanup:
 775         fio_http_cleanup(td);
 776         return 1;
 777 }
 778
 779 static int fio_http_open(struct thread_data *td, struct fio_file *f)
 780 {
 781         return 0;
 782 }
 783 static int fio_http_invalidate(struct thread_data *td, struct fio_file *f)
 784 {
 785         return 0;
 786 }
 787
 788 FIO_STATIC struct ioengine_ops ioengine = {
 789         .name = "http",
 790         .version                = FIO_IOOPS_VERSION,
 791         .flags                  = FIO_DISKLESSIO | FIO_SYNCIO,
 792         .setup                  = fio_http_setup,
 793         .queue                  = fio_http_queue,
 794         .getevents              = fio_http_getevents,
 795         .event                  = fio_http_event,
 796         .cleanup                = fio_http_cleanup,
 797         .open_file              = fio_http_open,
 798         .invalidate             = fio_http_invalidate,
 799         .options                = options,
 800         .option_struct_size     = sizeof(struct http_options),
 801 };
 802
 803 static void fio_init fio_http_register(void)
 804 {
 805         register_ioengine(&ioengine);
 806 }
 807
 808 static void fio_exit fio_http_unregister(void)
 809 {
 810         unregister_ioengine(&ioengine);
 811 }