projects / fio.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'master' of https://github.com/celestinechen/fio
[fio.git] / engines / http.c
CommitLineData
c2f6a13d
LMB		 1/*
2 * HTTP GET/PUT IO engine
3 *
4 * IO engine to perform HTTP(S) GET/PUT requests via libcurl-easy.
5 *
6 * Copyright (C) 2018 SUSE LLC
7 *
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License,
10 * version 2 as published by the Free Software Foundation..
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public
18 * License along with this program; if not, write to the Free
19 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
20 * Boston, MA 02110-1301, USA.
21 */
22
23#include <pthread.h>
24#include <time.h>
25#include <curl/curl.h>
26#include <openssl/hmac.h>
27#include <openssl/sha.h>
09fd2966 28#include <openssl/md5.h>
c2f6a13d
LMB		 29#include "fio.h"
30#include "../optgroup.h"
31
9c1c1a8d
JA		 32/*
33 * Silence OpenSSL 3.0 deprecated function warnings
34 */
35#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
c2f6a13d 36
09fd2966
LMB		 37enum {
38 FIO_HTTP_WEBDAV = 0,
39 FIO_HTTP_S3 = 1,
40 FIO_HTTP_SWIFT = 2,
41
42 FIO_HTTPS_OFF = 0,
43 FIO_HTTPS_ON = 1,
44 FIO_HTTPS_INSECURE = 2,
45};
46
c2f6a13d
LMB		 47struct http_data {
48 CURL *curl;
49};
50
51struct http_options {
52 void *pad;
09fd2966 53 unsigned int https;
c2f6a13d
LMB		 54 char *host;
55 char *user;
56 char *pass;
57 char *s3_key;
58 char *s3_keyid;
59 char *s3_region;
91af79c4
FH		 60 char *s3_sse_customer_key;
61 char *s3_sse_customer_algorithm;
f2d79184 62 char *s3_storage_class;
09fd2966 63 char *swift_auth_token;
c2f6a13d 64 int verbose;
09fd2966 65 unsigned int mode;
c2f6a13d
LMB		 66};
67
68struct http_curl_stream {
69 char *buf;
70 size_t pos;
71 size_t max;
72};
73
74static struct fio_option options[] = {
75 {
76 .name = "https",
77 .lname = "https",
09fd2966 78 .type = FIO_OPT_STR,
c2f6a13d
LMB		 79 .help = "Enable https",
80 .off1 = offsetof(struct http_options, https),
09fd2966
LMB		 81 .def = "off",
82 .posval = {
83 { .ival = "off",
84 .oval = FIO_HTTPS_OFF,
85 .help = "No HTTPS",
86 },
87 { .ival = "on",
88 .oval = FIO_HTTPS_ON,
89 .help = "Enable HTTPS",
90 },
91 { .ival = "insecure",
92 .oval = FIO_HTTPS_INSECURE,
93 .help = "Enable HTTPS, disable peer verification",
94 },
95 },
c2f6a13d
LMB		 96 .category = FIO_OPT_C_ENGINE,
97 .group = FIO_OPT_G_HTTP,
98 },
99 {
100 .name = "http_host",
101 .lname = "http_host",
102 .type = FIO_OPT_STR_STORE,
103 .help = "Hostname (S3 bucket)",
104 .off1 = offsetof(struct http_options, host),
105 .def = "localhost",
106 .category = FIO_OPT_C_ENGINE,
107 .group = FIO_OPT_G_HTTP,
108 },
109 {
110 .name = "http_user",
111 .lname = "http_user",
112 .type = FIO_OPT_STR_STORE,
113 .help = "HTTP user name",
114 .off1 = offsetof(struct http_options, user),
115 .category = FIO_OPT_C_ENGINE,
116 .group = FIO_OPT_G_HTTP,
117 },
118 {
119 .name = "http_pass",
120 .lname = "http_pass",
121 .type = FIO_OPT_STR_STORE,
122 .help = "HTTP password",
123 .off1 = offsetof(struct http_options, pass),
124 .category = FIO_OPT_C_ENGINE,
125 .group = FIO_OPT_G_HTTP,
126 },
127 {
128 .name = "http_s3_key",
129 .lname = "S3 secret key",
130 .type = FIO_OPT_STR_STORE,
131 .help = "S3 secret key",
132 .off1 = offsetof(struct http_options, s3_key),
133 .def = "",
134 .category = FIO_OPT_C_ENGINE,
135 .group = FIO_OPT_G_HTTP,
136 },
137 {
138 .name = "http_s3_keyid",
139 .lname = "S3 key id",
140 .type = FIO_OPT_STR_STORE,
141 .help = "S3 key id",
142 .off1 = offsetof(struct http_options, s3_keyid),
143 .def = "",
144 .category = FIO_OPT_C_ENGINE,
145 .group = FIO_OPT_G_HTTP,
146 },
09fd2966
LMB		 147 {
148 .name = "http_swift_auth_token",
149 .lname = "Swift auth token",
150 .type = FIO_OPT_STR_STORE,
151 .help = "OpenStack Swift auth token",
152 .off1 = offsetof(struct http_options, swift_auth_token),
153 .def = "",
154 .category = FIO_OPT_C_ENGINE,
155 .group = FIO_OPT_G_HTTP,
156 },
c2f6a13d
LMB		 157 {
158 .name = "http_s3_region",
159 .lname = "S3 region",
160 .type = FIO_OPT_STR_STORE,
161 .help = "S3 region",
162 .off1 = offsetof(struct http_options, s3_region),
163 .def = "us-east-1",
164 .category = FIO_OPT_C_ENGINE,
165 .group = FIO_OPT_G_HTTP,
166 },
91af79c4
FH		 167 {
168 .name = "http_s3_sse_customer_key",
169 .lname = "SSE Customer Key",
170 .type = FIO_OPT_STR_STORE,
171 .help = "S3 SSE Customer Key",
172 .off1 = offsetof(struct http_options, s3_sse_customer_key),
173 .def = "",
174 .category = FIO_OPT_C_ENGINE,
175 .group = FIO_OPT_G_HTTP,
176 },
177 {
178 .name = "http_s3_sse_customer_algorithm",
179 .lname = "SSE Customer Algorithm",
180 .type = FIO_OPT_STR_STORE,
181 .help = "S3 SSE Customer Algorithm",
182 .off1 = offsetof(struct http_options, s3_sse_customer_algorithm),
183 .def = "AES256",
184 .category = FIO_OPT_C_ENGINE,
185 .group = FIO_OPT_G_HTTP,
186 },
f2d79184
FH		 187 {
188 .name = "http_s3_storage_class",
189 .lname = "S3 Storage class",
190 .type = FIO_OPT_STR_STORE,
191 .help = "S3 Storage Class",
192 .off1 = offsetof(struct http_options, s3_storage_class),
193 .def = "STANDARD",
194 .category = FIO_OPT_C_ENGINE,
195 .group = FIO_OPT_G_HTTP,
196 },
c2f6a13d 197 {
09fd2966
LMB		 198 .name = "http_mode",
199 .lname = "Request mode to use",
200 .type = FIO_OPT_STR,
201 .help = "Whether to use WebDAV, Swift, or S3",
202 .off1 = offsetof(struct http_options, mode),
203 .def = "webdav",
204 .posval = {
205 { .ival = "webdav",
206 .oval = FIO_HTTP_WEBDAV,
207 .help = "WebDAV server",
208 },
209 { .ival = "s3",
210 .oval = FIO_HTTP_S3,
211 .help = "S3 storage backend",
212 },
213 { .ival = "swift",
214 .oval = FIO_HTTP_SWIFT,
215 .help = "OpenStack Swift storage",
216 },
217 },
c2f6a13d
LMB		 218 .category = FIO_OPT_C_ENGINE,
219 .group = FIO_OPT_G_HTTP,
220 },
221 {
222 .name = "http_verbose",
09fd2966 223 .lname = "HTTP verbosity level",
c2f6a13d
LMB		 224 .type = FIO_OPT_INT,
225 .help = "increase http engine verbosity",
226 .off1 = offsetof(struct http_options, verbose),
227 .def = "0",
228 .category = FIO_OPT_C_ENGINE,
229 .group = FIO_OPT_G_HTTP,
230 },
231 {
232 .name = NULL,
233 },
234};
235
236static char *_aws_uriencode(const char *uri)
237{
238 size_t bufsize = 1024;
239 char *r = malloc(bufsize);
240 char c;
241 int i, n;
242 const char *hex = "0123456789ABCDEF";
243
244 if (!r) {
245 log_err("malloc failed\n");
246 return NULL;
247 }
248
249 n = 0;
250 for (i = 0; (c = uri[i]); i++) {
251 if (n > bufsize-5) {
252 log_err("encoding the URL failed\n");
9e66b060 253 free(r);
c2f6a13d
LMB		 254 return NULL;
255 }
256
257 if ( (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
258 || (c >= '0' && c <= '9') || c == '_' || c == '-'
259 || c == '~' || c == '.' || c == '/')
260 r[n++] = c;
261 else {
262 r[n++] = '%';
263 r[n++] = hex[(c >> 4 ) & 0xF];
264 r[n++] = hex[c & 0xF];
265 }
266 }
267 r[n++] = 0;
268 return r;
269}
270
271static char *_conv_hex(const unsigned char *p, size_t len)
272{
273 char *r;
274 int i,n;
275 const char *hex = "0123456789abcdef";
276 r = malloc(len * 2 + 1);
277 n = 0;
278 for (i = 0; i < len; i++) {
279 r[n++] = hex[(p[i] >> 4 ) & 0xF];
280 r[n++] = hex[p[i] & 0xF];
281 }
282 r[n] = 0;
283
284 return r;
285}
286
287static char *_gen_hex_sha256(const char *p, size_t len)
288{
289 unsigned char hash[SHA256_DIGEST_LENGTH];
290
291 SHA256((unsigned char*)p, len, hash);
292 return _conv_hex(hash, SHA256_DIGEST_LENGTH);
293}
294
09fd2966
LMB		 295static char *_gen_hex_md5(const char *p, size_t len)
296{
297 unsigned char hash[MD5_DIGEST_LENGTH];
298
299 MD5((unsigned char*)p, len, hash);
300 return _conv_hex(hash, MD5_DIGEST_LENGTH);
301}
302
91af79c4
FH		 303static char *_conv_base64_encode(const unsigned char *p, size_t len)
304{
305 char *r, *ret;
306 int i;
307 static const char sEncodingTable[] = {
308 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
309 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
310 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
311 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
312 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
313 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
314 'w', 'x', 'y', 'z', '0', '1', '2', '3',
315 '4', '5', '6', '7', '8', '9', '+', '/'
316 };
317
318 size_t out_len = 4 * ((len + 2) / 3);
319 ret = r = malloc(out_len + 1);
320
321 for (i = 0; i < len - 2; i += 3) {
322 *r++ = sEncodingTable[(p[i] >> 2) & 0x3F];
323 *r++ = sEncodingTable[((p[i] & 0x3) << 4) | ((int) (p[i + 1] & 0xF0) >> 4)];
324 *r++ = sEncodingTable[((p[i + 1] & 0xF) << 2) | ((int) (p[i + 2] & 0xC0) >> 6)];
325 *r++ = sEncodingTable[p[i + 2] & 0x3F];
326 }
327
328 if (i < len) {
329 *r++ = sEncodingTable[(p[i] >> 2) & 0x3F];
330 if (i == (len - 1)) {
331 *r++ = sEncodingTable[((p[i] & 0x3) << 4)];
332 *r++ = '=';
333 } else {
334 *r++ = sEncodingTable[((p[i] & 0x3) << 4) | ((int) (p[i + 1] & 0xF0) >> 4)];
335 *r++ = sEncodingTable[((p[i + 1] & 0xF) << 2)];
336 }
337 *r++ = '=';
338 }
339
340 ret[out_len]=0;
341 return ret;
342}
343
344static char *_gen_base64_md5(const unsigned char *p, size_t len)
345{
346 unsigned char hash[MD5_DIGEST_LENGTH];
347 MD5((unsigned char*)p, len, hash);
348 return _conv_base64_encode(hash, MD5_DIGEST_LENGTH);
349}
350
c2f6a13d 351static void _hmac(unsigned char *md, void *key, int key_len, char *data) {
b61a5f46
DD		 352#ifndef CONFIG_HAVE_OPAQUE_HMAC_CTX
353 HMAC_CTX _ctx;
354#endif
c2f6a13d
LMB		 355 HMAC_CTX *ctx;
356 unsigned int hmac_len;
357
b61a5f46 358#ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX
c2f6a13d 359 ctx = HMAC_CTX_new();
b61a5f46
DD		 360#else
361 ctx = &_ctx;
e8aaa776
JA		 362 /* work-around crash in certain versions of libssl */
363 HMAC_CTX_init(ctx);
b61a5f46 364#endif
c2f6a13d
LMB		 365 HMAC_Init_ex(ctx, key, key_len, EVP_sha256(), NULL);
366 HMAC_Update(ctx, (unsigned char*)data, strlen(data));
367 HMAC_Final(ctx, md, &hmac_len);
b61a5f46 368#ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX
c2f6a13d 369 HMAC_CTX_free(ctx);
b61a5f46
DD		 370#else
371 HMAC_CTX_cleanup(ctx);
372#endif
c2f6a13d
LMB		 373}
374
375static int _curl_trace(CURL *handle, curl_infotype type,
376 char *data, size_t size,
377 void *userp)
378{
379 const char *text;
380 (void)handle; /* prevent compiler warning */
381 (void)userp;
382
383 switch (type) {
384 case CURLINFO_TEXT:
24010223 385 fprintf(stderr, "== Info: %s", data);
87933e32 386 fio_fallthrough;
c2f6a13d
LMB		 387 default:
388 case CURLINFO_SSL_DATA_OUT:
389 case CURLINFO_SSL_DATA_IN:
390 return 0;
391
392 case CURLINFO_HEADER_OUT:
393 text = "=> Send header";
394 break;
395 case CURLINFO_DATA_OUT:
396 text = "=> Send data";
397 break;
398 case CURLINFO_HEADER_IN:
399 text = "<= Recv header";
400 break;
401 case CURLINFO_DATA_IN:
402 text = "<= Recv data";
403 break;
404 }
405
406 log_info("%s: %s", text, data);
407 return 0;
408}
409
410/* https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
411 * https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
412 */
413static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct http_options *o,
414 int op, const char *uri, char *buf, size_t len)
415{
416 char date_short[16];
417 char date_iso[32];
418 char method[8];
419 char dkey[128];
f2d79184
FH		 420 char creq[4096];
421 char sts[512];
c2f6a13d
LMB		 422 char s[512];
423 char *uri_encoded = NULL;
424 char *dsha = NULL;
425 char *csha = NULL;
426 char *signature = NULL;
427 const char *service = "s3";
428 const char *aws = "aws4_request";
429 unsigned char md[SHA256_DIGEST_LENGTH];
91af79c4
FH		 430 unsigned char sse_key[33] = {0};
431 char *sse_key_base64 = NULL;
432 char *sse_key_md5_base64 = NULL;
c2f6a13d
LMB		 433
434 time_t t = time(NULL);
435 struct tm *gtm = gmtime(&t);
436
437 strftime (date_short, sizeof(date_short), "%Y%m%d", gtm);
438 strftime (date_iso, sizeof(date_iso), "%Y%m%dT%H%M%SZ", gtm);
439 uri_encoded = _aws_uriencode(uri);
440
91af79c4
FH		 441 if (o->s3_sse_customer_key != NULL)
442 strncpy((char*)sse_key, o->s3_sse_customer_key, sizeof(sse_key) - 1);
443
c2f6a13d
LMB		 444 if (op == DDIR_WRITE) {
445 dsha = _gen_hex_sha256(buf, len);
446 sprintf(method, "PUT");
447 } else {
448 /* DDIR_READ && DDIR_TRIM supply an empty body */
449 if (op == DDIR_READ)
450 sprintf(method, "GET");
451 else
452 sprintf(method, "DELETE");
453 dsha = _gen_hex_sha256("", 0);
454 }
455
456 /* Create the canonical request first */
91af79c4
FH		 457 if (sse_key[0] != '\0') {
458 sse_key_base64 = _conv_base64_encode(sse_key, sizeof(sse_key) - 1);
459 sse_key_md5_base64 = _gen_base64_md5(sse_key, sizeof(sse_key) - 1);
460 snprintf(creq, sizeof(creq),
461 "%s\n"
462 "%s\n"
463 "\n"
464 "host:%s\n"
465 "x-amz-content-sha256:%s\n"
466 "x-amz-date:%s\n"
467 "x-amz-server-side-encryption-customer-algorithm:%s\n"
468 "x-amz-server-side-encryption-customer-key:%s\n"
469 "x-amz-server-side-encryption-customer-key-md5:%s\n"
470 "x-amz-storage-class:%s\n"
471 "\n"
472 "host;x-amz-content-sha256;x-amz-date;"
473 "x-amz-server-side-encryption-customer-algorithm;"
474 "x-amz-server-side-encryption-customer-key;"
475 "x-amz-server-side-encryption-customer-key-md5;"
476 "x-amz-storage-class\n"
477 "%s"
478 , method
479 , uri_encoded, o->host, dsha, date_iso
480 , o->s3_sse_customer_algorithm, sse_key_base64
481 , sse_key_md5_base64, o->s3_storage_class, dsha);
482 } else {
483 snprintf(creq, sizeof(creq),
484 "%s\n"
485 "%s\n"
486 "\n"
487 "host:%s\n"
488 "x-amz-content-sha256:%s\n"
489 "x-amz-date:%s\n"
490 "x-amz-storage-class:%s\n"
491 "\n"
492 "host;x-amz-content-sha256;x-amz-date;x-amz-storage-class\n"
493 "%s"
494 , method
495 , uri_encoded, o->host, dsha, date_iso, o->s3_storage_class, dsha);
496 }
c2f6a13d
LMB		 497
498 csha = _gen_hex_sha256(creq, strlen(creq));
499 snprintf(sts, sizeof(sts), "AWS4-HMAC-SHA256\n%s\n%s/%s/%s/%s\n%s",
91af79c4 500 date_iso, date_short, o->s3_region, service, aws, csha);
c2f6a13d
LMB		 501
502 snprintf((char *)dkey, sizeof(dkey), "AWS4%s", o->s3_key);
503 _hmac(md, dkey, strlen(dkey), date_short);
504 _hmac(md, md, SHA256_DIGEST_LENGTH, o->s3_region);
505 _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) service);
506 _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) aws);
507 _hmac(md, md, SHA256_DIGEST_LENGTH, sts);
508
509 signature = _conv_hex(md, SHA256_DIGEST_LENGTH);
510
fc002f14 511 /* Suppress automatic Accept: header */
c2f6a13d
LMB		 512 slist = curl_slist_append(slist, "Accept:");
513
514 snprintf(s, sizeof(s), "x-amz-content-sha256: %s", dsha);
515 slist = curl_slist_append(slist, s);
516
517 snprintf(s, sizeof(s), "x-amz-date: %s", date_iso);
518 slist = curl_slist_append(slist, s);
91af79c4
FH		 519
520 if (sse_key[0] != '\0') {
521 snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-algorithm: %s", o->s3_sse_customer_algorithm);
522 slist = curl_slist_append(slist, s);
523 snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-key: %s", sse_key_base64);
524 slist = curl_slist_append(slist, s);
525 snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-key-md5: %s", sse_key_md5_base64);
526 slist = curl_slist_append(slist, s);
527 }
528
f2d79184
FH		 529 snprintf(s, sizeof(s), "x-amz-storage-class: %s", o->s3_storage_class);
530 slist = curl_slist_append(slist, s);
91af79c4
FH		 531
532 if (sse_key[0] != '\0') {
533 snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
534 "SignedHeaders=host;x-amz-content-sha256;"
535 "x-amz-date;x-amz-server-side-encryption-customer-algorithm;"
536 "x-amz-server-side-encryption-customer-key;"
537 "x-amz-server-side-encryption-customer-key-md5;"
538 "x-amz-storage-class,"
539 "Signature=%s",
540 o->s3_keyid, date_short, o->s3_region, signature);
541 } else {
542 snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
543 "SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=%s",
544 o->s3_keyid, date_short, o->s3_region, signature);
545 }
c2f6a13d
LMB		 546 slist = curl_slist_append(slist, s);
547
548 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
549
550 free(uri_encoded);
551 free(csha);
552 free(dsha);
553 free(signature);
91af79c4
FH		 554 if (sse_key_base64 != NULL) {
555 free(sse_key_base64);
556 free(sse_key_md5_base64);
557 }
c2f6a13d
LMB		 558}
559
09fd2966
LMB		 560static void _add_swift_header(CURL *curl, struct curl_slist *slist, struct http_options *o,
561 int op, const char *uri, char *buf, size_t len)
562{
563 char *dsha = NULL;
564 char s[512];
565
566 if (op == DDIR_WRITE) {
567 dsha = _gen_hex_md5(buf, len);
568 }
fc002f14 569 /* Suppress automatic Accept: header */
09fd2966
LMB		 570 slist = curl_slist_append(slist, "Accept:");
571
572 snprintf(s, sizeof(s), "etag: %s", dsha);
573 slist = curl_slist_append(slist, s);
574
575 snprintf(s, sizeof(s), "x-auth-token: %s", o->swift_auth_token);
576 slist = curl_slist_append(slist, s);
577
578 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
579
580 free(dsha);
581}
582
c2f6a13d
LMB		 583static void fio_http_cleanup(struct thread_data *td)
584{
585 struct http_data *http = td->io_ops_data;
586
587 if (http) {
588 curl_easy_cleanup(http->curl);
589 free(http);
590 }
591}
592
593static size_t _http_read(void *ptr, size_t size, size_t nmemb, void *stream)
594{
595 struct http_curl_stream *state = stream;
596 size_t len = size * nmemb;
597 /* We're retrieving; nothing is supposed to be read locally */
598 if (!stream)
599 return 0;
600 if (len+state->pos > state->max)
601 len = state->max - state->pos;
602 memcpy(ptr, &state->buf[state->pos], len);
603 state->pos += len;
604 return len;
605}
606
607static size_t _http_write(void *ptr, size_t size, size_t nmemb, void *stream)
608{
609 struct http_curl_stream *state = stream;
610 /* We're just discarding the returned body after a PUT */
611 if (!stream)
612 return nmemb;
613 if (size != 1)
614 return CURLE_WRITE_ERROR;
615 if (nmemb + state->pos > state->max)
616 return CURLE_WRITE_ERROR;
617 memcpy(&state->buf[state->pos], ptr, nmemb);
618 state->pos += nmemb;
619 return nmemb;
620}
621
622static int _http_seek(void *stream, curl_off_t offset, int origin)
623{
624 struct http_curl_stream *state = stream;
625 if (offset < state->max && origin == SEEK_SET) {
626 state->pos = offset;
627 return CURL_SEEKFUNC_OK;
628 } else
629 return CURL_SEEKFUNC_FAIL;
630}
631
632static enum fio_q_status fio_http_queue(struct thread_data *td,
633 struct io_u *io_u)
634{
635 struct http_data *http = td->io_ops_data;
636 struct http_options *o = td->eo;
637 struct http_curl_stream _curl_stream;
638 struct curl_slist *slist = NULL;
639 char object[512];
640 char url[1024];
641 long status;
642 CURLcode res;
c2f6a13d
LMB		 643
644 fio_ro_check(td, io_u);
645 memset(&_curl_stream, 0, sizeof(_curl_stream));
09fd2966
LMB		 646 snprintf(object, sizeof(object), "%s_%llu_%llu", td->files[0]->file_name,
647 io_u->offset, io_u->xfer_buflen);
648 if (o->https == FIO_HTTPS_OFF)
649 snprintf(url, sizeof(url), "http://%s%s", o->host, object);
650 else
651 snprintf(url, sizeof(url), "https://%s%s", o->host, object);
c2f6a13d
LMB		 652 curl_easy_setopt(http->curl, CURLOPT_URL, url);
653 _curl_stream.buf = io_u->xfer_buf;
654 _curl_stream.max = io_u->xfer_buflen;
655 curl_easy_setopt(http->curl, CURLOPT_SEEKDATA, &_curl_stream);
656 curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)io_u->xfer_buflen);
657
09fd2966 658 if (o->mode == FIO_HTTP_S3)
c2f6a13d
LMB		 659 _add_aws_auth_header(http->curl, slist, o, io_u->ddir, object,
660 io_u->xfer_buf, io_u->xfer_buflen);
09fd2966
LMB		 661 else if (o->mode == FIO_HTTP_SWIFT)
662 _add_swift_header(http->curl, slist, o, io_u->ddir, object,
663 io_u->xfer_buf, io_u->xfer_buflen);
c2f6a13d
LMB		 664
665 if (io_u->ddir == DDIR_WRITE) {
666 curl_easy_setopt(http->curl, CURLOPT_READDATA, &_curl_stream);
667 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
668 curl_easy_setopt(http->curl, CURLOPT_UPLOAD, 1L);
669 res = curl_easy_perform(http->curl);
670 if (res == CURLE_OK) {
671 curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
672 if (status == 100 || (status >= 200 && status <= 204))
673 goto out;
674 log_err("DDIR_WRITE failed with HTTP status code %ld\n", status);
c2f6a13d 675 }
4a4c21bf 676 goto err;
c2f6a13d
LMB		 677 } else if (io_u->ddir == DDIR_READ) {
678 curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
679 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, &_curl_stream);
680 curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
681 res = curl_easy_perform(http->curl);
682 if (res == CURLE_OK) {
683 curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
684 if (status == 200)
685 goto out;
686 else if (status == 404) {
687 /* Object doesn't exist. Pretend we read
688 * zeroes */
689 memset(io_u->xfer_buf, 0, io_u->xfer_buflen);
690 goto out;
691 }
692 log_err("DDIR_READ failed with HTTP status code %ld\n", status);
693 }
694 goto err;
695 } else if (io_u->ddir == DDIR_TRIM) {
696 curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
697 curl_easy_setopt(http->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
bd915a6b 698 curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)0);
c2f6a13d
LMB		 699 curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
700 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
701 res = curl_easy_perform(http->curl);
702 if (res == CURLE_OK) {
703 curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
704 if (status == 200 || status == 202 || status == 204 || status == 404)
705 goto out;
706 log_err("DDIR_TRIM failed with HTTP status code %ld\n", status);
707 }
708 goto err;
709 }
710
711 log_err("WARNING: Only DDIR_READ/DDIR_WRITE/DDIR_TRIM are supported!\n");
712
713err:
c77fe685 714 io_u->error = EIO;
c2f6a13d
LMB		 715 td_verror(td, io_u->error, "transfer");
716out:
717 curl_slist_free_all(slist);
718 return FIO_Q_COMPLETED;
719}
720
721static struct io_u *fio_http_event(struct thread_data *td, int event)
722{
723 /* sync IO engine - never any outstanding events */
724 return NULL;
725}
726
727int fio_http_getevents(struct thread_data *td, unsigned int min,
728 unsigned int max, const struct timespec *t)
729{
730 /* sync IO engine - never any outstanding events */
731 return 0;
732}
733
734static int fio_http_setup(struct thread_data *td)
735{
736 struct http_data *http = NULL;
737 struct http_options *o = td->eo;
c65f3435 738
c2f6a13d
LMB		 739 /* allocate engine specific structure to deal with libhttp. */
740 http = calloc(1, sizeof(*http));
741 if (!http) {
742 log_err("calloc failed.\n");
743 goto cleanup;
744 }
745
746 http->curl = curl_easy_init();
747 if (o->verbose)
748 curl_easy_setopt(http->curl, CURLOPT_VERBOSE, 1L);
749 if (o->verbose > 1)
750 curl_easy_setopt(http->curl, CURLOPT_DEBUGFUNCTION, &_curl_trace);
751 curl_easy_setopt(http->curl, CURLOPT_NOPROGRESS, 1L);
752 curl_easy_setopt(http->curl, CURLOPT_FOLLOWLOCATION, 1L);
753 curl_easy_setopt(http->curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP|CURLPROTO_HTTPS);
09fd2966
LMB		 754 if (o->https == FIO_HTTPS_INSECURE) {
755 curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYPEER, 0L);
756 curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYHOST, 0L);
757 }
c2f6a13d
LMB		 758 curl_easy_setopt(http->curl, CURLOPT_READFUNCTION, _http_read);
759 curl_easy_setopt(http->curl, CURLOPT_WRITEFUNCTION, _http_write);
bd915a6b 760 curl_easy_setopt(http->curl, CURLOPT_SEEKFUNCTION, &_http_seek);
c2f6a13d
LMB		 761 if (o->user && o->pass) {
762 curl_easy_setopt(http->curl, CURLOPT_USERNAME, o->user);
763 curl_easy_setopt(http->curl, CURLOPT_PASSWORD, o->pass);
764 curl_easy_setopt(http->curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
765 }
766
767 td->io_ops_data = http;
768
769 /* Force single process mode. */
770 td->o.use_thread = 1;
771
772 return 0;
773cleanup:
774 fio_http_cleanup(td);
c65f3435 775 return 1;
c2f6a13d
LMB		 776}
777
778static int fio_http_open(struct thread_data *td, struct fio_file *f)
779{
780 return 0;
781}
782static int fio_http_invalidate(struct thread_data *td, struct fio_file *f)
783{
784 return 0;
785}
786
5a8a6a03 787FIO_STATIC struct ioengine_ops ioengine = {
c2f6a13d
LMB		 788 .name = "http",
789 .version = FIO_IOOPS_VERSION,
f32a30d4 790 .flags = FIO_DISKLESSIO | FIO_SYNCIO,
c2f6a13d
LMB		 791 .setup = fio_http_setup,
792 .queue = fio_http_queue,
793 .getevents = fio_http_getevents,
794 .event = fio_http_event,
795 .cleanup = fio_http_cleanup,
796 .open_file = fio_http_open,
797 .invalidate = fio_http_invalidate,
798 .options = options,
799 .option_struct_size = sizeof(struct http_options),
800};
801
802static void fio_init fio_http_register(void)
803{
804 register_ioengine(&ioengine);
805}
806
807static void fio_exit fio_http_unregister(void)
808{
809 unregister_ioengine(&ioengine);
810}
fio - Flexible IO Tester