Merge branch 'http_older_openssl' of https://github.com/ddiss/fio
[fio.git] / engines / http.c
1 /*
2  * HTTP GET/PUT IO engine
3  *
4  * IO engine to perform HTTP(S) GET/PUT requests via libcurl-easy.
5  *
6  * Copyright (C) 2018 SUSE LLC
7  *
8  * This program is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU General Public License,
10  * version 2 as published by the Free Software Foundation..
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public
18  * License along with this program; if not, write to the Free
19  * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
20  * Boston, MA 02110-1301, USA.
21  */
22
23 #include <pthread.h>
24 #include <time.h>
25 #include <curl/curl.h>
26 #include <openssl/hmac.h>
27 #include <openssl/sha.h>
28 #include "fio.h"
29 #include "../optgroup.h"
30
31
32 struct http_data {
33         CURL *curl;
34 };
35
36 struct http_options {
37         void *pad;
38         int  https;
39         char *host;
40         char *user;
41         char *pass;
42         char *s3_key;
43         char *s3_keyid;
44         char *s3_region;
45         int verbose;
46         int s3;
47 };
48
49 struct http_curl_stream {
50         char *buf;
51         size_t pos;
52         size_t max;
53 };
54
55 static struct fio_option options[] = {
56         {
57                 .name     = "https",
58                 .lname    = "https",
59                 .type     = FIO_OPT_BOOL,
60                 .help     = "Enable https",
61                 .off1     = offsetof(struct http_options, https),
62                 .def      = "0",
63                 .category = FIO_OPT_C_ENGINE,
64                 .group    = FIO_OPT_G_HTTP,
65         },
66         {
67                 .name     = "http_host",
68                 .lname    = "http_host",
69                 .type     = FIO_OPT_STR_STORE,
70                 .help     = "Hostname (S3 bucket)",
71                 .off1     = offsetof(struct http_options, host),
72                 .def      = "localhost",
73                 .category = FIO_OPT_C_ENGINE,
74                 .group    = FIO_OPT_G_HTTP,
75         },
76         {
77                 .name     = "http_user",
78                 .lname    = "http_user",
79                 .type     = FIO_OPT_STR_STORE,
80                 .help     = "HTTP user name",
81                 .off1     = offsetof(struct http_options, user),
82                 .category = FIO_OPT_C_ENGINE,
83                 .group    = FIO_OPT_G_HTTP,
84         },
85         {
86                 .name     = "http_pass",
87                 .lname    = "http_pass",
88                 .type     = FIO_OPT_STR_STORE,
89                 .help     = "HTTP password",
90                 .off1     = offsetof(struct http_options, pass),
91                 .category = FIO_OPT_C_ENGINE,
92                 .group    = FIO_OPT_G_HTTP,
93         },
94         {
95                 .name     = "http_s3_key",
96                 .lname    = "S3 secret key",
97                 .type     = FIO_OPT_STR_STORE,
98                 .help     = "S3 secret key",
99                 .off1     = offsetof(struct http_options, s3_key),
100                 .def      = "",
101                 .category = FIO_OPT_C_ENGINE,
102                 .group    = FIO_OPT_G_HTTP,
103         },
104         {
105                 .name     = "http_s3_keyid",
106                 .lname    = "S3 key id",
107                 .type     = FIO_OPT_STR_STORE,
108                 .help     = "S3 key id",
109                 .off1     = offsetof(struct http_options, s3_keyid),
110                 .def      = "",
111                 .category = FIO_OPT_C_ENGINE,
112                 .group    = FIO_OPT_G_HTTP,
113         },
114         {
115                 .name     = "http_s3_region",
116                 .lname    = "S3 region",
117                 .type     = FIO_OPT_STR_STORE,
118                 .help     = "S3 region",
119                 .off1     = offsetof(struct http_options, s3_region),
120                 .def      = "us-east-1",
121                 .category = FIO_OPT_C_ENGINE,
122                 .group    = FIO_OPT_G_HTTP,
123         },
124         {
125                 .name     = "http_s3",
126                 .lname    = "S3 extensions",
127                 .type     = FIO_OPT_BOOL,
128                 .help     = "Whether to enable S3 specific headers",
129                 .off1     = offsetof(struct http_options, s3),
130                 .def      = "0",
131                 .category = FIO_OPT_C_ENGINE,
132                 .group    = FIO_OPT_G_HTTP,
133         },
134         {
135                 .name     = "http_verbose",
136                 .lname    = "CURL verbosity",
137                 .type     = FIO_OPT_INT,
138                 .help     = "increase http engine verbosity",
139                 .off1     = offsetof(struct http_options, verbose),
140                 .def      = "0",
141                 .category = FIO_OPT_C_ENGINE,
142                 .group    = FIO_OPT_G_HTTP,
143         },
144         {
145                 .name     = NULL,
146         },
147 };
148
149 static char *_aws_uriencode(const char *uri)
150 {
151         size_t bufsize = 1024;
152         char *r = malloc(bufsize);
153         char c;
154         int i, n;
155         const char *hex = "0123456789ABCDEF";
156
157         if (!r) {
158                 log_err("malloc failed\n");
159                 return NULL;
160         }
161
162         n = 0;
163         for (i = 0; (c = uri[i]); i++) {
164                 if (n > bufsize-5) {
165                         log_err("encoding the URL failed\n");
166                         return NULL;
167                 }
168
169                 if ( (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
170                 || (c >= '0' && c <= '9') || c == '_' || c == '-'
171                 || c == '~' || c == '.' || c == '/')
172                         r[n++] = c;
173                 else {
174                         r[n++] = '%';
175                         r[n++] = hex[(c >> 4 ) & 0xF];
176                         r[n++] = hex[c & 0xF];
177                 }
178         }
179         r[n++] = 0;
180         return r;
181 }
182
183 static char *_conv_hex(const unsigned char *p, size_t len)
184 {
185         char *r;
186         int i,n;
187         const char *hex = "0123456789abcdef";
188         r = malloc(len * 2 + 1);
189         n = 0;
190         for (i = 0; i < len; i++) {
191                 r[n++] = hex[(p[i] >> 4 ) & 0xF];
192                 r[n++] = hex[p[i] & 0xF];
193         }
194         r[n] = 0;
195
196         return r;
197 }
198
199 static char *_gen_hex_sha256(const char *p, size_t len)
200 {
201         unsigned char hash[SHA256_DIGEST_LENGTH];
202
203         SHA256((unsigned char*)p, len, hash);
204         return _conv_hex(hash, SHA256_DIGEST_LENGTH);
205 }
206
207 static void _hmac(unsigned char *md, void *key, int key_len, char *data) {
208 #ifndef CONFIG_HAVE_OPAQUE_HMAC_CTX
209         HMAC_CTX _ctx;
210 #endif
211         HMAC_CTX *ctx;
212         unsigned int hmac_len;
213
214 #ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX
215         ctx = HMAC_CTX_new();
216 #else
217         ctx = &_ctx;
218 #endif
219         HMAC_Init_ex(ctx, key, key_len, EVP_sha256(), NULL);
220         HMAC_Update(ctx, (unsigned char*)data, strlen(data));
221         HMAC_Final(ctx, md, &hmac_len);
222 #ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX
223         HMAC_CTX_free(ctx);
224 #else
225         HMAC_CTX_cleanup(ctx);
226 #endif
227 }
228
229 static int _curl_trace(CURL *handle, curl_infotype type,
230              char *data, size_t size,
231              void *userp)
232 {
233         const char *text;
234         (void)handle; /* prevent compiler warning */
235         (void)userp;
236
237         switch (type) {
238         case CURLINFO_TEXT:
239         fprintf(stderr, "== Info: %s", data);
240         default:
241         case CURLINFO_SSL_DATA_OUT:
242         case CURLINFO_SSL_DATA_IN:
243                 return 0;
244
245         case CURLINFO_HEADER_OUT:
246                 text = "=> Send header";
247                 break;
248         case CURLINFO_DATA_OUT:
249                 text = "=> Send data";
250                 break;
251         case CURLINFO_HEADER_IN:
252                 text = "<= Recv header";
253                 break;
254         case CURLINFO_DATA_IN:
255                 text = "<= Recv data";
256                 break;
257         }
258
259         log_info("%s: %s", text, data);
260         return 0;
261 }
262
263 /* https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
264  * https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
265  */
266 static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct http_options *o,
267                 int op, const char *uri, char *buf, size_t len)
268 {
269         char date_short[16];
270         char date_iso[32];
271         char method[8];
272         char dkey[128];
273         char creq[512];
274         char sts[256];
275         char s[512];
276         char *uri_encoded = NULL;
277         char *dsha = NULL;
278         char *csha = NULL;
279         char *signature = NULL;
280         const char *service = "s3";
281         const char *aws = "aws4_request";
282         unsigned char md[SHA256_DIGEST_LENGTH];
283
284         time_t t = time(NULL);
285         struct tm *gtm = gmtime(&t);
286
287         strftime (date_short, sizeof(date_short), "%Y%m%d", gtm);
288         strftime (date_iso, sizeof(date_iso), "%Y%m%dT%H%M%SZ", gtm);
289         uri_encoded = _aws_uriencode(uri);
290
291         if (op == DDIR_WRITE) {
292                 dsha = _gen_hex_sha256(buf, len);
293                 sprintf(method, "PUT");
294         } else {
295                 /* DDIR_READ && DDIR_TRIM supply an empty body */
296                 if (op == DDIR_READ)
297                         sprintf(method, "GET");
298                 else
299                         sprintf(method, "DELETE");
300                 dsha = _gen_hex_sha256("", 0);
301         }
302
303         /* Create the canonical request first */
304         snprintf(creq, sizeof(creq),
305         "%s\n"
306         "%s\n"
307         "\n"
308         "host:%s\n"
309         "x-amz-content-sha256:%s\n"
310         "x-amz-date:%s\n"
311         "\n"
312         "host;x-amz-content-sha256;x-amz-date\n"
313         "%s"
314         , method
315         , uri_encoded, o->host, dsha, date_iso, dsha);
316
317         csha = _gen_hex_sha256(creq, strlen(creq));
318         snprintf(sts, sizeof(sts), "AWS4-HMAC-SHA256\n%s\n%s/%s/%s/%s\n%s",
319                 date_iso, date_short, o->s3_region, service, aws, csha);
320
321         snprintf((char *)dkey, sizeof(dkey), "AWS4%s", o->s3_key);
322         _hmac(md, dkey, strlen(dkey), date_short);
323         _hmac(md, md, SHA256_DIGEST_LENGTH, o->s3_region);
324         _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) service);
325         _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) aws);
326         _hmac(md, md, SHA256_DIGEST_LENGTH, sts);
327
328         signature = _conv_hex(md, SHA256_DIGEST_LENGTH);
329
330         /* Surpress automatic Accept: header */
331         slist = curl_slist_append(slist, "Accept:");
332
333         snprintf(s, sizeof(s), "x-amz-content-sha256: %s", dsha);
334         slist = curl_slist_append(slist, s);
335
336         snprintf(s, sizeof(s), "x-amz-date: %s", date_iso);
337         slist = curl_slist_append(slist, s);
338
339         snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
340         "SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=%s",
341         o->s3_keyid, date_short, o->s3_region, signature);
342         slist = curl_slist_append(slist, s);
343
344         curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
345
346         free(uri_encoded);
347         free(csha);
348         free(dsha);
349         free(signature);
350 }
351
352 static void fio_http_cleanup(struct thread_data *td)
353 {
354         struct http_data *http = td->io_ops_data;
355
356         if (http) {
357                 curl_easy_cleanup(http->curl);
358                 free(http);
359         }
360 }
361
362 static size_t _http_read(void *ptr, size_t size, size_t nmemb, void *stream)
363 {
364         struct http_curl_stream *state = stream;
365         size_t len = size * nmemb;
366         /* We're retrieving; nothing is supposed to be read locally */
367         if (!stream)
368                 return 0;
369         if (len+state->pos > state->max)
370                 len = state->max - state->pos;
371         memcpy(ptr, &state->buf[state->pos], len);
372         state->pos += len;
373         return len;
374 }
375
376 static size_t _http_write(void *ptr, size_t size, size_t nmemb, void *stream)
377 {
378         struct http_curl_stream *state = stream;
379         /* We're just discarding the returned body after a PUT */
380         if (!stream)
381                 return nmemb;
382         if (size != 1)
383                 return CURLE_WRITE_ERROR;
384         if (nmemb + state->pos > state->max)
385                 return CURLE_WRITE_ERROR;
386         memcpy(&state->buf[state->pos], ptr, nmemb);
387         state->pos += nmemb;
388         return nmemb;
389 }
390
391 static int _http_seek(void *stream, curl_off_t offset, int origin)
392 {
393         struct http_curl_stream *state = stream;
394         if (offset < state->max && origin == SEEK_SET) {
395                 state->pos = offset;
396                 return CURL_SEEKFUNC_OK;
397         } else
398                 return CURL_SEEKFUNC_FAIL;
399 }
400
401 static enum fio_q_status fio_http_queue(struct thread_data *td,
402                                          struct io_u *io_u)
403 {
404         struct http_data *http = td->io_ops_data;
405         struct http_options *o = td->eo;
406         struct http_curl_stream _curl_stream;
407         struct curl_slist *slist = NULL;
408         char object[512];
409         char url[1024];
410         long status;
411         CURLcode res;
412         int r = -1;
413
414         fio_ro_check(td, io_u);
415         memset(&_curl_stream, 0, sizeof(_curl_stream));
416         snprintf(object, sizeof(object), "%s_%llu_%llu", td->files[0]->file_name, io_u->offset, io_u->xfer_buflen);
417         snprintf(url, sizeof(url), "%s://%s%s", o->https ? "https" : "http", o->host, object);
418         curl_easy_setopt(http->curl, CURLOPT_URL, url);
419         _curl_stream.buf = io_u->xfer_buf;
420         _curl_stream.max = io_u->xfer_buflen;
421         curl_easy_setopt(http->curl, CURLOPT_SEEKDATA, &_curl_stream);
422         curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)io_u->xfer_buflen);
423
424         if (o->s3)
425                 _add_aws_auth_header(http->curl, slist, o, io_u->ddir, object,
426                         io_u->xfer_buf, io_u->xfer_buflen);
427
428         if (io_u->ddir == DDIR_WRITE) {
429                 curl_easy_setopt(http->curl, CURLOPT_READDATA, &_curl_stream);
430                 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
431                 curl_easy_setopt(http->curl, CURLOPT_UPLOAD, 1L);
432                 res = curl_easy_perform(http->curl);
433                 if (res == CURLE_OK) {
434                         curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
435                         if (status == 100 || (status >= 200 && status <= 204))
436                                 goto out;
437                         log_err("DDIR_WRITE failed with HTTP status code %ld\n", status);
438                         goto err;
439                 }
440         } else if (io_u->ddir == DDIR_READ) {
441                 curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
442                 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, &_curl_stream);
443                 curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
444                 res = curl_easy_perform(http->curl);
445                 if (res == CURLE_OK) {
446                         curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
447                         if (status == 200)
448                                 goto out;
449                         else if (status == 404) {
450                                 /* Object doesn't exist. Pretend we read
451                                  * zeroes */
452                                 memset(io_u->xfer_buf, 0, io_u->xfer_buflen);
453                                 goto out;
454                         }
455                         log_err("DDIR_READ failed with HTTP status code %ld\n", status);
456                 }
457                 goto err;
458         } else if (io_u->ddir == DDIR_TRIM) {
459                 curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
460                 curl_easy_setopt(http->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
461                 curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, 0);
462                 curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
463                 curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
464                 res = curl_easy_perform(http->curl);
465                 if (res == CURLE_OK) {
466                         curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
467                         if (status == 200 || status == 202 || status == 204 || status == 404)
468                                 goto out;
469                         log_err("DDIR_TRIM failed with HTTP status code %ld\n", status);
470                 }
471                 goto err;
472         }
473
474         log_err("WARNING: Only DDIR_READ/DDIR_WRITE/DDIR_TRIM are supported!\n");
475
476 err:
477         io_u->error = r;
478         td_verror(td, io_u->error, "transfer");
479 out:
480         curl_slist_free_all(slist);
481         return FIO_Q_COMPLETED;
482 }
483
484 static struct io_u *fio_http_event(struct thread_data *td, int event)
485 {
486         /* sync IO engine - never any outstanding events */
487         return NULL;
488 }
489
490 int fio_http_getevents(struct thread_data *td, unsigned int min,
491         unsigned int max, const struct timespec *t)
492 {
493         /* sync IO engine - never any outstanding events */
494         return 0;
495 }
496
497 static int fio_http_setup(struct thread_data *td)
498 {
499         struct http_data *http = NULL;
500         struct http_options *o = td->eo;
501         int r;
502         /* allocate engine specific structure to deal with libhttp. */
503         http = calloc(1, sizeof(*http));
504         if (!http) {
505                 log_err("calloc failed.\n");
506                 goto cleanup;
507         }
508
509         http->curl = curl_easy_init();
510         if (o->verbose)
511                 curl_easy_setopt(http->curl, CURLOPT_VERBOSE, 1L);
512         if (o->verbose > 1)
513                 curl_easy_setopt(http->curl, CURLOPT_DEBUGFUNCTION, &_curl_trace);
514         curl_easy_setopt(http->curl, CURLOPT_NOPROGRESS, 1L);
515         curl_easy_setopt(http->curl, CURLOPT_FOLLOWLOCATION, 1L);
516         curl_easy_setopt(http->curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP|CURLPROTO_HTTPS);
517         curl_easy_setopt(http->curl, CURLOPT_READFUNCTION, _http_read);
518         curl_easy_setopt(http->curl, CURLOPT_WRITEFUNCTION, _http_write);
519         curl_easy_setopt(http->curl, CURLOPT_SEEKFUNCTION, _http_seek);
520         if (o->user && o->pass) {
521                 curl_easy_setopt(http->curl, CURLOPT_USERNAME, o->user);
522                 curl_easy_setopt(http->curl, CURLOPT_PASSWORD, o->pass);
523                 curl_easy_setopt(http->curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
524         }
525
526         td->io_ops_data = http;
527
528         /* Force single process mode. */
529         td->o.use_thread = 1;
530
531         return 0;
532 cleanup:
533         fio_http_cleanup(td);
534         return r;
535 }
536
537 static int fio_http_open(struct thread_data *td, struct fio_file *f)
538 {
539         return 0;
540 }
541 static int fio_http_invalidate(struct thread_data *td, struct fio_file *f)
542 {
543         return 0;
544 }
545
546 static struct ioengine_ops ioengine = {
547         .name = "http",
548         .version                = FIO_IOOPS_VERSION,
549         .flags                  = FIO_DISKLESSIO,
550         .setup                  = fio_http_setup,
551         .queue                  = fio_http_queue,
552         .getevents              = fio_http_getevents,
553         .event                  = fio_http_event,
554         .cleanup                = fio_http_cleanup,
555         .open_file              = fio_http_open,
556         .invalidate             = fio_http_invalidate,
557         .options                = options,
558         .option_struct_size     = sizeof(struct http_options),
559 };
560
561 static void fio_init fio_http_register(void)
562 {
563         register_ioengine(&ioengine);
564 }
565
566 static void fio_exit fio_http_unregister(void)
567 {
568         unregister_ioengine(&ioengine);
569 }