Commit | Line | Data |
---|---|---|
c2f6a13d LMB |
1 | /* |
2 | * HTTP GET/PUT IO engine | |
3 | * | |
4 | * IO engine to perform HTTP(S) GET/PUT requests via libcurl-easy. | |
5 | * | |
6 | * Copyright (C) 2018 SUSE LLC | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU General Public License, | |
10 | * version 2 as published by the Free Software Foundation.. | |
11 | * | |
12 | * This program is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | * GNU General Public License for more details. | |
16 | * | |
17 | * You should have received a copy of the GNU General Public | |
18 | * License along with this program; if not, write to the Free | |
19 | * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, | |
20 | * Boston, MA 02110-1301, USA. | |
21 | */ | |
22 | ||
23 | #include <pthread.h> | |
24 | #include <time.h> | |
25 | #include <curl/curl.h> | |
26 | #include <openssl/hmac.h> | |
27 | #include <openssl/sha.h> | |
09fd2966 | 28 | #include <openssl/md5.h> |
c2f6a13d LMB |
29 | #include "fio.h" |
30 | #include "../optgroup.h" | |
31 | ||
9c1c1a8d JA |
32 | /* |
33 | * Silence OpenSSL 3.0 deprecated function warnings | |
34 | */ | |
35 | #pragma GCC diagnostic ignored "-Wdeprecated-declarations" | |
c2f6a13d | 36 | |
09fd2966 LMB |
37 | enum { |
38 | FIO_HTTP_WEBDAV = 0, | |
39 | FIO_HTTP_S3 = 1, | |
40 | FIO_HTTP_SWIFT = 2, | |
41 | ||
42 | FIO_HTTPS_OFF = 0, | |
43 | FIO_HTTPS_ON = 1, | |
44 | FIO_HTTPS_INSECURE = 2, | |
45 | }; | |
46 | ||
c2f6a13d LMB |
47 | struct http_data { |
48 | CURL *curl; | |
49 | }; | |
50 | ||
51 | struct http_options { | |
52 | void *pad; | |
09fd2966 | 53 | unsigned int https; |
c2f6a13d LMB |
54 | char *host; |
55 | char *user; | |
56 | char *pass; | |
57 | char *s3_key; | |
58 | char *s3_keyid; | |
59 | char *s3_region; | |
91af79c4 FH |
60 | char *s3_sse_customer_key; |
61 | char *s3_sse_customer_algorithm; | |
f2d79184 | 62 | char *s3_storage_class; |
09fd2966 | 63 | char *swift_auth_token; |
c2f6a13d | 64 | int verbose; |
09fd2966 | 65 | unsigned int mode; |
c2f6a13d LMB |
66 | }; |
67 | ||
68 | struct http_curl_stream { | |
69 | char *buf; | |
70 | size_t pos; | |
71 | size_t max; | |
72 | }; | |
73 | ||
74 | static struct fio_option options[] = { | |
75 | { | |
76 | .name = "https", | |
77 | .lname = "https", | |
09fd2966 | 78 | .type = FIO_OPT_STR, |
c2f6a13d LMB |
79 | .help = "Enable https", |
80 | .off1 = offsetof(struct http_options, https), | |
09fd2966 LMB |
81 | .def = "off", |
82 | .posval = { | |
83 | { .ival = "off", | |
84 | .oval = FIO_HTTPS_OFF, | |
85 | .help = "No HTTPS", | |
86 | }, | |
87 | { .ival = "on", | |
88 | .oval = FIO_HTTPS_ON, | |
89 | .help = "Enable HTTPS", | |
90 | }, | |
91 | { .ival = "insecure", | |
92 | .oval = FIO_HTTPS_INSECURE, | |
93 | .help = "Enable HTTPS, disable peer verification", | |
94 | }, | |
95 | }, | |
c2f6a13d LMB |
96 | .category = FIO_OPT_C_ENGINE, |
97 | .group = FIO_OPT_G_HTTP, | |
98 | }, | |
99 | { | |
100 | .name = "http_host", | |
101 | .lname = "http_host", | |
102 | .type = FIO_OPT_STR_STORE, | |
103 | .help = "Hostname (S3 bucket)", | |
104 | .off1 = offsetof(struct http_options, host), | |
105 | .def = "localhost", | |
106 | .category = FIO_OPT_C_ENGINE, | |
107 | .group = FIO_OPT_G_HTTP, | |
108 | }, | |
109 | { | |
110 | .name = "http_user", | |
111 | .lname = "http_user", | |
112 | .type = FIO_OPT_STR_STORE, | |
113 | .help = "HTTP user name", | |
114 | .off1 = offsetof(struct http_options, user), | |
115 | .category = FIO_OPT_C_ENGINE, | |
116 | .group = FIO_OPT_G_HTTP, | |
117 | }, | |
118 | { | |
119 | .name = "http_pass", | |
120 | .lname = "http_pass", | |
121 | .type = FIO_OPT_STR_STORE, | |
122 | .help = "HTTP password", | |
123 | .off1 = offsetof(struct http_options, pass), | |
124 | .category = FIO_OPT_C_ENGINE, | |
125 | .group = FIO_OPT_G_HTTP, | |
126 | }, | |
127 | { | |
128 | .name = "http_s3_key", | |
129 | .lname = "S3 secret key", | |
130 | .type = FIO_OPT_STR_STORE, | |
131 | .help = "S3 secret key", | |
132 | .off1 = offsetof(struct http_options, s3_key), | |
133 | .def = "", | |
134 | .category = FIO_OPT_C_ENGINE, | |
135 | .group = FIO_OPT_G_HTTP, | |
136 | }, | |
137 | { | |
138 | .name = "http_s3_keyid", | |
139 | .lname = "S3 key id", | |
140 | .type = FIO_OPT_STR_STORE, | |
141 | .help = "S3 key id", | |
142 | .off1 = offsetof(struct http_options, s3_keyid), | |
143 | .def = "", | |
144 | .category = FIO_OPT_C_ENGINE, | |
145 | .group = FIO_OPT_G_HTTP, | |
146 | }, | |
09fd2966 LMB |
147 | { |
148 | .name = "http_swift_auth_token", | |
149 | .lname = "Swift auth token", | |
150 | .type = FIO_OPT_STR_STORE, | |
151 | .help = "OpenStack Swift auth token", | |
152 | .off1 = offsetof(struct http_options, swift_auth_token), | |
153 | .def = "", | |
154 | .category = FIO_OPT_C_ENGINE, | |
155 | .group = FIO_OPT_G_HTTP, | |
156 | }, | |
c2f6a13d LMB |
157 | { |
158 | .name = "http_s3_region", | |
159 | .lname = "S3 region", | |
160 | .type = FIO_OPT_STR_STORE, | |
161 | .help = "S3 region", | |
162 | .off1 = offsetof(struct http_options, s3_region), | |
163 | .def = "us-east-1", | |
164 | .category = FIO_OPT_C_ENGINE, | |
165 | .group = FIO_OPT_G_HTTP, | |
166 | }, | |
91af79c4 FH |
167 | { |
168 | .name = "http_s3_sse_customer_key", | |
169 | .lname = "SSE Customer Key", | |
170 | .type = FIO_OPT_STR_STORE, | |
171 | .help = "S3 SSE Customer Key", | |
172 | .off1 = offsetof(struct http_options, s3_sse_customer_key), | |
173 | .def = "", | |
174 | .category = FIO_OPT_C_ENGINE, | |
175 | .group = FIO_OPT_G_HTTP, | |
176 | }, | |
177 | { | |
178 | .name = "http_s3_sse_customer_algorithm", | |
179 | .lname = "SSE Customer Algorithm", | |
180 | .type = FIO_OPT_STR_STORE, | |
181 | .help = "S3 SSE Customer Algorithm", | |
182 | .off1 = offsetof(struct http_options, s3_sse_customer_algorithm), | |
183 | .def = "AES256", | |
184 | .category = FIO_OPT_C_ENGINE, | |
185 | .group = FIO_OPT_G_HTTP, | |
186 | }, | |
f2d79184 FH |
187 | { |
188 | .name = "http_s3_storage_class", | |
189 | .lname = "S3 Storage class", | |
190 | .type = FIO_OPT_STR_STORE, | |
191 | .help = "S3 Storage Class", | |
192 | .off1 = offsetof(struct http_options, s3_storage_class), | |
193 | .def = "STANDARD", | |
194 | .category = FIO_OPT_C_ENGINE, | |
195 | .group = FIO_OPT_G_HTTP, | |
196 | }, | |
c2f6a13d | 197 | { |
09fd2966 LMB |
198 | .name = "http_mode", |
199 | .lname = "Request mode to use", | |
200 | .type = FIO_OPT_STR, | |
201 | .help = "Whether to use WebDAV, Swift, or S3", | |
202 | .off1 = offsetof(struct http_options, mode), | |
203 | .def = "webdav", | |
204 | .posval = { | |
205 | { .ival = "webdav", | |
206 | .oval = FIO_HTTP_WEBDAV, | |
207 | .help = "WebDAV server", | |
208 | }, | |
209 | { .ival = "s3", | |
210 | .oval = FIO_HTTP_S3, | |
211 | .help = "S3 storage backend", | |
212 | }, | |
213 | { .ival = "swift", | |
214 | .oval = FIO_HTTP_SWIFT, | |
215 | .help = "OpenStack Swift storage", | |
216 | }, | |
217 | }, | |
c2f6a13d LMB |
218 | .category = FIO_OPT_C_ENGINE, |
219 | .group = FIO_OPT_G_HTTP, | |
220 | }, | |
221 | { | |
222 | .name = "http_verbose", | |
09fd2966 | 223 | .lname = "HTTP verbosity level", |
c2f6a13d LMB |
224 | .type = FIO_OPT_INT, |
225 | .help = "increase http engine verbosity", | |
226 | .off1 = offsetof(struct http_options, verbose), | |
227 | .def = "0", | |
228 | .category = FIO_OPT_C_ENGINE, | |
229 | .group = FIO_OPT_G_HTTP, | |
230 | }, | |
231 | { | |
232 | .name = NULL, | |
233 | }, | |
234 | }; | |
235 | ||
236 | static char *_aws_uriencode(const char *uri) | |
237 | { | |
238 | size_t bufsize = 1024; | |
239 | char *r = malloc(bufsize); | |
240 | char c; | |
241 | int i, n; | |
242 | const char *hex = "0123456789ABCDEF"; | |
243 | ||
244 | if (!r) { | |
245 | log_err("malloc failed\n"); | |
246 | return NULL; | |
247 | } | |
248 | ||
249 | n = 0; | |
250 | for (i = 0; (c = uri[i]); i++) { | |
251 | if (n > bufsize-5) { | |
252 | log_err("encoding the URL failed\n"); | |
253 | return NULL; | |
254 | } | |
255 | ||
256 | if ( (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') | |
257 | || (c >= '0' && c <= '9') || c == '_' || c == '-' | |
258 | || c == '~' || c == '.' || c == '/') | |
259 | r[n++] = c; | |
260 | else { | |
261 | r[n++] = '%'; | |
262 | r[n++] = hex[(c >> 4 ) & 0xF]; | |
263 | r[n++] = hex[c & 0xF]; | |
264 | } | |
265 | } | |
266 | r[n++] = 0; | |
267 | return r; | |
268 | } | |
269 | ||
270 | static char *_conv_hex(const unsigned char *p, size_t len) | |
271 | { | |
272 | char *r; | |
273 | int i,n; | |
274 | const char *hex = "0123456789abcdef"; | |
275 | r = malloc(len * 2 + 1); | |
276 | n = 0; | |
277 | for (i = 0; i < len; i++) { | |
278 | r[n++] = hex[(p[i] >> 4 ) & 0xF]; | |
279 | r[n++] = hex[p[i] & 0xF]; | |
280 | } | |
281 | r[n] = 0; | |
282 | ||
283 | return r; | |
284 | } | |
285 | ||
286 | static char *_gen_hex_sha256(const char *p, size_t len) | |
287 | { | |
288 | unsigned char hash[SHA256_DIGEST_LENGTH]; | |
289 | ||
290 | SHA256((unsigned char*)p, len, hash); | |
291 | return _conv_hex(hash, SHA256_DIGEST_LENGTH); | |
292 | } | |
293 | ||
09fd2966 LMB |
294 | static char *_gen_hex_md5(const char *p, size_t len) |
295 | { | |
296 | unsigned char hash[MD5_DIGEST_LENGTH]; | |
297 | ||
298 | MD5((unsigned char*)p, len, hash); | |
299 | return _conv_hex(hash, MD5_DIGEST_LENGTH); | |
300 | } | |
301 | ||
91af79c4 FH |
302 | static char *_conv_base64_encode(const unsigned char *p, size_t len) |
303 | { | |
304 | char *r, *ret; | |
305 | int i; | |
306 | static const char sEncodingTable[] = { | |
307 | 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', | |
308 | 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', | |
309 | 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', | |
310 | 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', | |
311 | 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', | |
312 | 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', | |
313 | 'w', 'x', 'y', 'z', '0', '1', '2', '3', | |
314 | '4', '5', '6', '7', '8', '9', '+', '/' | |
315 | }; | |
316 | ||
317 | size_t out_len = 4 * ((len + 2) / 3); | |
318 | ret = r = malloc(out_len + 1); | |
319 | ||
320 | for (i = 0; i < len - 2; i += 3) { | |
321 | *r++ = sEncodingTable[(p[i] >> 2) & 0x3F]; | |
322 | *r++ = sEncodingTable[((p[i] & 0x3) << 4) | ((int) (p[i + 1] & 0xF0) >> 4)]; | |
323 | *r++ = sEncodingTable[((p[i + 1] & 0xF) << 2) | ((int) (p[i + 2] & 0xC0) >> 6)]; | |
324 | *r++ = sEncodingTable[p[i + 2] & 0x3F]; | |
325 | } | |
326 | ||
327 | if (i < len) { | |
328 | *r++ = sEncodingTable[(p[i] >> 2) & 0x3F]; | |
329 | if (i == (len - 1)) { | |
330 | *r++ = sEncodingTable[((p[i] & 0x3) << 4)]; | |
331 | *r++ = '='; | |
332 | } else { | |
333 | *r++ = sEncodingTable[((p[i] & 0x3) << 4) | ((int) (p[i + 1] & 0xF0) >> 4)]; | |
334 | *r++ = sEncodingTable[((p[i + 1] & 0xF) << 2)]; | |
335 | } | |
336 | *r++ = '='; | |
337 | } | |
338 | ||
339 | ret[out_len]=0; | |
340 | return ret; | |
341 | } | |
342 | ||
343 | static char *_gen_base64_md5(const unsigned char *p, size_t len) | |
344 | { | |
345 | unsigned char hash[MD5_DIGEST_LENGTH]; | |
346 | MD5((unsigned char*)p, len, hash); | |
347 | return _conv_base64_encode(hash, MD5_DIGEST_LENGTH); | |
348 | } | |
349 | ||
c2f6a13d | 350 | static void _hmac(unsigned char *md, void *key, int key_len, char *data) { |
b61a5f46 DD |
351 | #ifndef CONFIG_HAVE_OPAQUE_HMAC_CTX |
352 | HMAC_CTX _ctx; | |
353 | #endif | |
c2f6a13d LMB |
354 | HMAC_CTX *ctx; |
355 | unsigned int hmac_len; | |
356 | ||
b61a5f46 | 357 | #ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX |
c2f6a13d | 358 | ctx = HMAC_CTX_new(); |
b61a5f46 DD |
359 | #else |
360 | ctx = &_ctx; | |
e8aaa776 JA |
361 | /* work-around crash in certain versions of libssl */ |
362 | HMAC_CTX_init(ctx); | |
b61a5f46 | 363 | #endif |
c2f6a13d LMB |
364 | HMAC_Init_ex(ctx, key, key_len, EVP_sha256(), NULL); |
365 | HMAC_Update(ctx, (unsigned char*)data, strlen(data)); | |
366 | HMAC_Final(ctx, md, &hmac_len); | |
b61a5f46 | 367 | #ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX |
c2f6a13d | 368 | HMAC_CTX_free(ctx); |
b61a5f46 DD |
369 | #else |
370 | HMAC_CTX_cleanup(ctx); | |
371 | #endif | |
c2f6a13d LMB |
372 | } |
373 | ||
374 | static int _curl_trace(CURL *handle, curl_infotype type, | |
375 | char *data, size_t size, | |
376 | void *userp) | |
377 | { | |
378 | const char *text; | |
379 | (void)handle; /* prevent compiler warning */ | |
380 | (void)userp; | |
381 | ||
382 | switch (type) { | |
383 | case CURLINFO_TEXT: | |
24010223 | 384 | fprintf(stderr, "== Info: %s", data); |
87933e32 | 385 | fio_fallthrough; |
c2f6a13d LMB |
386 | default: |
387 | case CURLINFO_SSL_DATA_OUT: | |
388 | case CURLINFO_SSL_DATA_IN: | |
389 | return 0; | |
390 | ||
391 | case CURLINFO_HEADER_OUT: | |
392 | text = "=> Send header"; | |
393 | break; | |
394 | case CURLINFO_DATA_OUT: | |
395 | text = "=> Send data"; | |
396 | break; | |
397 | case CURLINFO_HEADER_IN: | |
398 | text = "<= Recv header"; | |
399 | break; | |
400 | case CURLINFO_DATA_IN: | |
401 | text = "<= Recv data"; | |
402 | break; | |
403 | } | |
404 | ||
405 | log_info("%s: %s", text, data); | |
406 | return 0; | |
407 | } | |
408 | ||
409 | /* https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html | |
410 | * https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro | |
411 | */ | |
412 | static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct http_options *o, | |
413 | int op, const char *uri, char *buf, size_t len) | |
414 | { | |
415 | char date_short[16]; | |
416 | char date_iso[32]; | |
417 | char method[8]; | |
418 | char dkey[128]; | |
f2d79184 FH |
419 | char creq[4096]; |
420 | char sts[512]; | |
c2f6a13d LMB |
421 | char s[512]; |
422 | char *uri_encoded = NULL; | |
423 | char *dsha = NULL; | |
424 | char *csha = NULL; | |
425 | char *signature = NULL; | |
426 | const char *service = "s3"; | |
427 | const char *aws = "aws4_request"; | |
428 | unsigned char md[SHA256_DIGEST_LENGTH]; | |
91af79c4 FH |
429 | unsigned char sse_key[33] = {0}; |
430 | char *sse_key_base64 = NULL; | |
431 | char *sse_key_md5_base64 = NULL; | |
c2f6a13d LMB |
432 | |
433 | time_t t = time(NULL); | |
434 | struct tm *gtm = gmtime(&t); | |
435 | ||
436 | strftime (date_short, sizeof(date_short), "%Y%m%d", gtm); | |
437 | strftime (date_iso, sizeof(date_iso), "%Y%m%dT%H%M%SZ", gtm); | |
438 | uri_encoded = _aws_uriencode(uri); | |
439 | ||
91af79c4 FH |
440 | if (o->s3_sse_customer_key != NULL) |
441 | strncpy((char*)sse_key, o->s3_sse_customer_key, sizeof(sse_key) - 1); | |
442 | ||
c2f6a13d LMB |
443 | if (op == DDIR_WRITE) { |
444 | dsha = _gen_hex_sha256(buf, len); | |
445 | sprintf(method, "PUT"); | |
446 | } else { | |
447 | /* DDIR_READ && DDIR_TRIM supply an empty body */ | |
448 | if (op == DDIR_READ) | |
449 | sprintf(method, "GET"); | |
450 | else | |
451 | sprintf(method, "DELETE"); | |
452 | dsha = _gen_hex_sha256("", 0); | |
453 | } | |
454 | ||
455 | /* Create the canonical request first */ | |
91af79c4 FH |
456 | if (sse_key[0] != '\0') { |
457 | sse_key_base64 = _conv_base64_encode(sse_key, sizeof(sse_key) - 1); | |
458 | sse_key_md5_base64 = _gen_base64_md5(sse_key, sizeof(sse_key) - 1); | |
459 | snprintf(creq, sizeof(creq), | |
460 | "%s\n" | |
461 | "%s\n" | |
462 | "\n" | |
463 | "host:%s\n" | |
464 | "x-amz-content-sha256:%s\n" | |
465 | "x-amz-date:%s\n" | |
466 | "x-amz-server-side-encryption-customer-algorithm:%s\n" | |
467 | "x-amz-server-side-encryption-customer-key:%s\n" | |
468 | "x-amz-server-side-encryption-customer-key-md5:%s\n" | |
469 | "x-amz-storage-class:%s\n" | |
470 | "\n" | |
471 | "host;x-amz-content-sha256;x-amz-date;" | |
472 | "x-amz-server-side-encryption-customer-algorithm;" | |
473 | "x-amz-server-side-encryption-customer-key;" | |
474 | "x-amz-server-side-encryption-customer-key-md5;" | |
475 | "x-amz-storage-class\n" | |
476 | "%s" | |
477 | , method | |
478 | , uri_encoded, o->host, dsha, date_iso | |
479 | , o->s3_sse_customer_algorithm, sse_key_base64 | |
480 | , sse_key_md5_base64, o->s3_storage_class, dsha); | |
481 | } else { | |
482 | snprintf(creq, sizeof(creq), | |
483 | "%s\n" | |
484 | "%s\n" | |
485 | "\n" | |
486 | "host:%s\n" | |
487 | "x-amz-content-sha256:%s\n" | |
488 | "x-amz-date:%s\n" | |
489 | "x-amz-storage-class:%s\n" | |
490 | "\n" | |
491 | "host;x-amz-content-sha256;x-amz-date;x-amz-storage-class\n" | |
492 | "%s" | |
493 | , method | |
494 | , uri_encoded, o->host, dsha, date_iso, o->s3_storage_class, dsha); | |
495 | } | |
c2f6a13d LMB |
496 | |
497 | csha = _gen_hex_sha256(creq, strlen(creq)); | |
498 | snprintf(sts, sizeof(sts), "AWS4-HMAC-SHA256\n%s\n%s/%s/%s/%s\n%s", | |
91af79c4 | 499 | date_iso, date_short, o->s3_region, service, aws, csha); |
c2f6a13d LMB |
500 | |
501 | snprintf((char *)dkey, sizeof(dkey), "AWS4%s", o->s3_key); | |
502 | _hmac(md, dkey, strlen(dkey), date_short); | |
503 | _hmac(md, md, SHA256_DIGEST_LENGTH, o->s3_region); | |
504 | _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) service); | |
505 | _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) aws); | |
506 | _hmac(md, md, SHA256_DIGEST_LENGTH, sts); | |
507 | ||
508 | signature = _conv_hex(md, SHA256_DIGEST_LENGTH); | |
509 | ||
fc002f14 | 510 | /* Suppress automatic Accept: header */ |
c2f6a13d LMB |
511 | slist = curl_slist_append(slist, "Accept:"); |
512 | ||
513 | snprintf(s, sizeof(s), "x-amz-content-sha256: %s", dsha); | |
514 | slist = curl_slist_append(slist, s); | |
515 | ||
516 | snprintf(s, sizeof(s), "x-amz-date: %s", date_iso); | |
517 | slist = curl_slist_append(slist, s); | |
91af79c4 FH |
518 | |
519 | if (sse_key[0] != '\0') { | |
520 | snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-algorithm: %s", o->s3_sse_customer_algorithm); | |
521 | slist = curl_slist_append(slist, s); | |
522 | snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-key: %s", sse_key_base64); | |
523 | slist = curl_slist_append(slist, s); | |
524 | snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-key-md5: %s", sse_key_md5_base64); | |
525 | slist = curl_slist_append(slist, s); | |
526 | } | |
527 | ||
f2d79184 FH |
528 | snprintf(s, sizeof(s), "x-amz-storage-class: %s", o->s3_storage_class); |
529 | slist = curl_slist_append(slist, s); | |
91af79c4 FH |
530 | |
531 | if (sse_key[0] != '\0') { | |
532 | snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request," | |
533 | "SignedHeaders=host;x-amz-content-sha256;" | |
534 | "x-amz-date;x-amz-server-side-encryption-customer-algorithm;" | |
535 | "x-amz-server-side-encryption-customer-key;" | |
536 | "x-amz-server-side-encryption-customer-key-md5;" | |
537 | "x-amz-storage-class," | |
538 | "Signature=%s", | |
539 | o->s3_keyid, date_short, o->s3_region, signature); | |
540 | } else { | |
541 | snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request," | |
542 | "SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=%s", | |
543 | o->s3_keyid, date_short, o->s3_region, signature); | |
544 | } | |
c2f6a13d LMB |
545 | slist = curl_slist_append(slist, s); |
546 | ||
547 | curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); | |
548 | ||
549 | free(uri_encoded); | |
550 | free(csha); | |
551 | free(dsha); | |
552 | free(signature); | |
91af79c4 FH |
553 | if (sse_key_base64 != NULL) { |
554 | free(sse_key_base64); | |
555 | free(sse_key_md5_base64); | |
556 | } | |
c2f6a13d LMB |
557 | } |
558 | ||
09fd2966 LMB |
559 | static void _add_swift_header(CURL *curl, struct curl_slist *slist, struct http_options *o, |
560 | int op, const char *uri, char *buf, size_t len) | |
561 | { | |
562 | char *dsha = NULL; | |
563 | char s[512]; | |
564 | ||
565 | if (op == DDIR_WRITE) { | |
566 | dsha = _gen_hex_md5(buf, len); | |
567 | } | |
fc002f14 | 568 | /* Suppress automatic Accept: header */ |
09fd2966 LMB |
569 | slist = curl_slist_append(slist, "Accept:"); |
570 | ||
571 | snprintf(s, sizeof(s), "etag: %s", dsha); | |
572 | slist = curl_slist_append(slist, s); | |
573 | ||
574 | snprintf(s, sizeof(s), "x-auth-token: %s", o->swift_auth_token); | |
575 | slist = curl_slist_append(slist, s); | |
576 | ||
577 | curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); | |
578 | ||
579 | free(dsha); | |
580 | } | |
581 | ||
c2f6a13d LMB |
582 | static void fio_http_cleanup(struct thread_data *td) |
583 | { | |
584 | struct http_data *http = td->io_ops_data; | |
585 | ||
586 | if (http) { | |
587 | curl_easy_cleanup(http->curl); | |
588 | free(http); | |
589 | } | |
590 | } | |
591 | ||
592 | static size_t _http_read(void *ptr, size_t size, size_t nmemb, void *stream) | |
593 | { | |
594 | struct http_curl_stream *state = stream; | |
595 | size_t len = size * nmemb; | |
596 | /* We're retrieving; nothing is supposed to be read locally */ | |
597 | if (!stream) | |
598 | return 0; | |
599 | if (len+state->pos > state->max) | |
600 | len = state->max - state->pos; | |
601 | memcpy(ptr, &state->buf[state->pos], len); | |
602 | state->pos += len; | |
603 | return len; | |
604 | } | |
605 | ||
606 | static size_t _http_write(void *ptr, size_t size, size_t nmemb, void *stream) | |
607 | { | |
608 | struct http_curl_stream *state = stream; | |
609 | /* We're just discarding the returned body after a PUT */ | |
610 | if (!stream) | |
611 | return nmemb; | |
612 | if (size != 1) | |
613 | return CURLE_WRITE_ERROR; | |
614 | if (nmemb + state->pos > state->max) | |
615 | return CURLE_WRITE_ERROR; | |
616 | memcpy(&state->buf[state->pos], ptr, nmemb); | |
617 | state->pos += nmemb; | |
618 | return nmemb; | |
619 | } | |
620 | ||
621 | static int _http_seek(void *stream, curl_off_t offset, int origin) | |
622 | { | |
623 | struct http_curl_stream *state = stream; | |
624 | if (offset < state->max && origin == SEEK_SET) { | |
625 | state->pos = offset; | |
626 | return CURL_SEEKFUNC_OK; | |
627 | } else | |
628 | return CURL_SEEKFUNC_FAIL; | |
629 | } | |
630 | ||
631 | static enum fio_q_status fio_http_queue(struct thread_data *td, | |
632 | struct io_u *io_u) | |
633 | { | |
634 | struct http_data *http = td->io_ops_data; | |
635 | struct http_options *o = td->eo; | |
636 | struct http_curl_stream _curl_stream; | |
637 | struct curl_slist *slist = NULL; | |
638 | char object[512]; | |
639 | char url[1024]; | |
640 | long status; | |
641 | CURLcode res; | |
642 | int r = -1; | |
643 | ||
644 | fio_ro_check(td, io_u); | |
645 | memset(&_curl_stream, 0, sizeof(_curl_stream)); | |
09fd2966 LMB |
646 | snprintf(object, sizeof(object), "%s_%llu_%llu", td->files[0]->file_name, |
647 | io_u->offset, io_u->xfer_buflen); | |
648 | if (o->https == FIO_HTTPS_OFF) | |
649 | snprintf(url, sizeof(url), "http://%s%s", o->host, object); | |
650 | else | |
651 | snprintf(url, sizeof(url), "https://%s%s", o->host, object); | |
c2f6a13d LMB |
652 | curl_easy_setopt(http->curl, CURLOPT_URL, url); |
653 | _curl_stream.buf = io_u->xfer_buf; | |
654 | _curl_stream.max = io_u->xfer_buflen; | |
655 | curl_easy_setopt(http->curl, CURLOPT_SEEKDATA, &_curl_stream); | |
656 | curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)io_u->xfer_buflen); | |
657 | ||
09fd2966 | 658 | if (o->mode == FIO_HTTP_S3) |
c2f6a13d LMB |
659 | _add_aws_auth_header(http->curl, slist, o, io_u->ddir, object, |
660 | io_u->xfer_buf, io_u->xfer_buflen); | |
09fd2966 LMB |
661 | else if (o->mode == FIO_HTTP_SWIFT) |
662 | _add_swift_header(http->curl, slist, o, io_u->ddir, object, | |
663 | io_u->xfer_buf, io_u->xfer_buflen); | |
c2f6a13d LMB |
664 | |
665 | if (io_u->ddir == DDIR_WRITE) { | |
666 | curl_easy_setopt(http->curl, CURLOPT_READDATA, &_curl_stream); | |
667 | curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL); | |
668 | curl_easy_setopt(http->curl, CURLOPT_UPLOAD, 1L); | |
669 | res = curl_easy_perform(http->curl); | |
670 | if (res == CURLE_OK) { | |
671 | curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status); | |
672 | if (status == 100 || (status >= 200 && status <= 204)) | |
673 | goto out; | |
674 | log_err("DDIR_WRITE failed with HTTP status code %ld\n", status); | |
c2f6a13d | 675 | } |
4a4c21bf | 676 | goto err; |
c2f6a13d LMB |
677 | } else if (io_u->ddir == DDIR_READ) { |
678 | curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL); | |
679 | curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, &_curl_stream); | |
680 | curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L); | |
681 | res = curl_easy_perform(http->curl); | |
682 | if (res == CURLE_OK) { | |
683 | curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status); | |
684 | if (status == 200) | |
685 | goto out; | |
686 | else if (status == 404) { | |
687 | /* Object doesn't exist. Pretend we read | |
688 | * zeroes */ | |
689 | memset(io_u->xfer_buf, 0, io_u->xfer_buflen); | |
690 | goto out; | |
691 | } | |
692 | log_err("DDIR_READ failed with HTTP status code %ld\n", status); | |
693 | } | |
694 | goto err; | |
695 | } else if (io_u->ddir == DDIR_TRIM) { | |
696 | curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L); | |
697 | curl_easy_setopt(http->curl, CURLOPT_CUSTOMREQUEST, "DELETE"); | |
bd915a6b | 698 | curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)0); |
c2f6a13d LMB |
699 | curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL); |
700 | curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL); | |
701 | res = curl_easy_perform(http->curl); | |
702 | if (res == CURLE_OK) { | |
703 | curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status); | |
704 | if (status == 200 || status == 202 || status == 204 || status == 404) | |
705 | goto out; | |
706 | log_err("DDIR_TRIM failed with HTTP status code %ld\n", status); | |
707 | } | |
708 | goto err; | |
709 | } | |
710 | ||
711 | log_err("WARNING: Only DDIR_READ/DDIR_WRITE/DDIR_TRIM are supported!\n"); | |
712 | ||
713 | err: | |
714 | io_u->error = r; | |
715 | td_verror(td, io_u->error, "transfer"); | |
716 | out: | |
717 | curl_slist_free_all(slist); | |
718 | return FIO_Q_COMPLETED; | |
719 | } | |
720 | ||
721 | static struct io_u *fio_http_event(struct thread_data *td, int event) | |
722 | { | |
723 | /* sync IO engine - never any outstanding events */ | |
724 | return NULL; | |
725 | } | |
726 | ||
727 | int fio_http_getevents(struct thread_data *td, unsigned int min, | |
728 | unsigned int max, const struct timespec *t) | |
729 | { | |
730 | /* sync IO engine - never any outstanding events */ | |
731 | return 0; | |
732 | } | |
733 | ||
734 | static int fio_http_setup(struct thread_data *td) | |
735 | { | |
736 | struct http_data *http = NULL; | |
737 | struct http_options *o = td->eo; | |
c65f3435 | 738 | |
c2f6a13d LMB |
739 | /* allocate engine specific structure to deal with libhttp. */ |
740 | http = calloc(1, sizeof(*http)); | |
741 | if (!http) { | |
742 | log_err("calloc failed.\n"); | |
743 | goto cleanup; | |
744 | } | |
745 | ||
746 | http->curl = curl_easy_init(); | |
747 | if (o->verbose) | |
748 | curl_easy_setopt(http->curl, CURLOPT_VERBOSE, 1L); | |
749 | if (o->verbose > 1) | |
750 | curl_easy_setopt(http->curl, CURLOPT_DEBUGFUNCTION, &_curl_trace); | |
751 | curl_easy_setopt(http->curl, CURLOPT_NOPROGRESS, 1L); | |
752 | curl_easy_setopt(http->curl, CURLOPT_FOLLOWLOCATION, 1L); | |
753 | curl_easy_setopt(http->curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP|CURLPROTO_HTTPS); | |
09fd2966 LMB |
754 | if (o->https == FIO_HTTPS_INSECURE) { |
755 | curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYPEER, 0L); | |
756 | curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYHOST, 0L); | |
757 | } | |
c2f6a13d LMB |
758 | curl_easy_setopt(http->curl, CURLOPT_READFUNCTION, _http_read); |
759 | curl_easy_setopt(http->curl, CURLOPT_WRITEFUNCTION, _http_write); | |
bd915a6b | 760 | curl_easy_setopt(http->curl, CURLOPT_SEEKFUNCTION, &_http_seek); |
c2f6a13d LMB |
761 | if (o->user && o->pass) { |
762 | curl_easy_setopt(http->curl, CURLOPT_USERNAME, o->user); | |
763 | curl_easy_setopt(http->curl, CURLOPT_PASSWORD, o->pass); | |
764 | curl_easy_setopt(http->curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY); | |
765 | } | |
766 | ||
767 | td->io_ops_data = http; | |
768 | ||
769 | /* Force single process mode. */ | |
770 | td->o.use_thread = 1; | |
771 | ||
772 | return 0; | |
773 | cleanup: | |
774 | fio_http_cleanup(td); | |
c65f3435 | 775 | return 1; |
c2f6a13d LMB |
776 | } |
777 | ||
778 | static int fio_http_open(struct thread_data *td, struct fio_file *f) | |
779 | { | |
780 | return 0; | |
781 | } | |
782 | static int fio_http_invalidate(struct thread_data *td, struct fio_file *f) | |
783 | { | |
784 | return 0; | |
785 | } | |
786 | ||
5a8a6a03 | 787 | FIO_STATIC struct ioengine_ops ioengine = { |
c2f6a13d LMB |
788 | .name = "http", |
789 | .version = FIO_IOOPS_VERSION, | |
f32a30d4 | 790 | .flags = FIO_DISKLESSIO | FIO_SYNCIO, |
c2f6a13d LMB |
791 | .setup = fio_http_setup, |
792 | .queue = fio_http_queue, | |
793 | .getevents = fio_http_getevents, | |
794 | .event = fio_http_event, | |
795 | .cleanup = fio_http_cleanup, | |
796 | .open_file = fio_http_open, | |
797 | .invalidate = fio_http_invalidate, | |
798 | .options = options, | |
799 | .option_struct_size = sizeof(struct http_options), | |
800 | }; | |
801 | ||
802 | static void fio_init fio_http_register(void) | |
803 | { | |
804 | register_ioengine(&ioengine); | |
805 | } | |
806 | ||
807 | static void fio_exit fio_http_unregister(void) | |
808 | { | |
809 | unregister_ioengine(&ioengine); | |
810 | } |