#include <curl/curl.h>
#include <openssl/hmac.h>
#include <openssl/sha.h>
+#include <openssl/md5.h>
#include "fio.h"
#include "../optgroup.h"
+/*
+ * Silence OpenSSL 3.0 deprecated function warnings
+ */
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+
+enum {
+ FIO_HTTP_WEBDAV = 0,
+ FIO_HTTP_S3 = 1,
+ FIO_HTTP_SWIFT = 2,
+
+ FIO_HTTPS_OFF = 0,
+ FIO_HTTPS_ON = 1,
+ FIO_HTTPS_INSECURE = 2,
+};
struct http_data {
CURL *curl;
struct http_options {
void *pad;
- int https;
+ unsigned int https;
char *host;
char *user;
char *pass;
char *s3_key;
char *s3_keyid;
char *s3_region;
+ char *s3_sse_customer_key;
+ char *s3_sse_customer_algorithm;
+ char *s3_storage_class;
+ char *swift_auth_token;
int verbose;
- int s3;
+ unsigned int mode;
};
struct http_curl_stream {
{
.name = "https",
.lname = "https",
- .type = FIO_OPT_BOOL,
+ .type = FIO_OPT_STR,
.help = "Enable https",
.off1 = offsetof(struct http_options, https),
- .def = "0",
+ .def = "off",
+ .posval = {
+ { .ival = "off",
+ .oval = FIO_HTTPS_OFF,
+ .help = "No HTTPS",
+ },
+ { .ival = "on",
+ .oval = FIO_HTTPS_ON,
+ .help = "Enable HTTPS",
+ },
+ { .ival = "insecure",
+ .oval = FIO_HTTPS_INSECURE,
+ .help = "Enable HTTPS, disable peer verification",
+ },
+ },
.category = FIO_OPT_C_ENGINE,
.group = FIO_OPT_G_HTTP,
},
.category = FIO_OPT_C_ENGINE,
.group = FIO_OPT_G_HTTP,
},
+ {
+ .name = "http_swift_auth_token",
+ .lname = "Swift auth token",
+ .type = FIO_OPT_STR_STORE,
+ .help = "OpenStack Swift auth token",
+ .off1 = offsetof(struct http_options, swift_auth_token),
+ .def = "",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_HTTP,
+ },
{
.name = "http_s3_region",
.lname = "S3 region",
.group = FIO_OPT_G_HTTP,
},
{
- .name = "http_s3",
- .lname = "S3 extensions",
- .type = FIO_OPT_BOOL,
- .help = "Whether to enable S3 specific headers",
- .off1 = offsetof(struct http_options, s3),
- .def = "0",
+ .name = "http_s3_sse_customer_key",
+ .lname = "SSE Customer Key",
+ .type = FIO_OPT_STR_STORE,
+ .help = "S3 SSE Customer Key",
+ .off1 = offsetof(struct http_options, s3_sse_customer_key),
+ .def = "",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_HTTP,
+ },
+ {
+ .name = "http_s3_sse_customer_algorithm",
+ .lname = "SSE Customer Algorithm",
+ .type = FIO_OPT_STR_STORE,
+ .help = "S3 SSE Customer Algorithm",
+ .off1 = offsetof(struct http_options, s3_sse_customer_algorithm),
+ .def = "AES256",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_HTTP,
+ },
+ {
+ .name = "http_s3_storage_class",
+ .lname = "S3 Storage class",
+ .type = FIO_OPT_STR_STORE,
+ .help = "S3 Storage Class",
+ .off1 = offsetof(struct http_options, s3_storage_class),
+ .def = "STANDARD",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_HTTP,
+ },
+ {
+ .name = "http_mode",
+ .lname = "Request mode to use",
+ .type = FIO_OPT_STR,
+ .help = "Whether to use WebDAV, Swift, or S3",
+ .off1 = offsetof(struct http_options, mode),
+ .def = "webdav",
+ .posval = {
+ { .ival = "webdav",
+ .oval = FIO_HTTP_WEBDAV,
+ .help = "WebDAV server",
+ },
+ { .ival = "s3",
+ .oval = FIO_HTTP_S3,
+ .help = "S3 storage backend",
+ },
+ { .ival = "swift",
+ .oval = FIO_HTTP_SWIFT,
+ .help = "OpenStack Swift storage",
+ },
+ },
.category = FIO_OPT_C_ENGINE,
.group = FIO_OPT_G_HTTP,
},
{
.name = "http_verbose",
- .lname = "CURL verbosity",
+ .lname = "HTTP verbosity level",
.type = FIO_OPT_INT,
.help = "increase http engine verbosity",
.off1 = offsetof(struct http_options, verbose),
for (i = 0; (c = uri[i]); i++) {
if (n > bufsize-5) {
log_err("encoding the URL failed\n");
+ free(r);
return NULL;
}
return _conv_hex(hash, SHA256_DIGEST_LENGTH);
}
+static char *_gen_hex_md5(const char *p, size_t len)
+{
+ unsigned char hash[MD5_DIGEST_LENGTH];
+
+ MD5((unsigned char*)p, len, hash);
+ return _conv_hex(hash, MD5_DIGEST_LENGTH);
+}
+
+static char *_conv_base64_encode(const unsigned char *p, size_t len)
+{
+ char *r, *ret;
+ int i;
+ static const char sEncodingTable[] = {
+ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
+ 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
+ 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
+ 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
+ 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
+ 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
+ 'w', 'x', 'y', 'z', '0', '1', '2', '3',
+ '4', '5', '6', '7', '8', '9', '+', '/'
+ };
+
+ size_t out_len = 4 * ((len + 2) / 3);
+ ret = r = malloc(out_len + 1);
+
+ for (i = 0; i < len - 2; i += 3) {
+ *r++ = sEncodingTable[(p[i] >> 2) & 0x3F];
+ *r++ = sEncodingTable[((p[i] & 0x3) << 4) | ((int) (p[i + 1] & 0xF0) >> 4)];
+ *r++ = sEncodingTable[((p[i + 1] & 0xF) << 2) | ((int) (p[i + 2] & 0xC0) >> 6)];
+ *r++ = sEncodingTable[p[i + 2] & 0x3F];
+ }
+
+ if (i < len) {
+ *r++ = sEncodingTable[(p[i] >> 2) & 0x3F];
+ if (i == (len - 1)) {
+ *r++ = sEncodingTable[((p[i] & 0x3) << 4)];
+ *r++ = '=';
+ } else {
+ *r++ = sEncodingTable[((p[i] & 0x3) << 4) | ((int) (p[i + 1] & 0xF0) >> 4)];
+ *r++ = sEncodingTable[((p[i + 1] & 0xF) << 2)];
+ }
+ *r++ = '=';
+ }
+
+ ret[out_len]=0;
+ return ret;
+}
+
+static char *_gen_base64_md5(const unsigned char *p, size_t len)
+{
+ unsigned char hash[MD5_DIGEST_LENGTH];
+ MD5((unsigned char*)p, len, hash);
+ return _conv_base64_encode(hash, MD5_DIGEST_LENGTH);
+}
+
static void _hmac(unsigned char *md, void *key, int key_len, char *data) {
#ifndef CONFIG_HAVE_OPAQUE_HMAC_CTX
HMAC_CTX _ctx;
ctx = HMAC_CTX_new();
#else
ctx = &_ctx;
+ /* work-around crash in certain versions of libssl */
+ HMAC_CTX_init(ctx);
#endif
HMAC_Init_ex(ctx, key, key_len, EVP_sha256(), NULL);
HMAC_Update(ctx, (unsigned char*)data, strlen(data));
switch (type) {
case CURLINFO_TEXT:
- fprintf(stderr, "== Info: %s", data);
+ fprintf(stderr, "== Info: %s", data);
+ fio_fallthrough;
default:
case CURLINFO_SSL_DATA_OUT:
case CURLINFO_SSL_DATA_IN:
char date_iso[32];
char method[8];
char dkey[128];
- char creq[512];
- char sts[256];
+ char creq[4096];
+ char sts[512];
char s[512];
char *uri_encoded = NULL;
char *dsha = NULL;
const char *service = "s3";
const char *aws = "aws4_request";
unsigned char md[SHA256_DIGEST_LENGTH];
+ unsigned char sse_key[33] = {0};
+ char *sse_key_base64 = NULL;
+ char *sse_key_md5_base64 = NULL;
time_t t = time(NULL);
struct tm *gtm = gmtime(&t);
strftime (date_iso, sizeof(date_iso), "%Y%m%dT%H%M%SZ", gtm);
uri_encoded = _aws_uriencode(uri);
+ if (o->s3_sse_customer_key != NULL)
+ strncpy((char*)sse_key, o->s3_sse_customer_key, sizeof(sse_key) - 1);
+
if (op == DDIR_WRITE) {
dsha = _gen_hex_sha256(buf, len);
sprintf(method, "PUT");
}
/* Create the canonical request first */
- snprintf(creq, sizeof(creq),
- "%s\n"
- "%s\n"
- "\n"
- "host:%s\n"
- "x-amz-content-sha256:%s\n"
- "x-amz-date:%s\n"
- "\n"
- "host;x-amz-content-sha256;x-amz-date\n"
- "%s"
- , method
- , uri_encoded, o->host, dsha, date_iso, dsha);
+ if (sse_key[0] != '\0') {
+ sse_key_base64 = _conv_base64_encode(sse_key, sizeof(sse_key) - 1);
+ sse_key_md5_base64 = _gen_base64_md5(sse_key, sizeof(sse_key) - 1);
+ snprintf(creq, sizeof(creq),
+ "%s\n"
+ "%s\n"
+ "\n"
+ "host:%s\n"
+ "x-amz-content-sha256:%s\n"
+ "x-amz-date:%s\n"
+ "x-amz-server-side-encryption-customer-algorithm:%s\n"
+ "x-amz-server-side-encryption-customer-key:%s\n"
+ "x-amz-server-side-encryption-customer-key-md5:%s\n"
+ "x-amz-storage-class:%s\n"
+ "\n"
+ "host;x-amz-content-sha256;x-amz-date;"
+ "x-amz-server-side-encryption-customer-algorithm;"
+ "x-amz-server-side-encryption-customer-key;"
+ "x-amz-server-side-encryption-customer-key-md5;"
+ "x-amz-storage-class\n"
+ "%s"
+ , method
+ , uri_encoded, o->host, dsha, date_iso
+ , o->s3_sse_customer_algorithm, sse_key_base64
+ , sse_key_md5_base64, o->s3_storage_class, dsha);
+ } else {
+ snprintf(creq, sizeof(creq),
+ "%s\n"
+ "%s\n"
+ "\n"
+ "host:%s\n"
+ "x-amz-content-sha256:%s\n"
+ "x-amz-date:%s\n"
+ "x-amz-storage-class:%s\n"
+ "\n"
+ "host;x-amz-content-sha256;x-amz-date;x-amz-storage-class\n"
+ "%s"
+ , method
+ , uri_encoded, o->host, dsha, date_iso, o->s3_storage_class, dsha);
+ }
csha = _gen_hex_sha256(creq, strlen(creq));
snprintf(sts, sizeof(sts), "AWS4-HMAC-SHA256\n%s\n%s/%s/%s/%s\n%s",
- date_iso, date_short, o->s3_region, service, aws, csha);
+ date_iso, date_short, o->s3_region, service, aws, csha);
snprintf((char *)dkey, sizeof(dkey), "AWS4%s", o->s3_key);
_hmac(md, dkey, strlen(dkey), date_short);
signature = _conv_hex(md, SHA256_DIGEST_LENGTH);
- /* Surpress automatic Accept: header */
+ /* Suppress automatic Accept: header */
slist = curl_slist_append(slist, "Accept:");
snprintf(s, sizeof(s), "x-amz-content-sha256: %s", dsha);
snprintf(s, sizeof(s), "x-amz-date: %s", date_iso);
slist = curl_slist_append(slist, s);
- snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
- "SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=%s",
- o->s3_keyid, date_short, o->s3_region, signature);
+ if (sse_key[0] != '\0') {
+ snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-algorithm: %s", o->s3_sse_customer_algorithm);
+ slist = curl_slist_append(slist, s);
+ snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-key: %s", sse_key_base64);
+ slist = curl_slist_append(slist, s);
+ snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-key-md5: %s", sse_key_md5_base64);
+ slist = curl_slist_append(slist, s);
+ }
+
+ snprintf(s, sizeof(s), "x-amz-storage-class: %s", o->s3_storage_class);
+ slist = curl_slist_append(slist, s);
+
+ if (sse_key[0] != '\0') {
+ snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
+ "SignedHeaders=host;x-amz-content-sha256;"
+ "x-amz-date;x-amz-server-side-encryption-customer-algorithm;"
+ "x-amz-server-side-encryption-customer-key;"
+ "x-amz-server-side-encryption-customer-key-md5;"
+ "x-amz-storage-class,"
+ "Signature=%s",
+ o->s3_keyid, date_short, o->s3_region, signature);
+ } else {
+ snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
+ "SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=%s",
+ o->s3_keyid, date_short, o->s3_region, signature);
+ }
slist = curl_slist_append(slist, s);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
free(csha);
free(dsha);
free(signature);
+ if (sse_key_base64 != NULL) {
+ free(sse_key_base64);
+ free(sse_key_md5_base64);
+ }
+}
+
+static void _add_swift_header(CURL *curl, struct curl_slist *slist, struct http_options *o,
+ int op, const char *uri, char *buf, size_t len)
+{
+ char *dsha = NULL;
+ char s[512];
+
+ if (op == DDIR_WRITE) {
+ dsha = _gen_hex_md5(buf, len);
+ }
+ /* Suppress automatic Accept: header */
+ slist = curl_slist_append(slist, "Accept:");
+
+ snprintf(s, sizeof(s), "etag: %s", dsha);
+ slist = curl_slist_append(slist, s);
+
+ snprintf(s, sizeof(s), "x-auth-token: %s", o->swift_auth_token);
+ slist = curl_slist_append(slist, s);
+
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
+
+ free(dsha);
}
static void fio_http_cleanup(struct thread_data *td)
char url[1024];
long status;
CURLcode res;
- int r = -1;
fio_ro_check(td, io_u);
memset(&_curl_stream, 0, sizeof(_curl_stream));
- snprintf(object, sizeof(object), "%s_%llu_%llu", td->files[0]->file_name, io_u->offset, io_u->xfer_buflen);
- snprintf(url, sizeof(url), "%s://%s%s", o->https ? "https" : "http", o->host, object);
+ snprintf(object, sizeof(object), "%s_%llu_%llu", td->files[0]->file_name,
+ io_u->offset, io_u->xfer_buflen);
+ if (o->https == FIO_HTTPS_OFF)
+ snprintf(url, sizeof(url), "http://%s%s", o->host, object);
+ else
+ snprintf(url, sizeof(url), "https://%s%s", o->host, object);
curl_easy_setopt(http->curl, CURLOPT_URL, url);
_curl_stream.buf = io_u->xfer_buf;
_curl_stream.max = io_u->xfer_buflen;
curl_easy_setopt(http->curl, CURLOPT_SEEKDATA, &_curl_stream);
curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)io_u->xfer_buflen);
- if (o->s3)
+ if (o->mode == FIO_HTTP_S3)
_add_aws_auth_header(http->curl, slist, o, io_u->ddir, object,
io_u->xfer_buf, io_u->xfer_buflen);
+ else if (o->mode == FIO_HTTP_SWIFT)
+ _add_swift_header(http->curl, slist, o, io_u->ddir, object,
+ io_u->xfer_buf, io_u->xfer_buflen);
if (io_u->ddir == DDIR_WRITE) {
curl_easy_setopt(http->curl, CURLOPT_READDATA, &_curl_stream);
if (status == 100 || (status >= 200 && status <= 204))
goto out;
log_err("DDIR_WRITE failed with HTTP status code %ld\n", status);
- goto err;
}
+ goto err;
} else if (io_u->ddir == DDIR_READ) {
curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, &_curl_stream);
} else if (io_u->ddir == DDIR_TRIM) {
curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
curl_easy_setopt(http->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
- curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, 0);
+ curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)0);
curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
res = curl_easy_perform(http->curl);
log_err("WARNING: Only DDIR_READ/DDIR_WRITE/DDIR_TRIM are supported!\n");
err:
- io_u->error = r;
+ io_u->error = EIO;
td_verror(td, io_u->error, "transfer");
out:
curl_slist_free_all(slist);
{
struct http_data *http = NULL;
struct http_options *o = td->eo;
- int r;
+
/* allocate engine specific structure to deal with libhttp. */
http = calloc(1, sizeof(*http));
if (!http) {
curl_easy_setopt(http->curl, CURLOPT_NOPROGRESS, 1L);
curl_easy_setopt(http->curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(http->curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP|CURLPROTO_HTTPS);
+ if (o->https == FIO_HTTPS_INSECURE) {
+ curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYPEER, 0L);
+ curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYHOST, 0L);
+ }
curl_easy_setopt(http->curl, CURLOPT_READFUNCTION, _http_read);
curl_easy_setopt(http->curl, CURLOPT_WRITEFUNCTION, _http_write);
- curl_easy_setopt(http->curl, CURLOPT_SEEKFUNCTION, _http_seek);
+ curl_easy_setopt(http->curl, CURLOPT_SEEKFUNCTION, &_http_seek);
if (o->user && o->pass) {
curl_easy_setopt(http->curl, CURLOPT_USERNAME, o->user);
curl_easy_setopt(http->curl, CURLOPT_PASSWORD, o->pass);
return 0;
cleanup:
fio_http_cleanup(td);
- return r;
+ return 1;
}
static int fio_http_open(struct thread_data *td, struct fio_file *f)
return 0;
}
-static struct ioengine_ops ioengine = {
+FIO_STATIC struct ioengine_ops ioengine = {
.name = "http",
.version = FIO_IOOPS_VERSION,
- .flags = FIO_DISKLESSIO,
+ .flags = FIO_DISKLESSIO | FIO_SYNCIO,
.setup = fio_http_setup,
.queue = fio_http_queue,
.getevents = fio_http_getevents,
projects / fio.git / blobdiff