[fio.git] / engines / http.c

/*
 * HTTP GET/PUT IO engine
 *
 * IO engine to perform HTTP(S) GET/PUT requests via libcurl-easy.
 *
 * Copyright (C) 2018 SUSE LLC
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License,
 * version 2 as published by the Free Software Foundation..
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the Free
 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */

#include <pthread.h>
#include <time.h>
#include <curl/curl.h>
#include <openssl/hmac.h>
#include <openssl/sha.h>
#include <openssl/md5.h>
#include "fio.h"
#include "../optgroup.h"

/*
 * Silence OpenSSL 3.0 deprecated function warnings
 */
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"

enum {
	FIO_HTTP_WEBDAV	    = 0,
	FIO_HTTP_S3	    = 1,
	FIO_HTTP_SWIFT	    = 2,

	FIO_HTTPS_OFF	    = 0,
	FIO_HTTPS_ON	    = 1,
	FIO_HTTPS_INSECURE  = 2,
};

struct http_data {
	CURL *curl;
};

struct http_options {
	void *pad;
	unsigned int https;
	char *host;
	char *user;
	char *pass;
	char *s3_key;
	char *s3_keyid;
	char *s3_region;
	char *s3_storage_class;
	char *swift_auth_token;
	int verbose;
	unsigned int mode;
};

struct http_curl_stream {
	char *buf;
	size_t pos;
	size_t max;
};

static struct fio_option options[] = {
	{
		.name     = "https",
		.lname    = "https",
		.type     = FIO_OPT_STR,
		.help     = "Enable https",
		.off1     = offsetof(struct http_options, https),
		.def      = "off",
		.posval = {
			  { .ival = "off",
			    .oval = FIO_HTTPS_OFF,
			    .help = "No HTTPS",
			  },
			  { .ival = "on",
			    .oval = FIO_HTTPS_ON,
			    .help = "Enable HTTPS",
			  },
			  { .ival = "insecure",
			    .oval = FIO_HTTPS_INSECURE,
			    .help = "Enable HTTPS, disable peer verification",
			  },
		},
		.category = FIO_OPT_C_ENGINE,
		.group    = FIO_OPT_G_HTTP,
	},
	{
		.name     = "http_host",
		.lname    = "http_host",
		.type     = FIO_OPT_STR_STORE,
		.help     = "Hostname (S3 bucket)",
		.off1     = offsetof(struct http_options, host),
		.def	  = "localhost",
		.category = FIO_OPT_C_ENGINE,
		.group    = FIO_OPT_G_HTTP,
	},
	{
		.name     = "http_user",
		.lname    = "http_user",
		.type     = FIO_OPT_STR_STORE,
		.help     = "HTTP user name",
		.off1     = offsetof(struct http_options, user),
		.category = FIO_OPT_C_ENGINE,
		.group    = FIO_OPT_G_HTTP,
	},
	{
		.name     = "http_pass",
		.lname    = "http_pass",
		.type     = FIO_OPT_STR_STORE,
		.help     = "HTTP password",
		.off1     = offsetof(struct http_options, pass),
		.category = FIO_OPT_C_ENGINE,
		.group    = FIO_OPT_G_HTTP,
	},
	{
		.name     = "http_s3_key",
		.lname    = "S3 secret key",
		.type     = FIO_OPT_STR_STORE,
		.help     = "S3 secret key",
		.off1     = offsetof(struct http_options, s3_key),
		.def	  = "",
		.category = FIO_OPT_C_ENGINE,
		.group    = FIO_OPT_G_HTTP,
	},
	{
		.name     = "http_s3_keyid",
		.lname    = "S3 key id",
		.type     = FIO_OPT_STR_STORE,
		.help     = "S3 key id",
		.off1     = offsetof(struct http_options, s3_keyid),
		.def	  = "",
		.category = FIO_OPT_C_ENGINE,
		.group    = FIO_OPT_G_HTTP,
	},
	{
		.name     = "http_swift_auth_token",
		.lname    = "Swift auth token",
		.type     = FIO_OPT_STR_STORE,
		.help     = "OpenStack Swift auth token",
		.off1     = offsetof(struct http_options, swift_auth_token),
		.def	  = "",
		.category = FIO_OPT_C_ENGINE,
		.group    = FIO_OPT_G_HTTP,
	},
	{
		.name     = "http_s3_region",
		.lname    = "S3 region",
		.type     = FIO_OPT_STR_STORE,
		.help     = "S3 region",
		.off1     = offsetof(struct http_options, s3_region),
		.def	  = "us-east-1",
		.category = FIO_OPT_C_ENGINE,
		.group    = FIO_OPT_G_HTTP,
	},
	{
		.name     = "http_s3_storage_class",
		.lname    = "S3 Storage class",
		.type     = FIO_OPT_STR_STORE,
		.help     = "S3 Storage Class",
		.off1     = offsetof(struct http_options, s3_storage_class),
		.def	  = "STANDARD",
		.category = FIO_OPT_C_ENGINE,
		.group    = FIO_OPT_G_HTTP,
	},
	{
		.name     = "http_mode",
		.lname    = "Request mode to use",
		.type     = FIO_OPT_STR,
		.help     = "Whether to use WebDAV, Swift, or S3",
		.off1     = offsetof(struct http_options, mode),
		.def	  = "webdav",
		.posval = {
			  { .ival = "webdav",
			    .oval = FIO_HTTP_WEBDAV,
			    .help = "WebDAV server",
			  },
			  { .ival = "s3",
			    .oval = FIO_HTTP_S3,
			    .help = "S3 storage backend",
			  },
			  { .ival = "swift",
			    .oval = FIO_HTTP_SWIFT,
			    .help = "OpenStack Swift storage",
			  },
		},
		.category = FIO_OPT_C_ENGINE,
		.group    = FIO_OPT_G_HTTP,
	},
	{
		.name     = "http_verbose",
		.lname    = "HTTP verbosity level",
		.type     = FIO_OPT_INT,
		.help     = "increase http engine verbosity",
		.off1     = offsetof(struct http_options, verbose),
		.def	  = "0",
		.category = FIO_OPT_C_ENGINE,
		.group    = FIO_OPT_G_HTTP,
	},
	{
		.name     = NULL,
	},
};

static char *_aws_uriencode(const char *uri)
{
	size_t bufsize = 1024;
	char *r = malloc(bufsize);
	char c;
	int i, n;
	const char *hex = "0123456789ABCDEF";

	if (!r) {
		log_err("malloc failed\n");
		return NULL;
	}

	n = 0;
	for (i = 0; (c = uri[i]); i++) {
		if (n > bufsize-5) {
			log_err("encoding the URL failed\n");
			return NULL;
		}

		if ( (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
		|| (c >= '0' && c <= '9') || c == '_' || c == '-'
		|| c == '~' || c == '.' || c == '/')
			r[n++] = c;
		else {
			r[n++] = '%';
			r[n++] = hex[(c >> 4 ) & 0xF];
			r[n++] = hex[c & 0xF];
		}
	}
	r[n++] = 0;
	return r;
}

static char *_conv_hex(const unsigned char *p, size_t len)
{
	char *r;
	int i,n;
	const char *hex = "0123456789abcdef";
	r = malloc(len * 2 + 1);
	n = 0;
	for (i = 0; i < len; i++) {
		r[n++] = hex[(p[i] >> 4 ) & 0xF];
		r[n++] = hex[p[i] & 0xF];
	}
	r[n] = 0;

	return r;
}

static char *_gen_hex_sha256(const char *p, size_t len)
{
	unsigned char hash[SHA256_DIGEST_LENGTH];

	SHA256((unsigned char*)p, len, hash);
	return _conv_hex(hash, SHA256_DIGEST_LENGTH);
}

static char *_gen_hex_md5(const char *p, size_t len)
{
	unsigned char hash[MD5_DIGEST_LENGTH];

	MD5((unsigned char*)p, len, hash);
	return _conv_hex(hash, MD5_DIGEST_LENGTH);
}

static void _hmac(unsigned char *md, void *key, int key_len, char *data) {
#ifndef CONFIG_HAVE_OPAQUE_HMAC_CTX
	HMAC_CTX _ctx;
#endif
	HMAC_CTX *ctx;
	unsigned int hmac_len;

#ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX
	ctx = HMAC_CTX_new();
#else
	ctx = &_ctx;
	/* work-around crash in certain versions of libssl */
	HMAC_CTX_init(ctx);
#endif
	HMAC_Init_ex(ctx, key, key_len, EVP_sha256(), NULL);
	HMAC_Update(ctx, (unsigned char*)data, strlen(data));
	HMAC_Final(ctx, md, &hmac_len);
#ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX
	HMAC_CTX_free(ctx);
#else
	HMAC_CTX_cleanup(ctx);
#endif
}

static int _curl_trace(CURL *handle, curl_infotype type,
	     char *data, size_t size,
	     void *userp)
{
	const char *text;
	(void)handle; /* prevent compiler warning */
	(void)userp;

	switch (type) {
	case CURLINFO_TEXT:
		fprintf(stderr, "== Info: %s", data);
		fio_fallthrough;
	default:
	case CURLINFO_SSL_DATA_OUT:
	case CURLINFO_SSL_DATA_IN:
		return 0;

	case CURLINFO_HEADER_OUT:
		text = "=> Send header";
		break;
	case CURLINFO_DATA_OUT:
		text = "=> Send data";
		break;
	case CURLINFO_HEADER_IN:
		text = "<= Recv header";
		break;
	case CURLINFO_DATA_IN:
		text = "<= Recv data";
		break;
	}

	log_info("%s: %s", text, data);
	return 0;
}

/* https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
 * https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
 */
static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct http_options *o,
		int op, const char *uri, char *buf, size_t len)
{
	char date_short[16];
	char date_iso[32];
	char method[8];
	char dkey[128];
	char creq[4096];
	char sts[512];
	char s[512];
	char *uri_encoded = NULL;
	char *dsha = NULL;
	char *csha = NULL;
	char *signature = NULL;
	const char *service = "s3";
	const char *aws = "aws4_request";
	unsigned char md[SHA256_DIGEST_LENGTH];

	time_t t = time(NULL);
	struct tm *gtm = gmtime(&t);

	strftime (date_short, sizeof(date_short), "%Y%m%d", gtm);
	strftime (date_iso, sizeof(date_iso), "%Y%m%dT%H%M%SZ", gtm);
	uri_encoded = _aws_uriencode(uri);

	if (op == DDIR_WRITE) {
		dsha = _gen_hex_sha256(buf, len);
		sprintf(method, "PUT");
	} else {
		/* DDIR_READ && DDIR_TRIM supply an empty body */
		if (op == DDIR_READ)
			sprintf(method, "GET");
		else
			sprintf(method, "DELETE");
		dsha = _gen_hex_sha256("", 0);
	}

	/* Create the canonical request first */
	snprintf(creq, sizeof(creq),
	"%s\n"
	"%s\n"
	"\n"
	"host:%s\n"
	"x-amz-content-sha256:%s\n"
	"x-amz-date:%s\n"
	"x-amz-storage-class:%s\n"
	"\n"
	"host;x-amz-content-sha256;x-amz-date;x-amz-storage-class\n"
	"%s"
	, method
	, uri_encoded, o->host, dsha, date_iso, o->s3_storage_class, dsha);

	csha = _gen_hex_sha256(creq, strlen(creq));
	snprintf(sts, sizeof(sts), "AWS4-HMAC-SHA256\n%s\n%s/%s/%s/%s\n%s",
		date_iso, date_short, o->s3_region, service, aws, csha);

	snprintf((char *)dkey, sizeof(dkey), "AWS4%s", o->s3_key);
	_hmac(md, dkey, strlen(dkey), date_short);
	_hmac(md, md, SHA256_DIGEST_LENGTH, o->s3_region);
	_hmac(md, md, SHA256_DIGEST_LENGTH, (char*) service);
	_hmac(md, md, SHA256_DIGEST_LENGTH, (char*) aws);
	_hmac(md, md, SHA256_DIGEST_LENGTH, sts);

	signature = _conv_hex(md, SHA256_DIGEST_LENGTH);

	/* Suppress automatic Accept: header */
	slist = curl_slist_append(slist, "Accept:");

	snprintf(s, sizeof(s), "x-amz-content-sha256: %s", dsha);
	slist = curl_slist_append(slist, s);

	snprintf(s, sizeof(s), "x-amz-date: %s", date_iso);
	slist = curl_slist_append(slist, s);
	snprintf(s, sizeof(s), "x-amz-storage-class: %s", o->s3_storage_class);
	slist = curl_slist_append(slist, s);
	snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
	"SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=%s",
	o->s3_keyid, date_short, o->s3_region, signature);
	slist = curl_slist_append(slist, s);

	curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);

	free(uri_encoded);
	free(csha);
	free(dsha);
	free(signature);
}

static void _add_swift_header(CURL *curl, struct curl_slist *slist, struct http_options *o,
		int op, const char *uri, char *buf, size_t len)
{
	char *dsha = NULL;
	char s[512];

	if (op == DDIR_WRITE) {
		dsha = _gen_hex_md5(buf, len);
	}
	/* Suppress automatic Accept: header */
	slist = curl_slist_append(slist, "Accept:");

	snprintf(s, sizeof(s), "etag: %s", dsha);
	slist = curl_slist_append(slist, s);

	snprintf(s, sizeof(s), "x-auth-token: %s", o->swift_auth_token);
	slist = curl_slist_append(slist, s);

	curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);

	free(dsha);
}

static void fio_http_cleanup(struct thread_data *td)
{
	struct http_data *http = td->io_ops_data;

	if (http) {
		curl_easy_cleanup(http->curl);
		free(http);
	}
}

static size_t _http_read(void *ptr, size_t size, size_t nmemb, void *stream)
{
	struct http_curl_stream *state = stream;
	size_t len = size * nmemb;
	/* We're retrieving; nothing is supposed to be read locally */
	if (!stream)
		return 0;
	if (len+state->pos > state->max)
		len = state->max - state->pos;
	memcpy(ptr, &state->buf[state->pos], len);
	state->pos += len;
	return len;
}

static size_t _http_write(void *ptr, size_t size, size_t nmemb, void *stream)
{
	struct http_curl_stream *state = stream;
	/* We're just discarding the returned body after a PUT */
	if (!stream)
		return nmemb;
	if (size != 1)
		return CURLE_WRITE_ERROR;
	if (nmemb + state->pos > state->max)
		return CURLE_WRITE_ERROR;
	memcpy(&state->buf[state->pos], ptr, nmemb);
	state->pos += nmemb;
	return nmemb;
}

static int _http_seek(void *stream, curl_off_t offset, int origin)
{
	struct http_curl_stream *state = stream;
	if (offset < state->max && origin == SEEK_SET) {
		state->pos = offset;
		return CURL_SEEKFUNC_OK;
	} else
		return CURL_SEEKFUNC_FAIL;
}

static enum fio_q_status fio_http_queue(struct thread_data *td,
					 struct io_u *io_u)
{
	struct http_data *http = td->io_ops_data;
	struct http_options *o = td->eo;
	struct http_curl_stream _curl_stream;
	struct curl_slist *slist = NULL;
	char object[512];
	char url[1024];
	long status;
	CURLcode res;
	int r = -1;

	fio_ro_check(td, io_u);
	memset(&_curl_stream, 0, sizeof(_curl_stream));
	snprintf(object, sizeof(object), "%s_%llu_%llu", td->files[0]->file_name,
		io_u->offset, io_u->xfer_buflen);
	if (o->https == FIO_HTTPS_OFF)
		snprintf(url, sizeof(url), "http://%s%s", o->host, object);
	else
		snprintf(url, sizeof(url), "https://%s%s", o->host, object);
	curl_easy_setopt(http->curl, CURLOPT_URL, url);
	_curl_stream.buf = io_u->xfer_buf;
	_curl_stream.max = io_u->xfer_buflen;
	curl_easy_setopt(http->curl, CURLOPT_SEEKDATA, &_curl_stream);
	curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)io_u->xfer_buflen);

	if (o->mode == FIO_HTTP_S3)
		_add_aws_auth_header(http->curl, slist, o, io_u->ddir, object,
			io_u->xfer_buf, io_u->xfer_buflen);
	else if (o->mode == FIO_HTTP_SWIFT)
		_add_swift_header(http->curl, slist, o, io_u->ddir, object,
			io_u->xfer_buf, io_u->xfer_buflen);

	if (io_u->ddir == DDIR_WRITE) {
		curl_easy_setopt(http->curl, CURLOPT_READDATA, &_curl_stream);
		curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
		curl_easy_setopt(http->curl, CURLOPT_UPLOAD, 1L);
		res = curl_easy_perform(http->curl);
		if (res == CURLE_OK) {
			curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
			if (status == 100 || (status >= 200 && status <= 204))
				goto out;
			log_err("DDIR_WRITE failed with HTTP status code %ld\n", status);
		}
		goto err;
	} else if (io_u->ddir == DDIR_READ) {
		curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
		curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, &_curl_stream);
		curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
		res = curl_easy_perform(http->curl);
		if (res == CURLE_OK) {
			curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
			if (status == 200)
				goto out;
			else if (status == 404) {
				/* Object doesn't exist. Pretend we read
				 * zeroes */
				memset(io_u->xfer_buf, 0, io_u->xfer_buflen);
				goto out;
			}
			log_err("DDIR_READ failed with HTTP status code %ld\n", status);
		}
		goto err;
	} else if (io_u->ddir == DDIR_TRIM) {
		curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
		curl_easy_setopt(http->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
		curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)0);
		curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
		curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
		res = curl_easy_perform(http->curl);
		if (res == CURLE_OK) {
			curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
			if (status == 200 || status == 202 || status == 204 || status == 404)
				goto out;
			log_err("DDIR_TRIM failed with HTTP status code %ld\n", status);
		}
		goto err;
	}

	log_err("WARNING: Only DDIR_READ/DDIR_WRITE/DDIR_TRIM are supported!\n");

err:
	io_u->error = r;
	td_verror(td, io_u->error, "transfer");
out:
	curl_slist_free_all(slist);
	return FIO_Q_COMPLETED;
}

static struct io_u *fio_http_event(struct thread_data *td, int event)
{
	/* sync IO engine - never any outstanding events */
	return NULL;
}

int fio_http_getevents(struct thread_data *td, unsigned int min,
	unsigned int max, const struct timespec *t)
{
	/* sync IO engine - never any outstanding events */
	return 0;
}

static int fio_http_setup(struct thread_data *td)
{
	struct http_data *http = NULL;
	struct http_options *o = td->eo;

	/* allocate engine specific structure to deal with libhttp. */
	http = calloc(1, sizeof(*http));
	if (!http) {
		log_err("calloc failed.\n");
		goto cleanup;
	}

	http->curl = curl_easy_init();
	if (o->verbose)
		curl_easy_setopt(http->curl, CURLOPT_VERBOSE, 1L);
	if (o->verbose > 1)
		curl_easy_setopt(http->curl, CURLOPT_DEBUGFUNCTION, &_curl_trace);
	curl_easy_setopt(http->curl, CURLOPT_NOPROGRESS, 1L);
	curl_easy_setopt(http->curl, CURLOPT_FOLLOWLOCATION, 1L);
	curl_easy_setopt(http->curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP|CURLPROTO_HTTPS);
	if (o->https == FIO_HTTPS_INSECURE) {
		curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYPEER, 0L);
		curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYHOST, 0L);
	}
	curl_easy_setopt(http->curl, CURLOPT_READFUNCTION, _http_read);
	curl_easy_setopt(http->curl, CURLOPT_WRITEFUNCTION, _http_write);
	curl_easy_setopt(http->curl, CURLOPT_SEEKFUNCTION, &_http_seek);
	if (o->user && o->pass) {
		curl_easy_setopt(http->curl, CURLOPT_USERNAME, o->user);
		curl_easy_setopt(http->curl, CURLOPT_PASSWORD, o->pass);
		curl_easy_setopt(http->curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
	}

	td->io_ops_data = http;

	/* Force single process mode. */
	td->o.use_thread = 1;

	return 0;
cleanup:
	fio_http_cleanup(td);
	return 1;
}

static int fio_http_open(struct thread_data *td, struct fio_file *f)
{
	return 0;
}
static int fio_http_invalidate(struct thread_data *td, struct fio_file *f)
{
	return 0;
}

FIO_STATIC struct ioengine_ops ioengine = {
	.name = "http",
	.version		= FIO_IOOPS_VERSION,
	.flags			= FIO_DISKLESSIO | FIO_SYNCIO,
	.setup			= fio_http_setup,
	.queue			= fio_http_queue,
	.getevents		= fio_http_getevents,
	.event			= fio_http_event,
	.cleanup		= fio_http_cleanup,
	.open_file		= fio_http_open,
	.invalidate		= fio_http_invalidate,
	.options		= options,
	.option_struct_size	= sizeof(struct http_options),
};

static void fio_init fio_http_register(void)
{
	register_ioengine(&ioengine);
}

static void fio_exit fio_http_unregister(void)
{
	unregister_ioengine(&ioengine);
}
Commit	Line	Data
c2f6a13d LMB	1	/*
	2	* HTTP GET/PUT IO engine
	3	*
	4	* IO engine to perform HTTP(S) GET/PUT requests via libcurl-easy.
	5	*
	6	* Copyright (C) 2018 SUSE LLC
	7	*
	8	* This program is free software; you can redistribute it and/or
	9	* modify it under the terms of the GNU General Public License,
	10	* version 2 as published by the Free Software Foundation..
	11	*
	12	* This program is distributed in the hope that it will be useful,
	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	* GNU General Public License for more details.
	16	*
	17	* You should have received a copy of the GNU General Public
	18	* License along with this program; if not, write to the Free
	19	* Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
	20	* Boston, MA 02110-1301, USA.
	21	*/
	22
	23	#include <pthread.h>
	24	#include <time.h>
	25	#include <curl/curl.h>
	26	#include <openssl/hmac.h>
	27	#include <openssl/sha.h>
09fd2966	28	#include <openssl/md5.h>
c2f6a13d LMB	29	#include "fio.h"
	30	#include "../optgroup.h"
	31
9c1c1a8d JA	32	/*
	33	* Silence OpenSSL 3.0 deprecated function warnings
	34	*/
	35	#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
c2f6a13d	36
09fd2966 LMB	37	enum {
	38	FIO_HTTP_WEBDAV = 0,
	39	FIO_HTTP_S3 = 1,
	40	FIO_HTTP_SWIFT = 2,
	41
	42	FIO_HTTPS_OFF = 0,
	43	FIO_HTTPS_ON = 1,
	44	FIO_HTTPS_INSECURE = 2,
	45	};
	46
c2f6a13d LMB	47	struct http_data {
	48	CURL *curl;
	49	};
	50
	51	struct http_options {
	52	void *pad;
09fd2966	53	unsigned int https;
c2f6a13d LMB	54	char *host;
	55	char *user;
	56	char *pass;
	57	char *s3_key;
	58	char *s3_keyid;
	59	char *s3_region;
f2d79184	60	char *s3_storage_class;
09fd2966	61	char *swift_auth_token;
c2f6a13d	62	int verbose;
09fd2966	63	unsigned int mode;
c2f6a13d LMB	64	};
	65
	66	struct http_curl_stream {
	67	char *buf;
	68	size_t pos;
	69	size_t max;
	70	};
	71
	72	static struct fio_option options[] = {
	73	{
	74	.name = "https",
	75	.lname = "https",
09fd2966	76	.type = FIO_OPT_STR,
c2f6a13d LMB	77	.help = "Enable https",
c2f6a13d LMB	78	.off1 = offsetof(struct http_options, https),
09fd2966 LMB	79	.def = "off",
	80	.posval = {
	81	{ .ival = "off",
	82	.oval = FIO_HTTPS_OFF,
	83	.help = "No HTTPS",
	84	},
	85	{ .ival = "on",
	86	.oval = FIO_HTTPS_ON,
	87	.help = "Enable HTTPS",
	88	},
	89	{ .ival = "insecure",
	90	.oval = FIO_HTTPS_INSECURE,
	91	.help = "Enable HTTPS, disable peer verification",
	92	},
	93	},
c2f6a13d LMB	94	.category = FIO_OPT_C_ENGINE,
	95	.group = FIO_OPT_G_HTTP,
	96	},
	97	{
	98	.name = "http_host",
	99	.lname = "http_host",
	100	.type = FIO_OPT_STR_STORE,
	101	.help = "Hostname (S3 bucket)",
	102	.off1 = offsetof(struct http_options, host),
	103	.def = "localhost",
	104	.category = FIO_OPT_C_ENGINE,
	105	.group = FIO_OPT_G_HTTP,
	106	},
	107	{
	108	.name = "http_user",
	109	.lname = "http_user",
	110	.type = FIO_OPT_STR_STORE,
	111	.help = "HTTP user name",
	112	.off1 = offsetof(struct http_options, user),
	113	.category = FIO_OPT_C_ENGINE,
	114	.group = FIO_OPT_G_HTTP,
	115	},
	116	{
	117	.name = "http_pass",
	118	.lname = "http_pass",
	119	.type = FIO_OPT_STR_STORE,
	120	.help = "HTTP password",
	121	.off1 = offsetof(struct http_options, pass),
	122	.category = FIO_OPT_C_ENGINE,
	123	.group = FIO_OPT_G_HTTP,
	124	},
	125	{
	126	.name = "http_s3_key",
	127	.lname = "S3 secret key",
	128	.type = FIO_OPT_STR_STORE,
	129	.help = "S3 secret key",
	130	.off1 = offsetof(struct http_options, s3_key),
	131	.def = "",
	132	.category = FIO_OPT_C_ENGINE,
	133	.group = FIO_OPT_G_HTTP,
	134	},
	135	{
	136	.name = "http_s3_keyid",
	137	.lname = "S3 key id",
	138	.type = FIO_OPT_STR_STORE,
	139	.help = "S3 key id",
	140	.off1 = offsetof(struct http_options, s3_keyid),
	141	.def = "",
	142	.category = FIO_OPT_C_ENGINE,
	143	.group = FIO_OPT_G_HTTP,
	144	},
09fd2966 LMB	145	{
	146	.name = "http_swift_auth_token",
	147	.lname = "Swift auth token",
	148	.type = FIO_OPT_STR_STORE,
	149	.help = "OpenStack Swift auth token",
	150	.off1 = offsetof(struct http_options, swift_auth_token),
	151	.def = "",
	152	.category = FIO_OPT_C_ENGINE,
	153	.group = FIO_OPT_G_HTTP,
	154	},
c2f6a13d LMB	155	{
	156	.name = "http_s3_region",
	157	.lname = "S3 region",
	158	.type = FIO_OPT_STR_STORE,
	159	.help = "S3 region",
	160	.off1 = offsetof(struct http_options, s3_region),
	161	.def = "us-east-1",
	162	.category = FIO_OPT_C_ENGINE,
	163	.group = FIO_OPT_G_HTTP,
	164	},
f2d79184 FH	165	{
	166	.name = "http_s3_storage_class",
	167	.lname = "S3 Storage class",
	168	.type = FIO_OPT_STR_STORE,
	169	.help = "S3 Storage Class",
	170	.off1 = offsetof(struct http_options, s3_storage_class),
	171	.def = "STANDARD",
	172	.category = FIO_OPT_C_ENGINE,
	173	.group = FIO_OPT_G_HTTP,
	174	},
c2f6a13d	175	{
09fd2966 LMB	176	.name = "http_mode",
	177	.lname = "Request mode to use",
	178	.type = FIO_OPT_STR,
	179	.help = "Whether to use WebDAV, Swift, or S3",
	180	.off1 = offsetof(struct http_options, mode),
	181	.def = "webdav",
	182	.posval = {
	183	{ .ival = "webdav",
	184	.oval = FIO_HTTP_WEBDAV,
	185	.help = "WebDAV server",
	186	},
	187	{ .ival = "s3",
	188	.oval = FIO_HTTP_S3,
	189	.help = "S3 storage backend",
	190	},
	191	{ .ival = "swift",
	192	.oval = FIO_HTTP_SWIFT,
	193	.help = "OpenStack Swift storage",
	194	},
	195	},
c2f6a13d LMB	196	.category = FIO_OPT_C_ENGINE,
	197	.group = FIO_OPT_G_HTTP,
	198	},
	199	{
	200	.name = "http_verbose",
09fd2966	201	.lname = "HTTP verbosity level",
c2f6a13d LMB	202	.type = FIO_OPT_INT,
	203	.help = "increase http engine verbosity",
	204	.off1 = offsetof(struct http_options, verbose),
	205	.def = "0",
	206	.category = FIO_OPT_C_ENGINE,
	207	.group = FIO_OPT_G_HTTP,
	208	},
	209	{
	210	.name = NULL,
	211	},
	212	};
	213
	214	static char _aws_uriencode(const char uri)
	215	{
	216	size_t bufsize = 1024;
	217	char *r = malloc(bufsize);
	218	char c;
	219	int i, n;
	220	const char *hex = "0123456789ABCDEF";
	221
	222	if (!r) {
	223	log_err("malloc failed\n");
	224	return NULL;
	225	}
	226
	227	n = 0;
	228	for (i = 0; (c = uri[i]); i++) {
	229	if (n > bufsize-5) {
	230	log_err("encoding the URL failed\n");
	231	return NULL;
	232	}
	233
	234	if ( (c >= 'A' && c <= 'Z') \|\| (c >= 'a' && c <= 'z')
	235	\|\| (c >= '0' && c <= '9') \|\| c == '_' \|\| c == '-'
	236	\|\| c == '~' \|\| c == '.' \|\| c == '/')
	237	r[n++] = c;
	238	else {
	239	r[n++] = '%';
	240	r[n++] = hex[(c >> 4 ) & 0xF];
	241	r[n++] = hex[c & 0xF];
	242	}
	243	}
	244	r[n++] = 0;
	245	return r;
	246	}
	247
	248	static char _conv_hex(const unsigned char p, size_t len)
	249	{
	250	char *r;
	251	int i,n;
	252	const char *hex = "0123456789abcdef";
	253	r = malloc(len * 2 + 1);
	254	n = 0;
	255	for (i = 0; i < len; i++) {
	256	r[n++] = hex[(p[i] >> 4 ) & 0xF];
	257	r[n++] = hex[p[i] & 0xF];
	258	}
	259	r[n] = 0;
	260
	261	return r;
	262	}
	263
	264	static char _gen_hex_sha256(const char p, size_t len)
	265	{
266	unsigned char hash[SHA256_DIGEST_LENGTH];
267
268	SHA256((unsigned char*)p, len, hash);
269	return _conv_hex(hash, SHA256_DIGEST_LENGTH);
270	}
271
09fd2966 LMB	272	static char _gen_hex_md5(const char p, size_t len)
	273	{
	274	unsigned char hash[MD5_DIGEST_LENGTH];
	275
	276	MD5((unsigned char*)p, len, hash);
	277	return _conv_hex(hash, MD5_DIGEST_LENGTH);
	278	}
	279
c2f6a13d	280	static void _hmac(unsigned char md, void key, int key_len, char *data) {
b61a5f46 DD	281	#ifndef CONFIG_HAVE_OPAQUE_HMAC_CTX
	282	HMAC_CTX _ctx;
	283	#endif
c2f6a13d LMB	284	HMAC_CTX *ctx;
	285	unsigned int hmac_len;
	286
b61a5f46	287	#ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX
c2f6a13d	288	ctx = HMAC_CTX_new();
b61a5f46 DD	289	#else
b61a5f46 DD	290	ctx = &_ctx;
e8aaa776 JA	291	/* work-around crash in certain versions of libssl */
e8aaa776 JA	292	HMAC_CTX_init(ctx);
b61a5f46	293	#endif
c2f6a13d LMB	294	HMAC_Init_ex(ctx, key, key_len, EVP_sha256(), NULL);
	295	HMAC_Update(ctx, (unsigned char*)data, strlen(data));
	296	HMAC_Final(ctx, md, &hmac_len);
b61a5f46	297	#ifdef CONFIG_HAVE_OPAQUE_HMAC_CTX
c2f6a13d	298	HMAC_CTX_free(ctx);
b61a5f46 DD	299	#else
	300	HMAC_CTX_cleanup(ctx);
	301	#endif
c2f6a13d LMB	302	}
	303
	304	static int _curl_trace(CURL *handle, curl_infotype type,
	305	char *data, size_t size,
	306	void *userp)
	307	{
	308	const char *text;
	309	(void)handle; /* prevent compiler warning */
	310	(void)userp;
	311
	312	switch (type) {
	313	case CURLINFO_TEXT:
24010223	314	fprintf(stderr, "== Info: %s", data);
87933e32	315	fio_fallthrough;
c2f6a13d LMB	316	default:
	317	case CURLINFO_SSL_DATA_OUT:
	318	case CURLINFO_SSL_DATA_IN:
	319	return 0;
	320
	321	case CURLINFO_HEADER_OUT:
	322	text = "=> Send header";
	323	break;
	324	case CURLINFO_DATA_OUT:
	325	text = "=> Send data";
	326	break;
	327	case CURLINFO_HEADER_IN:
	328	text = "<= Recv header";
	329	break;
	330	case CURLINFO_DATA_IN:
	331	text = "<= Recv data";
	332	break;
	333	}
	334
	335	log_info("%s: %s", text, data);
	336	return 0;
	337	}
	338
	339	/* https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
	340	* https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
	341	*/
	342	static void _add_aws_auth_header(CURL curl, struct curl_slist slist, struct http_options *o,
	343	int op, const char uri, char buf, size_t len)
	344	{
	345	char date_short[16];
	346	char date_iso[32];
	347	char method[8];
	348	char dkey[128];
f2d79184 FH	349	char creq[4096];
f2d79184 FH	350	char sts[512];
c2f6a13d LMB	351	char s[512];
	352	char *uri_encoded = NULL;
	353	char *dsha = NULL;
	354	char *csha = NULL;
	355	char *signature = NULL;
	356	const char *service = "s3";
	357	const char *aws = "aws4_request";
	358	unsigned char md[SHA256_DIGEST_LENGTH];
	359
	360	time_t t = time(NULL);
	361	struct tm *gtm = gmtime(&t);
	362
	363	strftime (date_short, sizeof(date_short), "%Y%m%d", gtm);
	364	strftime (date_iso, sizeof(date_iso), "%Y%m%dT%H%M%SZ", gtm);
	365	uri_encoded = _aws_uriencode(uri);
	366
	367	if (op == DDIR_WRITE) {
	368	dsha = _gen_hex_sha256(buf, len);
	369	sprintf(method, "PUT");
	370	} else {
	371	/* DDIR_READ && DDIR_TRIM supply an empty body */
	372	if (op == DDIR_READ)
	373	sprintf(method, "GET");
	374	else
	375	sprintf(method, "DELETE");
	376	dsha = _gen_hex_sha256("", 0);
	377	}
	378
	379	/* Create the canonical request first */
	380	snprintf(creq, sizeof(creq),
	381	"%s\n"
	382	"%s\n"
	383	"\n"
	384	"host:%s\n"
	385	"x-amz-content-sha256:%s\n"
	386	"x-amz-date:%s\n"
f2d79184	387	"x-amz-storage-class:%s\n"
c2f6a13d	388	"\n"
f2d79184	389	"host;x-amz-content-sha256;x-amz-date;x-amz-storage-class\n"
c2f6a13d LMB	390	"%s"
c2f6a13d LMB	391	, method
f2d79184	392	, uri_encoded, o->host, dsha, date_iso, o->s3_storage_class, dsha);
c2f6a13d LMB	393
	394	csha = _gen_hex_sha256(creq, strlen(creq));
	395	snprintf(sts, sizeof(sts), "AWS4-HMAC-SHA256\n%s\n%s/%s/%s/%s\n%s",
	396	date_iso, date_short, o->s3_region, service, aws, csha);
	397
	398	snprintf((char *)dkey, sizeof(dkey), "AWS4%s", o->s3_key);
	399	_hmac(md, dkey, strlen(dkey), date_short);
	400	_hmac(md, md, SHA256_DIGEST_LENGTH, o->s3_region);
	401	_hmac(md, md, SHA256_DIGEST_LENGTH, (char*) service);
	402	_hmac(md, md, SHA256_DIGEST_LENGTH, (char*) aws);
	403	_hmac(md, md, SHA256_DIGEST_LENGTH, sts);
	404
	405	signature = _conv_hex(md, SHA256_DIGEST_LENGTH);
	406
fc002f14	407	/* Suppress automatic Accept: header */
c2f6a13d LMB	408	slist = curl_slist_append(slist, "Accept:");
	409
	410	snprintf(s, sizeof(s), "x-amz-content-sha256: %s", dsha);
	411	slist = curl_slist_append(slist, s);
	412
	413	snprintf(s, sizeof(s), "x-amz-date: %s", date_iso);
	414	slist = curl_slist_append(slist, s);
f2d79184 FH	415	snprintf(s, sizeof(s), "x-amz-storage-class: %s", o->s3_storage_class);
f2d79184 FH	416	slist = curl_slist_append(slist, s);
c2f6a13d	417	snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
f2d79184	418	"SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=%s",
c2f6a13d LMB	419	o->s3_keyid, date_short, o->s3_region, signature);
	420	slist = curl_slist_append(slist, s);
	421
	422	curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
	423
	424	free(uri_encoded);
	425	free(csha);
	426	free(dsha);
	427	free(signature);
	428	}
	429
09fd2966 LMB	430	static void _add_swift_header(CURL curl, struct curl_slist slist, struct http_options *o,
	431	int op, const char uri, char buf, size_t len)
	432	{
	433	char *dsha = NULL;
	434	char s[512];
	435
	436	if (op == DDIR_WRITE) {
	437	dsha = _gen_hex_md5(buf, len);
	438	}
fc002f14	439	/* Suppress automatic Accept: header */
09fd2966 LMB	440	slist = curl_slist_append(slist, "Accept:");
	441
	442	snprintf(s, sizeof(s), "etag: %s", dsha);
	443	slist = curl_slist_append(slist, s);
	444
	445	snprintf(s, sizeof(s), "x-auth-token: %s", o->swift_auth_token);
	446	slist = curl_slist_append(slist, s);
	447
	448	curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
	449
	450	free(dsha);
	451	}
	452
c2f6a13d LMB	453	static void fio_http_cleanup(struct thread_data *td)
	454	{
	455	struct http_data *http = td->io_ops_data;
	456
	457	if (http) {
	458	curl_easy_cleanup(http->curl);
	459	free(http);
	460	}
	461	}
	462
	463	static size_t _http_read(void ptr, size_t size, size_t nmemb, void stream)
	464	{
	465	struct http_curl_stream *state = stream;
	466	size_t len = size * nmemb;
	467	/* We're retrieving; nothing is supposed to be read locally */
	468	if (!stream)
	469	return 0;
	470	if (len+state->pos > state->max)
	471	len = state->max - state->pos;
	472	memcpy(ptr, &state->buf[state->pos], len);
	473	state->pos += len;
	474	return len;
	475	}
	476
	477	static size_t _http_write(void ptr, size_t size, size_t nmemb, void stream)
	478	{
	479	struct http_curl_stream *state = stream;
	480	/* We're just discarding the returned body after a PUT */
	481	if (!stream)
	482	return nmemb;
	483	if (size != 1)
	484	return CURLE_WRITE_ERROR;
	485	if (nmemb + state->pos > state->max)
	486	return CURLE_WRITE_ERROR;
	487	memcpy(&state->buf[state->pos], ptr, nmemb);
	488	state->pos += nmemb;
	489	return nmemb;
	490	}
	491
	492	static int _http_seek(void *stream, curl_off_t offset, int origin)
	493	{
	494	struct http_curl_stream *state = stream;
	495	if (offset < state->max && origin == SEEK_SET) {
	496	state->pos = offset;
	497	return CURL_SEEKFUNC_OK;
	498	} else
	499	return CURL_SEEKFUNC_FAIL;
	500	}
	501
	502	static enum fio_q_status fio_http_queue(struct thread_data *td,
	503	struct io_u *io_u)
	504	{
	505	struct http_data *http = td->io_ops_data;
	506	struct http_options *o = td->eo;
	507	struct http_curl_stream _curl_stream;
	508	struct curl_slist *slist = NULL;
	509	char object[512];
	510	char url[1024];
	511	long status;
	512	CURLcode res;
	513	int r = -1;
	514
	515	fio_ro_check(td, io_u);
	516	memset(&_curl_stream, 0, sizeof(_curl_stream));
09fd2966 LMB	517	snprintf(object, sizeof(object), "%s_%llu_%llu", td->files[0]->file_name,
	518	io_u->offset, io_u->xfer_buflen);
	519	if (o->https == FIO_HTTPS_OFF)
	520	snprintf(url, sizeof(url), "http://%s%s", o->host, object);
	521	else
	522	snprintf(url, sizeof(url), "https://%s%s", o->host, object);
c2f6a13d LMB	523	curl_easy_setopt(http->curl, CURLOPT_URL, url);
	524	_curl_stream.buf = io_u->xfer_buf;
	525	_curl_stream.max = io_u->xfer_buflen;
	526	curl_easy_setopt(http->curl, CURLOPT_SEEKDATA, &_curl_stream);
	527	curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)io_u->xfer_buflen);
	528
09fd2966	529	if (o->mode == FIO_HTTP_S3)
c2f6a13d LMB	530	_add_aws_auth_header(http->curl, slist, o, io_u->ddir, object,
c2f6a13d LMB	531	io_u->xfer_buf, io_u->xfer_buflen);
09fd2966 LMB	532	else if (o->mode == FIO_HTTP_SWIFT)
	533	_add_swift_header(http->curl, slist, o, io_u->ddir, object,
	534	io_u->xfer_buf, io_u->xfer_buflen);
c2f6a13d LMB	535
	536	if (io_u->ddir == DDIR_WRITE) {
	537	curl_easy_setopt(http->curl, CURLOPT_READDATA, &_curl_stream);
	538	curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
	539	curl_easy_setopt(http->curl, CURLOPT_UPLOAD, 1L);
	540	res = curl_easy_perform(http->curl);
	541	if (res == CURLE_OK) {
	542	curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
	543	if (status == 100 \|\| (status >= 200 && status <= 204))
	544	goto out;
	545	log_err("DDIR_WRITE failed with HTTP status code %ld\n", status);
c2f6a13d	546	}
4a4c21bf	547	goto err;
c2f6a13d LMB	548	} else if (io_u->ddir == DDIR_READ) {
	549	curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
	550	curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, &_curl_stream);
	551	curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
	552	res = curl_easy_perform(http->curl);
	553	if (res == CURLE_OK) {
	554	curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
	555	if (status == 200)
	556	goto out;
	557	else if (status == 404) {
	558	/* Object doesn't exist. Pretend we read
	559	* zeroes */
	560	memset(io_u->xfer_buf, 0, io_u->xfer_buflen);
	561	goto out;
	562	}
	563	log_err("DDIR_READ failed with HTTP status code %ld\n", status);
	564	}
	565	goto err;
	566	} else if (io_u->ddir == DDIR_TRIM) {
	567	curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
	568	curl_easy_setopt(http->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
bd915a6b	569	curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)0);
c2f6a13d LMB	570	curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
	571	curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
	572	res = curl_easy_perform(http->curl);
	573	if (res == CURLE_OK) {
	574	curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
	575	if (status == 200 \|\| status == 202 \|\| status == 204 \|\| status == 404)
	576	goto out;
	577	log_err("DDIR_TRIM failed with HTTP status code %ld\n", status);
	578	}
	579	goto err;
	580	}
	581
	582	log_err("WARNING: Only DDIR_READ/DDIR_WRITE/DDIR_TRIM are supported!\n");
	583
	584	err:
	585	io_u->error = r;
	586	td_verror(td, io_u->error, "transfer");
	587	out:
	588	curl_slist_free_all(slist);
	589	return FIO_Q_COMPLETED;
	590	}
	591
	592	static struct io_u fio_http_event(struct thread_data td, int event)
	593	{
	594	/* sync IO engine - never any outstanding events */
	595	return NULL;
	596	}
	597
	598	int fio_http_getevents(struct thread_data *td, unsigned int min,
	599	unsigned int max, const struct timespec *t)
	600	{
	601	/* sync IO engine - never any outstanding events */
	602	return 0;
	603	}
	604
	605	static int fio_http_setup(struct thread_data *td)
	606	{
	607	struct http_data *http = NULL;
	608	struct http_options *o = td->eo;
c65f3435	609
c2f6a13d LMB	610	/* allocate engine specific structure to deal with libhttp. */
	611	http = calloc(1, sizeof(*http));
	612	if (!http) {
	613	log_err("calloc failed.\n");
	614	goto cleanup;
	615	}
	616
	617	http->curl = curl_easy_init();
	618	if (o->verbose)
	619	curl_easy_setopt(http->curl, CURLOPT_VERBOSE, 1L);
	620	if (o->verbose > 1)
	621	curl_easy_setopt(http->curl, CURLOPT_DEBUGFUNCTION, &_curl_trace);
	622	curl_easy_setopt(http->curl, CURLOPT_NOPROGRESS, 1L);
	623	curl_easy_setopt(http->curl, CURLOPT_FOLLOWLOCATION, 1L);
	624	curl_easy_setopt(http->curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP\|CURLPROTO_HTTPS);
09fd2966 LMB	625	if (o->https == FIO_HTTPS_INSECURE) {
	626	curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYPEER, 0L);
	627	curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYHOST, 0L);
	628	}
c2f6a13d LMB	629	curl_easy_setopt(http->curl, CURLOPT_READFUNCTION, _http_read);
c2f6a13d LMB	630	curl_easy_setopt(http->curl, CURLOPT_WRITEFUNCTION, _http_write);
bd915a6b	631	curl_easy_setopt(http->curl, CURLOPT_SEEKFUNCTION, &_http_seek);
c2f6a13d LMB	632	if (o->user && o->pass) {
	633	curl_easy_setopt(http->curl, CURLOPT_USERNAME, o->user);
	634	curl_easy_setopt(http->curl, CURLOPT_PASSWORD, o->pass);
	635	curl_easy_setopt(http->curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
	636	}
	637
	638	td->io_ops_data = http;
	639
	640	/* Force single process mode. */
	641	td->o.use_thread = 1;
	642
	643	return 0;
	644	cleanup:
	645	fio_http_cleanup(td);
c65f3435	646	return 1;
c2f6a13d LMB	647	}
	648
	649	static int fio_http_open(struct thread_data td, struct fio_file f)
	650	{
	651	return 0;
	652	}
	653	static int fio_http_invalidate(struct thread_data td, struct fio_file f)
	654	{
	655	return 0;
	656	}
	657
5a8a6a03	658	FIO_STATIC struct ioengine_ops ioengine = {
c2f6a13d LMB	659	.name = "http",
c2f6a13d LMB	660	.version = FIO_IOOPS_VERSION,
f32a30d4	661	.flags = FIO_DISKLESSIO \| FIO_SYNCIO,
c2f6a13d LMB	662	.setup = fio_http_setup,
	663	.queue = fio_http_queue,
	664	.getevents = fio_http_getevents,
	665	.event = fio_http_event,
	666	.cleanup = fio_http_cleanup,
	667	.open_file = fio_http_open,
	668	.invalidate = fio_http_invalidate,
	669	.options = options,
	670	.option_struct_size = sizeof(struct http_options),
	671	};
	672
	673	static void fio_init fio_http_register(void)
	674	{
	675	register_ioengine(&ioengine);
	676	}
	677
	678	static void fio_exit fio_http_unregister(void)
	679	{
	680	unregister_ioengine(&ioengine);
	681	}