Merge branch 's3_crypto' of github.com:hualongfeng/fio

diff --git a/HOWTO.rst b/HOWTO.rst
index 104cce2d211510e735497fdfeaa0fb23e8d85dc7..05fc117f2ffc02e308894b96dc4b03ad468158f4 100644 (file)
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -2692,6 +2692,20 @@ with the caveat that when used on the command line, they must come after the
 
        The S3 key/access id.
 
+.. option:: http_s3_sse_customer_key=str : [http]
+
+        The encryption customer key in SSE server side.
+
+.. option:: http_s3_sse_customer_algorithm=str : [http]
+
+        The encryption customer algorithm in SSE server side.
+        Default is **AES256**
+
+.. option:: http_s3_storage_class=str : [http]
+
+        Which storage class to access. User-customizable settings.
+        Default is **STANDARD**
+
 .. option:: http_swift_auth_token=str : [http]
 
        The Swift auth token. See the example configuration file on how

diff --git a/engines/http.c b/engines/http.c
index 1de9e66c75f793e641e1605c678f19c17b00c8db..56dc7d1b14f63d232d49693801109eb9dab5684c 100644 (file)
--- a/engines/http.c
+++ b/engines/http.c
@@ -57,6 +57,9 @@ struct http_options {
        char *s3_key;
        char *s3_keyid;
        char *s3_region;
+       char *s3_sse_customer_key;
+       char *s3_sse_customer_algorithm;
+       char *s3_storage_class;
        char *swift_auth_token;
        int verbose;
        unsigned int mode;
@@ -161,6 +164,36 @@ static struct fio_option options[] = {
                .category = FIO_OPT_C_ENGINE,
                .group    = FIO_OPT_G_HTTP,
        },
+       {
+               .name     = "http_s3_sse_customer_key",
+               .lname    = "SSE Customer Key",
+               .type     = FIO_OPT_STR_STORE,
+               .help     = "S3 SSE Customer Key",
+               .off1     = offsetof(struct http_options, s3_sse_customer_key),
+               .def      = "",
+               .category = FIO_OPT_C_ENGINE,
+               .group    = FIO_OPT_G_HTTP,
+       },
+       {
+               .name     = "http_s3_sse_customer_algorithm",
+               .lname    = "SSE Customer Algorithm",
+               .type     = FIO_OPT_STR_STORE,
+               .help     = "S3 SSE Customer Algorithm",
+               .off1     = offsetof(struct http_options, s3_sse_customer_algorithm),
+               .def      = "AES256",
+               .category = FIO_OPT_C_ENGINE,
+               .group    = FIO_OPT_G_HTTP,
+       },
+       {
+               .name     = "http_s3_storage_class",
+               .lname    = "S3 Storage class",
+               .type     = FIO_OPT_STR_STORE,
+               .help     = "S3 Storage Class",
+               .off1     = offsetof(struct http_options, s3_storage_class),
+               .def      = "STANDARD",
+               .category = FIO_OPT_C_ENGINE,
+               .group    = FIO_OPT_G_HTTP,
+       },
        {
                .name     = "http_mode",
                .lname    = "Request mode to use",
@@ -266,6 +299,54 @@ static char *_gen_hex_md5(const char *p, size_t len)
        return _conv_hex(hash, MD5_DIGEST_LENGTH);
 }
 
+static char *_conv_base64_encode(const unsigned char *p, size_t len)
+{
+       char *r, *ret;
+       int i;
+       static const char sEncodingTable[] = {
+               'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
+               'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
+               'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
+               'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
+               'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
+               'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
+               'w', 'x', 'y', 'z', '0', '1', '2', '3',
+               '4', '5', '6', '7', '8', '9', '+', '/'
+       };
+
+       size_t out_len = 4 * ((len + 2) / 3);
+       ret = r = malloc(out_len + 1);
+
+       for (i = 0; i < len - 2; i += 3) {
+               *r++ = sEncodingTable[(p[i] >> 2) & 0x3F];
+               *r++ = sEncodingTable[((p[i] & 0x3) << 4) | ((int) (p[i + 1] & 0xF0) >> 4)];
+               *r++ = sEncodingTable[((p[i + 1] & 0xF) << 2) | ((int) (p[i + 2] & 0xC0) >> 6)];
+               *r++ = sEncodingTable[p[i + 2] & 0x3F];
+       }
+
+       if (i < len) {
+               *r++ = sEncodingTable[(p[i] >> 2) & 0x3F];
+               if (i == (len - 1)) {
+                       *r++ = sEncodingTable[((p[i] & 0x3) << 4)];
+                       *r++ = '=';
+               } else {
+                       *r++ = sEncodingTable[((p[i] & 0x3) << 4) | ((int) (p[i + 1] & 0xF0) >> 4)];
+                       *r++ = sEncodingTable[((p[i + 1] & 0xF) << 2)];
+               }
+               *r++ = '=';
+       }
+
+       ret[out_len]=0;
+       return ret;
+}
+
+static char *_gen_base64_md5(const unsigned char *p, size_t len)
+{
+       unsigned char hash[MD5_DIGEST_LENGTH];
+       MD5((unsigned char*)p, len, hash);
+       return _conv_base64_encode(hash, MD5_DIGEST_LENGTH);
+}
+
 static void _hmac(unsigned char *md, void *key, int key_len, char *data) {
 #ifndef CONFIG_HAVE_OPAQUE_HMAC_CTX
        HMAC_CTX _ctx;
@@ -335,8 +416,8 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
        char date_iso[32];
        char method[8];
        char dkey[128];
-       char creq[512];
-       char sts[256];
+       char creq[4096];
+       char sts[512];
        char s[512];
        char *uri_encoded = NULL;
        char *dsha = NULL;
@@ -345,6 +426,9 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
        const char *service = "s3";
        const char *aws = "aws4_request";
        unsigned char md[SHA256_DIGEST_LENGTH];
+       unsigned char sse_key[33] = {0};
+       char *sse_key_base64 = NULL;
+       char *sse_key_md5_base64 = NULL;
 
        time_t t = time(NULL);
        struct tm *gtm = gmtime(&t);
@@ -353,6 +437,9 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
        strftime (date_iso, sizeof(date_iso), "%Y%m%dT%H%M%SZ", gtm);
        uri_encoded = _aws_uriencode(uri);
 
+       if (o->s3_sse_customer_key != NULL)
+               strncpy((char*)sse_key, o->s3_sse_customer_key, sizeof(sse_key) - 1);
+
        if (op == DDIR_WRITE) {
                dsha = _gen_hex_sha256(buf, len);
                sprintf(method, "PUT");
@@ -366,22 +453,50 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
        }
 
        /* Create the canonical request first */
-       snprintf(creq, sizeof(creq),
-       "%s\n"
-       "%s\n"
-       "\n"
-       "host:%s\n"
-       "x-amz-content-sha256:%s\n"
-       "x-amz-date:%s\n"
-       "\n"
-       "host;x-amz-content-sha256;x-amz-date\n"
-       "%s"
-       , method
-       , uri_encoded, o->host, dsha, date_iso, dsha);
+       if (sse_key[0] != '\0') {
+               sse_key_base64 = _conv_base64_encode(sse_key, sizeof(sse_key) - 1);
+               sse_key_md5_base64 = _gen_base64_md5(sse_key, sizeof(sse_key) - 1);
+               snprintf(creq, sizeof(creq),
+                       "%s\n"
+                       "%s\n"
+                       "\n"
+                       "host:%s\n"
+                       "x-amz-content-sha256:%s\n"
+                       "x-amz-date:%s\n"
+                       "x-amz-server-side-encryption-customer-algorithm:%s\n"
+                       "x-amz-server-side-encryption-customer-key:%s\n"
+                       "x-amz-server-side-encryption-customer-key-md5:%s\n"
+                       "x-amz-storage-class:%s\n"
+                       "\n"
+                       "host;x-amz-content-sha256;x-amz-date;"
+                       "x-amz-server-side-encryption-customer-algorithm;"
+                       "x-amz-server-side-encryption-customer-key;"
+                       "x-amz-server-side-encryption-customer-key-md5;"
+                       "x-amz-storage-class\n"
+                       "%s"
+                       , method
+                       , uri_encoded, o->host, dsha, date_iso
+                       , o->s3_sse_customer_algorithm, sse_key_base64
+                       , sse_key_md5_base64, o->s3_storage_class, dsha);
+       } else {
+               snprintf(creq, sizeof(creq),
+                       "%s\n"
+                       "%s\n"
+                       "\n"
+                       "host:%s\n"
+                       "x-amz-content-sha256:%s\n"
+                       "x-amz-date:%s\n"
+                       "x-amz-storage-class:%s\n"
+                       "\n"
+                       "host;x-amz-content-sha256;x-amz-date;x-amz-storage-class\n"
+                       "%s"
+                       , method
+                       , uri_encoded, o->host, dsha, date_iso, o->s3_storage_class, dsha);
+       }
 
        csha = _gen_hex_sha256(creq, strlen(creq));
        snprintf(sts, sizeof(sts), "AWS4-HMAC-SHA256\n%s\n%s/%s/%s/%s\n%s",
-               date_iso, date_short, o->s3_region, service, aws, csha);
+                       date_iso, date_short, o->s3_region, service, aws, csha);
 
        snprintf((char *)dkey, sizeof(dkey), "AWS4%s", o->s3_key);
        _hmac(md, dkey, strlen(dkey), date_short);
@@ -401,9 +516,32 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
        snprintf(s, sizeof(s), "x-amz-date: %s", date_iso);
        slist = curl_slist_append(slist, s);
 
-       snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
-       "SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=%s",
-       o->s3_keyid, date_short, o->s3_region, signature);
+       if (sse_key[0] != '\0') {
+               snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-algorithm: %s", o->s3_sse_customer_algorithm);
+               slist = curl_slist_append(slist, s);
+               snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-key: %s", sse_key_base64);
+               slist = curl_slist_append(slist, s);
+               snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-key-md5: %s", sse_key_md5_base64);
+               slist = curl_slist_append(slist, s);
+       }
+
+       snprintf(s, sizeof(s), "x-amz-storage-class: %s", o->s3_storage_class);
+       slist = curl_slist_append(slist, s);
+
+       if (sse_key[0] != '\0') {
+               snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
+                       "SignedHeaders=host;x-amz-content-sha256;"
+                       "x-amz-date;x-amz-server-side-encryption-customer-algorithm;"
+                       "x-amz-server-side-encryption-customer-key;"
+                       "x-amz-server-side-encryption-customer-key-md5;"
+                       "x-amz-storage-class,"
+                       "Signature=%s",
+               o->s3_keyid, date_short, o->s3_region, signature);
+       } else {
+               snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
+                       "SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=%s",
+                       o->s3_keyid, date_short, o->s3_region, signature);
+       }
        slist = curl_slist_append(slist, s);
 
        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
@@ -412,6 +550,10 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
        free(csha);
        free(dsha);
        free(signature);
+       if (sse_key_base64 != NULL) {
+               free(sse_key_base64);
+               free(sse_key_md5_base64);
+       }
 }
 
 static void _add_swift_header(CURL *curl, struct curl_slist *slist, struct http_options *o,

diff --git a/examples/http-s3-crypto.fio b/examples/http-s3-crypto.fio
new file mode 100644 (file)
index 0000000..2403746
--- /dev/null
+++ b/examples/http-s3-crypto.fio
@@ -0,0 +1,38 @@
+# Example test for the HTTP engine's S3 support against Amazon AWS.
+# Obviously, you have to adjust the S3 credentials; for this example,
+# they're passed in via the environment.
+# And you can set the SSE Customer Key and Algorithm to test Server
+# Side Encryption.
+#
+
+[global]
+ioengine=http
+name=test
+direct=1
+filename=/larsmb-fio-test/object
+http_verbose=0
+https=on
+http_mode=s3
+http_s3_key=${S3_KEY}
+http_s3_keyid=${S3_ID}
+http_host=s3.eu-central-1.amazonaws.com
+http_s3_region=eu-central-1
+http_s3_sse_customer_key=${SSE_KEY}
+http_s3_sse_customer_algorithm=AES256
+group_reporting
+
+# With verify, this both writes and reads the object
+[create]
+rw=write
+bs=4k
+size=64k
+io_size=4k
+verify=sha256
+
+[trim]
+stonewall
+rw=trim
+bs=4k
+size=64k
+io_size=4k
+

diff --git a/examples/http-s3-storage-class.fio b/examples/http-s3-storage-class.fio
new file mode 100644 (file)
index 0000000..9ee2383
--- /dev/null
+++ b/examples/http-s3-storage-class.fio
@@ -0,0 +1,37 @@
+# Example test for the HTTP engine's S3 support against Amazon AWS.
+# Obviously, you have to adjust the S3 credentials; for this example,
+# they're passed in via the environment.
+# And here add storage class parameter, you can set normal test for
+# STANDARD and compression test for another storage class.
+#
+
+[global]
+ioengine=http
+name=test
+direct=1
+filename=/larsmb-fio-test/object
+http_verbose=0
+https=on
+http_mode=s3
+http_s3_key=${S3_KEY}
+http_s3_keyid=${S3_ID}
+http_host=s3.eu-central-1.amazonaws.com
+http_s3_region=eu-central-1
+http_s3_storage_class=${STORAGE_CLASS}
+group_reporting
+
+# With verify, this both writes and reads the object
+[create]
+rw=write
+bs=4k
+size=64k
+io_size=4k
+verify=sha256
+
+[trim]
+stonewall
+rw=trim
+bs=4k
+size=64k
+io_size=4k
+

diff --git a/fio.1 b/fio.1
index ce9bf3ef4d104ac3c2be4a0a3056a29051975209..6630525f4c2dd24b0ea8883aeb9fb1295ff24dd0 100644 (file)
--- a/fio.1
+++ b/fio.1
@@ -2308,6 +2308,15 @@ The S3 secret key.
 .BI (http)http_s3_keyid \fR=\fPstr
 The S3 key/access id.
 .TP
+.BI (http)http_s3_sse_customer_key \fR=\fPstr
+The encryption customer key in SSE server side.
+.TP
+.BI (http)http_s3_sse_customer_algorithm \fR=\fPstr
+The encryption customer algorithm in SSE server side. Default is \fBAES256\fR
+.TP
+.BI (http)http_s3_storage_class \fR=\fPstr
+Which storage class to access. User-customizable settings. Default is \fBSTANDARD\fR
+.TP
 .BI (http)http_swift_auth_token \fR=\fPstr
 The Swift auth token. See the example configuration file on how to
 retrieve this.