engines/http: Add S3 security token support

Security tokens are an element of S3 authorization in some environments. This
change adds a parameter to allow users to specify a security token, and pass
this to S3 requests with the appropriate header.

Signed-off-by: Renar Narubin <renar.narubin@snowflake.com>

diff --git a/HOWTO.rst b/HOWTO.rst
index f082158a461a50093fc98c7e2c95909a02633d40..e5ddc89da5c49b7ad58905c3b732c6cf18c70bb9 100644 (file)
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -3013,6 +3013,10 @@ with the caveat that when used on the command line, they must come after the
 
        The S3 key/access id.
 
+.. option:: http_s3_security_token=str : [http]
+
+       The S3 security token.
+
 .. option:: http_s3_sse_customer_key=str : [http]
 
         The encryption customer key in SSE server side.

diff --git a/engines/http.c b/engines/http.c
index 99f4e119a4f5f0fcd5d59134ae07e0923c9559af..fb13aee8aeadc4e6223931f30312dd67d7ac09a0 100644 (file)
--- a/engines/http.c
+++ b/engines/http.c
@@ -56,6 +56,7 @@ struct http_options {
        char *pass;
        char *s3_key;
        char *s3_keyid;
+       char *s3_security_token;
        char *s3_region;
        char *s3_sse_customer_key;
        char *s3_sse_customer_algorithm;
@@ -144,6 +145,16 @@ static struct fio_option options[] = {
                .category = FIO_OPT_C_ENGINE,
                .group    = FIO_OPT_G_HTTP,
        },
+       {
+               .name     = "http_s3_security_token",
+               .lname    = "S3 security token",
+               .type     = FIO_OPT_STR_STORE,
+               .help     = "S3 security token",
+               .off1     = offsetof(struct http_options, s3_security_token),
+               .def      = "",
+               .category = FIO_OPT_C_ENGINE,
+               .group    = FIO_OPT_G_HTTP,
+       },
        {
                .name     = "http_swift_auth_token",
                .lname    = "Swift auth token",
@@ -419,7 +430,7 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
        char dkey[128];
        char creq[4096];
        char sts[512];
-       char s[512];
+       char s[2048];
        char *uri_encoded = NULL;
        char *dsha = NULL;
        char *csha = NULL;
@@ -430,6 +441,8 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
        unsigned char sse_key[33] = {0};
        char *sse_key_base64 = NULL;
        char *sse_key_md5_base64 = NULL;
+       char security_token_header[2048] = {0};
+       char security_token_list_item[24] = {0};
 
        time_t t = time(NULL);
        struct tm *gtm = gmtime(&t);
@@ -438,6 +451,12 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
        strftime (date_iso, sizeof(date_iso), "%Y%m%dT%H%M%SZ", gtm);
        uri_encoded = _aws_uriencode(uri);
 
+       if (o->s3_security_token != NULL) {
+               snprintf(security_token_header, sizeof(security_token_header),
+                               "x-amz-security-token:%s\n", o->s3_security_token);
+               sprintf(security_token_list_item, "x-amz-security-token;");
+       }
+
        if (o->s3_sse_customer_key != NULL)
                strncpy((char*)sse_key, o->s3_sse_customer_key, sizeof(sse_key) - 1);
 
@@ -467,18 +486,21 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
                        "x-amz-server-side-encryption-customer-algorithm:%s\n"
                        "x-amz-server-side-encryption-customer-key:%s\n"
                        "x-amz-server-side-encryption-customer-key-md5:%s\n"
+                       "%s" /* security token if provided */
                        "x-amz-storage-class:%s\n"
                        "\n"
                        "host;x-amz-content-sha256;x-amz-date;"
                        "x-amz-server-side-encryption-customer-algorithm;"
                        "x-amz-server-side-encryption-customer-key;"
                        "x-amz-server-side-encryption-customer-key-md5;"
+                       "%s"
                        "x-amz-storage-class\n"
                        "%s"
                        , method
                        , uri_encoded, o->host, dsha, date_iso
                        , o->s3_sse_customer_algorithm, sse_key_base64
-                       , sse_key_md5_base64, o->s3_storage_class, dsha);
+                       , sse_key_md5_base64, security_token_header
+                       , o->s3_storage_class, security_token_list_item, dsha);
        } else {
                snprintf(creq, sizeof(creq),
                        "%s\n"
@@ -487,12 +509,15 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
                        "host:%s\n"
                        "x-amz-content-sha256:%s\n"
                        "x-amz-date:%s\n"
+                       "%s" /* security token if provided */
                        "x-amz-storage-class:%s\n"
                        "\n"
-                       "host;x-amz-content-sha256;x-amz-date;x-amz-storage-class\n"
+                       "host;x-amz-content-sha256;x-amz-date;%sx-amz-storage-class\n"
                        "%s"
                        , method
-                       , uri_encoded, o->host, dsha, date_iso, o->s3_storage_class, dsha);
+                       , uri_encoded, o->host, dsha, date_iso
+                       , security_token_header, o->s3_storage_class
+                       , security_token_list_item, dsha);
        }
 
        csha = _gen_hex_sha256(creq, strlen(creq));
@@ -526,6 +551,11 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
                slist = curl_slist_append(slist, s);
        }
 
+       if (o->s3_security_token != NULL) {
+               snprintf(s, sizeof(s), "x-amz-security-token: %s", o->s3_security_token);
+               slist = curl_slist_append(slist, s);
+       }
+
        snprintf(s, sizeof(s), "x-amz-storage-class: %s", o->s3_storage_class);
        slist = curl_slist_append(slist, s);
 
@@ -535,13 +565,14 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
                        "x-amz-date;x-amz-server-side-encryption-customer-algorithm;"
                        "x-amz-server-side-encryption-customer-key;"
                        "x-amz-server-side-encryption-customer-key-md5;"
+                       "%s"
                        "x-amz-storage-class,"
                        "Signature=%s",
-               o->s3_keyid, date_short, o->s3_region, signature);
+               o->s3_keyid, date_short, o->s3_region, security_token_list_item, signature);
        } else {
                snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
-                       "SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=%s",
-                       o->s3_keyid, date_short, o->s3_region, signature);
+                       "SignedHeaders=host;x-amz-content-sha256;x-amz-date;%sx-amz-storage-class,Signature=%s",
+                       o->s3_keyid, date_short, o->s3_region, security_token_list_item, signature);
        }
        slist = curl_slist_append(slist, s);
 

diff --git a/fio.1 b/fio.1
index 0071c364e022784c2014340ff0c97e14d54a0354..cba1273b590043e37c25e5388c0de2abd999142f 100644 (file)
--- a/fio.1
+++ b/fio.1
@@ -2605,6 +2605,9 @@ The S3 secret key.
 .BI (http)http_s3_keyid \fR=\fPstr
 The S3 key/access id.
 .TP
+.BI (http)http_s3_security_token \fR=\fPstr
+The S3 security token.
+.TP
 .BI (http)http_s3_sse_customer_key \fR=\fPstr
 The encryption customer key in SSE server side.
 .TP