From 2daf8bb9db2ca165db89001018ef22e44bb570ab Mon Sep 17 00:00:00 2001 From: Prajwal T R Date: Fri, 24 Mar 2023 21:21:10 -0600 Subject: [PATCH 1/7] Fix : fix parsing of access_token key OAUTH2 --- device-src/s3.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/device-src/s3.c b/device-src/s3.c index b9028bbb9f..dfcb707f9f 100644 --- a/device-src/s3.c +++ b/device-src/s3.c @@ -2918,7 +2918,7 @@ compile_regexes(void) {"^[a-z0-9](-*[a-z0-9]){2,62}$", REG_EXTENDED | REG_NOSUB, &subdomain_regex}, {"(/>)|(>([^<]*))", REG_EXTENDED | REG_ICASE, &location_con_regex}, {"^Date:(.*)$",REG_EXTENDED | REG_ICASE | REG_NEWLINE, &date_sync_regex}, - {"\"access_token\" : \"([^\"]*)\",", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &access_token_regex}, + {"\"access_token\": \"([^\"]*)\",", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &access_token_regex}, {"\"expires_in\" : (.*)", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &expires_in_regex}, {"\"details\": \"([^\"]*)\",", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &details_regex}, {"\"code\": (.*),", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &code_regex}, @@ -2978,7 +2978,7 @@ compile_regexes(void) {"^Date:(.*)$", G_REGEX_OPTIMIZE | G_REGEX_CASELESS, &date_sync_regex}, - {"\"access_token\" : \"([^\"]*)\"", + {"\"access_token\": \"([^\"]*)\"", G_REGEX_OPTIMIZE | G_REGEX_CASELESS, &access_token_regex}, {"\"expires_n\" : (.*)", From 14d6c0dba06bff70f11115302697a1bdf3079d71 Mon Sep 17 00:00:00 2001 From: Prajwal T R Date: Tue, 28 Mar 2023 07:55:59 -0600 Subject: [PATCH 2/7] Fix : set http version to 1.1 for google cloud storage --- device-src/s3.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/device-src/s3.c b/device-src/s3.c index dfcb707f9f..580df30770 100644 --- a/device-src/s3.c +++ b/device-src/s3.c @@ -3372,6 +3372,9 @@ s3_new_curl( } #endif } + /* google cloud api's are not HTTP2 compatiable YET*/ + if (hdl->s3_api == S3_API_OAUTH2) + curl_easy_setopt(hdl->curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); } gboolean From f615d122b26af37699788b9dd1e461f36087a705 Mon Sep 17 00:00:00 2001 From: Prajwal T R Date: Tue, 28 Mar 2023 22:47:06 -0600 Subject: [PATCH 3/7] Fix: `expires_in` regex while fetching `access_token` - was not setting expires_in properly, causing fetch of access_token for every other request TODO: make use of HTTP/1.1 configurable --- device-src/s3.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/device-src/s3.c b/device-src/s3.c index 580df30770..b422169fe0 100644 --- a/device-src/s3.c +++ b/device-src/s3.c @@ -2919,7 +2919,7 @@ compile_regexes(void) {"(/>)|(>([^<]*))", REG_EXTENDED | REG_ICASE, &location_con_regex}, {"^Date:(.*)$",REG_EXTENDED | REG_ICASE | REG_NEWLINE, &date_sync_regex}, {"\"access_token\": \"([^\"]*)\",", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &access_token_regex}, - {"\"expires_in\" : (.*)", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &expires_in_regex}, + {"\"expires_in\": (.*)", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &expires_in_regex}, {"\"details\": \"([^\"]*)\",", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &details_regex}, {"\"code\": (.*),", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &code_regex}, {"\"message\": \"([^\"]*)\",", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &json_message_regex}, @@ -2981,7 +2981,7 @@ compile_regexes(void) {"\"access_token\": \"([^\"]*)\"", G_REGEX_OPTIMIZE | G_REGEX_CASELESS, &access_token_regex}, - {"\"expires_n\" : (.*)", + {"\"expires_in\": (.*)", G_REGEX_OPTIMIZE | G_REGEX_CASELESS, &expires_in_regex}, {"\"details\" : \"([^\"]*)\"", From 44ec52f816df5df2ee17edaffc7ddd23dc102901 Mon Sep 17 00:00:00 2001 From: Prajwal T R Date: Tue, 28 Mar 2023 22:55:31 -0600 Subject: [PATCH 4/7] account for trailing comma --- device-src/s3.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/device-src/s3.c b/device-src/s3.c index b422169fe0..dc5551e122 100644 --- a/device-src/s3.c +++ b/device-src/s3.c @@ -2919,7 +2919,7 @@ compile_regexes(void) {"(/>)|(>([^<]*))", REG_EXTENDED | REG_ICASE, &location_con_regex}, {"^Date:(.*)$",REG_EXTENDED | REG_ICASE | REG_NEWLINE, &date_sync_regex}, {"\"access_token\": \"([^\"]*)\",", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &access_token_regex}, - {"\"expires_in\": (.*)", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &expires_in_regex}, + {"\"expires_in\": (.*),", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &expires_in_regex}, {"\"details\": \"([^\"]*)\",", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &details_regex}, {"\"code\": (.*),", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &code_regex}, {"\"message\": \"([^\"]*)\",", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &json_message_regex}, @@ -2981,7 +2981,7 @@ compile_regexes(void) {"\"access_token\": \"([^\"]*)\"", G_REGEX_OPTIMIZE | G_REGEX_CASELESS, &access_token_regex}, - {"\"expires_in\": (.*)", + {"\"expires_in\": (.*),", G_REGEX_OPTIMIZE | G_REGEX_CASELESS, &expires_in_regex}, {"\"details\" : \"([^\"]*)\"", From ca0e02ab61ad0900e46b60106dfd634ffdaba65a Mon Sep 17 00:00:00 2001 From: Prajwal T R Date: Sun, 21 May 2023 23:34:59 -0600 Subject: [PATCH 5/7] fix : account for zero or more spaces --- device-src/s3.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/device-src/s3.c b/device-src/s3.c index dc5551e122..a575b7d47c 100644 --- a/device-src/s3.c +++ b/device-src/s3.c @@ -2918,8 +2918,8 @@ compile_regexes(void) {"^[a-z0-9](-*[a-z0-9]){2,62}$", REG_EXTENDED | REG_NOSUB, &subdomain_regex}, {"(/>)|(>([^<]*))", REG_EXTENDED | REG_ICASE, &location_con_regex}, {"^Date:(.*)$",REG_EXTENDED | REG_ICASE | REG_NEWLINE, &date_sync_regex}, - {"\"access_token\": \"([^\"]*)\",", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &access_token_regex}, - {"\"expires_in\": (.*),", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &expires_in_regex}, + {"\"access_token\"[[:space:]]*:[[:space:]]*\"([^\"]*)\",", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &access_token_regex}, + {"\"expires_in\"[[:space:]]*:[[:space:]]*(.*),", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &expires_in_regex}, {"\"details\": \"([^\"]*)\",", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &details_regex}, {"\"code\": (.*),", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &code_regex}, {"\"message\": \"([^\"]*)\",", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &json_message_regex}, @@ -2978,10 +2978,10 @@ compile_regexes(void) {"^Date:(.*)$", G_REGEX_OPTIMIZE | G_REGEX_CASELESS, &date_sync_regex}, - {"\"access_token\": \"([^\"]*)\"", + {"\"access_token\"\\s*:\\s*\"([^\"]*)\"", G_REGEX_OPTIMIZE | G_REGEX_CASELESS, &access_token_regex}, - {"\"expires_in\": (.*),", + {"\"expires_in\"\\s*:\\s*(.*),", G_REGEX_OPTIMIZE | G_REGEX_CASELESS, &expires_in_regex}, {"\"details\" : \"([^\"]*)\"", From e53bbc8fa0aa503ae31a2aa526a754fe2342cd46 Mon Sep 17 00:00:00 2001 From: Prajwal T R Date: Mon, 22 May 2023 16:03:20 -0600 Subject: [PATCH 6/7] feat : new option to set HTTP version to 1.1 - new property to be set `http_v1_1` --- device-src/s3-device.c | 35 +++++++++++++++++++++++++++++++++-- device-src/s3-device.h | 2 ++ device-src/s3.c | 12 +++++++++--- 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/device-src/s3-device.c b/device-src/s3-device.c index 9d1b718d8c..260d43cc00 100644 --- a/device-src/s3-device.c +++ b/device-src/s3-device.c @@ -174,10 +174,14 @@ static DevicePropertyBase device_property_refresh_token; static DevicePropertyBase device_property_project_id; #define PROPERTY_PROJECT_ID (device_property_project_id.ID) -/* The PROJECT ID */ +/* Create Bucket */ static DevicePropertyBase device_property_create_bucket; #define PROPERTY_CREATE_BUCKET (device_property_create_bucket.ID) +/* Set HTTP version for CURL requests */ +static DevicePropertyBase device_property_http_version; +#define PROPERTY_HTTP_V1_1_VERSION (device_property_http_v1_1_version.ID) + /* glacier */ static DevicePropertyBase device_property_read_from_glacier; #define PROPERTY_READ_FROM_GLACIER (device_property_read_from_glacier.ID) @@ -520,6 +524,10 @@ static gboolean s3_device_set_reps_bucket_fn(Device *self, DevicePropertyBase *base, GValue *val, PropertySurety surety, PropertySource source); +static gboolean s3_device_set_http_v1_1_version(Device *self, + DevicePropertyBase *base, GValue *val, + PropertySurety surety, PropertySource source); + static void s3_thread_read_block(gpointer thread_data, gpointer data); static void s3_thread_write_block(gpointer thread_data, @@ -1186,6 +1194,10 @@ s3_device_register(void) G_TYPE_UINT64, "timeout", "The timeout for one tranfer"); + device_property_fill_and_register(&device_property_http_version, + G_TYPE_BOOLEAN, "http_v1_1", + "Use HTTP version 1.1"); + /* register the device itself */ register_device(s3_device_factory, device_prefix_list); } @@ -1242,6 +1254,7 @@ s3_device_init(S3Device * self) self->reps = NULL; self->reps_bucket = NULL; self->transition_to_glacier = -1; + self->http_version_v1_1 = FALSE; /* Register property values * Note: Some aren't added until s3_device_open_device() @@ -1580,6 +1593,11 @@ s3_device_base_init( PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START, device_simple_property_get_fn, s3_device_set_reps_bucket_fn); + + device_class_register_property(device_class, PROPERTY_HTTP_V1_1_VERSION, + PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START, + device_simple_property_get_fn, + s3_device_set_http_version); } static gboolean @@ -2315,6 +2333,18 @@ s3_device_set_reps_bucket_fn(Device *p_self, DevicePropertyBase *base, return device_simple_property_set_fn(p_self, base, val, surety, source); } +static gboolean +s3_device_set_http_v1_1_version(Device *p_self, DevicePropertyBase *base, + GValue *val, PropertySurety surety, PropertySource source) +{ + S3Device *self = S3_DEVICE(p_self); + + self->reps_bucket = g_value_get_boolean(val); + + return device_simple_property_set_fn(p_self, base, val, surety, source); +} +/* end of setter functions */ + static Device* s3_device_factory(char * device_name, char * device_type, char * device_node) { @@ -2774,7 +2804,8 @@ setup_handle(S3Device * self) { self->reuse_connection, self->read_from_glacier, self->timeout, - self->reps, self->reps_bucket); + self->reps, self->reps_bucket + self->http_version_v1_1); if (self->s3t[thread].s3 == NULL) { device_set_error(d_self, g_strdup(_("Internal error creating S3 handle")), diff --git a/device-src/s3-device.h b/device-src/s3-device.h index 58a21a89fc..68256da190 100644 --- a/device-src/s3-device.h +++ b/device-src/s3-device.h @@ -180,6 +180,8 @@ struct _S3Device { /* CAStor */ char *reps; char *reps_bucket; + + gboolean http_version_v1_1; }; /* diff --git a/device-src/s3.c b/device-src/s3.c index a575b7d47c..12d3e3613c 100644 --- a/device-src/s3.c +++ b/device-src/s3.c @@ -237,6 +237,8 @@ struct S3Handle { /* CAStor */ char *reps; char *reps_bucket; + + gboolean http_v1_1; }; typedef struct { @@ -3205,7 +3207,8 @@ s3_open(const char *access_key, const gboolean read_from_glacier, const long timeout, const char *reps, - const char *reps_bucket) + const char *reps_bucket, + const gboolean http_v1_1) { S3Handle *hdl; char *hwp; @@ -3330,6 +3333,8 @@ s3_open(const char *access_key, hdl->service_path = NULL; } + hdl->http_v1_1 = http_v1_1; + s3_new_curl(hdl); if (!hdl->curl) goto error; return hdl; @@ -3372,8 +3377,9 @@ s3_new_curl( } #endif } - /* google cloud api's are not HTTP2 compatiable YET*/ - if (hdl->s3_api == S3_API_OAUTH2) + + /* if user wants to set HTTP version to 1.1 */ + if (hdl->http_v1_1) curl_easy_setopt(hdl->curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); } From ffb8d264c7d8c92ac60d37b453b06b6dce18ff15 Mon Sep 17 00:00:00 2001 From: Prajwal T R Date: Mon, 22 May 2023 16:11:49 -0600 Subject: [PATCH 7/7] fix : setter function for http_v1_1 --- device-src/s3-device.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/device-src/s3-device.c b/device-src/s3-device.c index 260d43cc00..8005f8e069 100644 --- a/device-src/s3-device.c +++ b/device-src/s3-device.c @@ -2339,7 +2339,7 @@ s3_device_set_http_v1_1_version(Device *p_self, DevicePropertyBase *base, { S3Device *self = S3_DEVICE(p_self); - self->reps_bucket = g_value_get_boolean(val); + self->http_v1_1 = g_value_get_boolean(val); return device_simple_property_set_fn(p_self, base, val, surety, source); }