From a21f96fb545a4527c5b9c40a1e4be5fca7dc7ebd Mon Sep 17 00:00:00 2001 From: Gershon Papiashvili Date: Mon, 27 Jun 2022 02:23:20 +0300 Subject: [PATCH 01/18] out_azure_kusto: create azure kusto output plugin Signed-off-by: Gershon Papiashvili --- .gitignore | 1 + CMakeLists.txt | 1 + cmake/windows-setup.cmake | 1 + plugins/CMakeLists.txt | 1 + plugins/out_azure_kusto/CMakeLists.txt | 7 + plugins/out_azure_kusto/azure_kusto.c | 509 +++++++++++++++ plugins/out_azure_kusto/azure_kusto.h | 110 ++++ plugins/out_azure_kusto/azure_kusto_conf.c | 638 +++++++++++++++++++ plugins/out_azure_kusto/azure_kusto_conf.h | 31 + plugins/out_azure_kusto/azure_kusto_ingest.c | 430 +++++++++++++ plugins/out_azure_kusto/azure_kusto_ingest.h | 28 + 11 files changed, 1757 insertions(+) create mode 100644 plugins/out_azure_kusto/CMakeLists.txt create mode 100644 plugins/out_azure_kusto/azure_kusto.c create mode 100644 plugins/out_azure_kusto/azure_kusto.h create mode 100644 plugins/out_azure_kusto/azure_kusto_conf.c create mode 100644 plugins/out_azure_kusto/azure_kusto_conf.h create mode 100644 plugins/out_azure_kusto/azure_kusto_ingest.c create mode 100644 plugins/out_azure_kusto/azure_kusto_ingest.h diff --git a/.gitignore b/.gitignore index 222e11ae6b6..ca097d6af46 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .idea .vscode +.clang-format .DS_Store *~ _book/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 97ccaf16af0..367ce5aaf1e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -171,6 +171,7 @@ option(FLB_IN_WINDOWS_EXPORTER_METRICS "Enable windows exporter metrics input pl option(FLB_IN_OPENTELEMETRY "Enable OpenTelemetry input plugin" Yes) option(FLB_OUT_AZURE "Enable Azure output plugin" Yes) option(FLB_OUT_AZURE_BLOB "Enable Azure output plugin" Yes) +option(FLB_OUT_AZURE_KUSTO "Enable Azure Kusto output plugin" Yes) option(FLB_OUT_BIGQUERY "Enable BigQuery output plugin" Yes) option(FLB_OUT_CALYPTIA "Enable Calyptia monitoring plugin" Yes) option(FLB_OUT_COUNTER "Enable Counter output plugin" Yes) diff --git a/cmake/windows-setup.cmake b/cmake/windows-setup.cmake index 26501f0952b..16e6a0364f8 100644 --- a/cmake/windows-setup.cmake +++ b/cmake/windows-setup.cmake @@ -51,6 +51,7 @@ if(FLB_WINDOWS_DEFAULTS) # ============== set(FLB_OUT_AZURE Yes) set(FLB_OUT_AZURE_BLOB Yes) + set(FLB_OUT_AZURE_KUSTO Yes) set(FLB_OUT_BIGQUERY No) set(FLB_OUT_COUNTER Yes) set(FLB_OUT_DATADOG Yes) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 1192bf9a04f..8aa35929398 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -223,6 +223,7 @@ REGISTER_IN_PLUGIN("in_forward") REGISTER_IN_PLUGIN("in_random") REGISTER_OUT_PLUGIN("out_azure") REGISTER_OUT_PLUGIN("out_azure_blob") +REGISTER_OUT_PLUGIN("out_azure_kusto") REGISTER_OUT_PLUGIN("out_bigquery") REGISTER_OUT_PLUGIN("out_calyptia") REGISTER_OUT_PLUGIN("out_counter") diff --git a/plugins/out_azure_kusto/CMakeLists.txt b/plugins/out_azure_kusto/CMakeLists.txt new file mode 100644 index 00000000000..6803bee09c2 --- /dev/null +++ b/plugins/out_azure_kusto/CMakeLists.txt @@ -0,0 +1,7 @@ +set(src + azure_kusto.c + azure_kusto_conf.c + azure_kusto_ingest.c + ) + +FLB_PLUGIN(out_azure_kusto "${src}" "") diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c new file mode 100644 index 00000000000..38ca6f5d1e9 --- /dev/null +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -0,0 +1,509 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "azure_kusto.h" +#include "azure_kusto_conf.h" +#include "azure_kusto_ingest.h" + +/* Create a new oauth2 context and get a oauth2 token */ +static int azure_kusto_get_oauth2_token(struct flb_azure_kusto *ctx) +{ + int ret; + char *token; + + /* Clear any previous oauth2 payload content */ + flb_oauth2_payload_clear(ctx->o); + + ret = flb_oauth2_payload_append(ctx->o, "grant_type", 10, "client_credentials", 18); + if (ret == -1) { + flb_plg_error(ctx->ins, "error appending oauth2 params"); + return -1; + } + + ret = flb_oauth2_payload_append(ctx->o, "scope", 5, FLB_AZURE_KUSTO_SCOPE, -1); + if (ret == -1) { + flb_plg_error(ctx->ins, "error appending oauth2 params"); + return -1; + } + + ret = flb_oauth2_payload_append(ctx->o, "client_id", -1, ctx->client_id, -1); + if (ret == -1) { + flb_plg_error(ctx->ins, "error appending oauth2 params"); + return -1; + } + + ret = flb_oauth2_payload_append(ctx->o, "client_secret", -1, ctx->client_secret, -1); + if (ret == -1) { + flb_plg_error(ctx->ins, "error appending oauth2 params"); + return -1; + } + + /* Retrieve access token */ + token = flb_oauth2_token_get(ctx->o); + if (!token) { + flb_plg_error(ctx->ins, "error retrieving oauth2 access token"); + return -1; + } + + return 0; +} + +flb_sds_t get_azure_kusto_token(struct flb_azure_kusto *ctx) +{ + int ret = 0; + flb_sds_t output = NULL; + + if (pthread_mutex_lock(&ctx->token_mutex)) { + flb_plg_error(ctx->ins, "error locking mutex"); + return NULL; + } + + if (flb_oauth2_token_expired(ctx->o) == FLB_TRUE) { + ret = azure_kusto_get_oauth2_token(ctx); + } + + /* Copy string to prevent race conditions (get_oauth2 can free the string) */ + if (ret == 0) { + output = flb_sds_create(ctx->o->token_type); + flb_sds_printf(&output, " %s", ctx->o->access_token); + } + + if (pthread_mutex_unlock(&ctx->token_mutex)) { + flb_plg_error(ctx->ins, "error unlocking mutex"); + if (output) { + flb_sds_destroy(output); + } + return NULL; + } + + return output; +} + +/** + * Executes a control command against kusto's endpoint + * + * @param ctx Plugin's context + * @param csl Kusto's control command + * @return flb_sds_t Returns the response or NULL on error. + */ +flb_sds_t execute_ingest_csl_command(struct flb_azure_kusto *ctx, const char *csl) +{ + flb_sds_t token; + flb_sds_t body; + size_t b_sent; + int ret; + struct flb_upstream_conn *u_conn; + struct flb_http_client *c; + flb_sds_t resp; + + /* Get upstream connection */ + u_conn = flb_upstream_conn_get(ctx->u); + if (!u_conn) { + return NULL; + } + + /* Get or renew Token */ + token = get_azure_kusto_token(ctx); + if (!token) { + flb_plg_error(ctx->ins, "cannot retrieve oauth2 token"); + goto execute_error; + } + + /* Compose request body */ + body = + flb_sds_create_size(sizeof(FLB_AZURE_KUSTO_MGMT_BODY_TEMPLATE) - 2 + strlen(csl)); + if (!body) { + flb_plg_error(ctx->ins, "cannot construct request body"); + goto execute_error; + } + + body = flb_sds_printf(&body, FLB_AZURE_KUSTO_MGMT_BODY_TEMPLATE, csl); + + /* Compose HTTP Client request */ + c = flb_http_client(u_conn, FLB_HTTP_POST, FLB_AZURE_KUSTO_MGMT_URI_PATH, body, + flb_sds_len(body), NULL, 0, NULL, 0); + if (!c) { + flb_plg_error(ctx->ins, "cannot create HTTP client context"); + goto execute_error; + } + + /* Add headers */ + flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); + flb_http_add_header(c, "Content-Type", 12, "application/json", 16); + flb_http_add_header(c, "Accept", 6, "application/json", 16); + flb_http_add_header(c, "Authorization", 13, token, flb_sds_len(token)); + + flb_http_buffer_size(c, FLB_HTTP_DATA_SIZE_MAX * 10); + + /* Send HTTP request */ + ret = flb_http_do(c, &b_sent); + flb_plg_debug(ctx->ins, "Kusto ingestion command request http_do=%i, HTTP Status: %i", + ret, c->resp.status); + + if (ret != 0) { + flb_plg_error(ctx->ins, "cannot send HTTP request"); + goto execute_error; + } + + /* Validate return status and HTTP status if set */ + if (c->resp.status != 200) { + if (c->resp.payload_size > 0) { + flb_plg_debug(ctx->ins, "Request failed and returned: \n%s", c->resp.payload); + } + else { + flb_plg_debug(ctx->ins, "Request failed"); + } + goto execute_error; + } + + ret = c->resp.payload_size; + /* Copy payload response to the response param */ + resp = flb_sds_create_len(c->resp.payload, c->resp.payload_size); + + flb_http_client_destroy(c); + flb_upstream_conn_release(u_conn); + flb_sds_destroy(token); + flb_sds_destroy(body); + + return resp; + +execute_error: + if (c) { + flb_http_client_destroy(c); + } + + if (u_conn) { + flb_upstream_conn_release(u_conn); + } + + if (token) { + flb_sds_destroy(token); + } + + if (body) { + flb_sds_destroy(body); + } + + return NULL; +} + +static int cb_azure_kusto_init(struct flb_output_instance *ins, struct flb_config *config, + void *data) +{ + char *token; + int io_flags = FLB_IO_TLS; + int ret; + struct flb_azure_kusto *ctx; + + /* Create config context */ + ctx = flb_azure_kusto_conf_create(ins, config); + if (!ctx) { + flb_plg_error(ins, "configuration failed"); + return -1; + } + + flb_output_set_context(ins, ctx); + + /* Network mode IPv6 */ + if (ins->host.ipv6 == FLB_TRUE) { + io_flags |= FLB_IO_IPV6; + } + + /* Create mutex for acquiring oauth tokens and getting ingestion resources (they are + * shared across flush coroutines) + */ + pthread_mutex_init(&ctx->token_mutex, NULL); + pthread_mutex_init(&ctx->resources_mutex, NULL); + + /* + * Create upstream context for Kusto Ingestion endpoint + */ + ctx->u = flb_upstream_create_url(config, ctx->ingestion_endpoint, io_flags, ins->tls); + if (!ctx->u) { + flb_plg_error(ctx->ins, "upstream creation failed"); + return -1; + } + + /* Create oauth2 context */ + ctx->o = + flb_oauth2_create(ctx->config, ctx->oauth_url, FLB_AZURE_KUSTO_TOKEN_REFRESH); + if (!ctx->o) { + flb_plg_error(ctx->ins, "cannot create oauth2 context"); + return -1; + } + flb_output_upstream_set(ctx->u, ins); + + /* Get or renew the OAuth2 token */ + token = get_azure_kusto_token(ctx); + + if (!token) { + flb_plg_warn(ctx->ins, "token retrieval failed"); + } + else { + flb_sds_destroy(token); + + /* load or refresh ingestion resources */ + ret = azure_kusto_load_ingestion_resources(ctx, config); + if (ret != 0) { + flb_plg_warn(ctx->ins, "ingestion resources loading failed"); + } + } + + return 0; +} + +static int azure_kusto_format(struct flb_azure_kusto *ctx, const char *tag, int tag_len, + const void *data, size_t bytes, void **out_data, + size_t *out_size) +{ + int records = 0; + size_t off = 0; + msgpack_unpacked result; + msgpack_sbuffer mp_sbuf; + msgpack_packer mp_pck; + /* for sub msgpack objs */ + int map_size; + struct flb_time tm; + struct tm tms; + msgpack_object root; + msgpack_object *obj; + char time_formatted[32]; + size_t s; + int len; + + /* output buffer */ + flb_sds_t out_buf; + + /* Create temporary msgpack buffer */ + msgpack_sbuffer_init(&mp_sbuf); + msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); + + /* Create array for all records */ + records = flb_mp_count(data, bytes); + if (records <= 0) { + msgpack_sbuffer_destroy(&mp_sbuf); + return -1; + } + msgpack_pack_array(&mp_pck, records); + + off = 0; + msgpack_unpacked_init(&result); + while (msgpack_unpack_next(&result, data, bytes, &off) == MSGPACK_UNPACK_SUCCESS) { + root = result.data; + /* Each array must have two entries: time and record */ + if (root.type != MSGPACK_OBJECT_ARRAY) { + continue; + } + if (root.via.array.size != 2) { + continue; + } + + /* Get timestamp and object */ + flb_time_pop_from_msgpack(&tm, &result, &obj); + + map_size = 1; + if (ctx->include_time_key == FLB_TRUE) { + map_size++; + } + + if (ctx->include_tag_key == FLB_TRUE) { + map_size++; + } + + msgpack_pack_map(&mp_pck, map_size); + + /* include_time_key */ + if (ctx->include_time_key == FLB_TRUE) { + msgpack_pack_str(&mp_pck, flb_sds_len(ctx->time_key)); + msgpack_pack_str_body(&mp_pck, ctx->time_key, flb_sds_len(ctx->time_key)); + + /* Append the time value as ISO 8601 */ + gmtime_r(&tm.tm.tv_sec, &tms); + s = strftime(time_formatted, sizeof(time_formatted) - 1, + FLB_PACK_JSON_DATE_ISO8601_FMT, &tms); + + len = snprintf(time_formatted + s, sizeof(time_formatted) - 1 - s, + ".%03" PRIu64 "Z", (uint64_t)tm.tm.tv_nsec / 1000000); + s += len; + msgpack_pack_str(&mp_pck, s); + msgpack_pack_str_body(&mp_pck, time_formatted, s); + } + + /* include_tag_key */ + if (ctx->include_tag_key == FLB_TRUE) { + msgpack_pack_str(&mp_pck, flb_sds_len(ctx->tag_key)); + msgpack_pack_str_body(&mp_pck, ctx->tag_key, flb_sds_len(ctx->tag_key)); + msgpack_pack_str(&mp_pck, tag_len); + msgpack_pack_str_body(&mp_pck, tag, tag_len); + } + + msgpack_pack_str(&mp_pck, flb_sds_len(ctx->log_key)); + msgpack_pack_str_body(&mp_pck, ctx->log_key, flb_sds_len(ctx->log_key)); + msgpack_pack_object(&mp_pck, *obj); + } + + /* Convert from msgpack to JSON */ + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + msgpack_sbuffer_destroy(&mp_sbuf); + + if (!out_buf) { + flb_plg_error(ctx->ins, "error formatting JSON payload"); + msgpack_unpacked_destroy(&result); + return -1; + } + + *out_data = out_buf; + *out_size = flb_sds_len(out_buf); + + /* Cleanup */ + msgpack_unpacked_destroy(&result); + + return 0; +} + +static void cb_azure_kusto_flush(struct flb_event_chunk *event_chunk, + struct flb_output_flush *out_flush, + struct flb_input_instance *i_ins, void *out_context, + struct flb_config *config) +{ + (void)i_ins; + (void)config; + int ret; + flb_sds_t json; + size_t json_size; + size_t tag_len; + + struct flb_azure_kusto *ctx = out_context; + + flb_plg_trace(ctx->ins, "flushing bytes %zu", event_chunk->size); + + tag_len = flb_sds_len(event_chunk->tag); + + /* Load or refresh ingestion resources */ + ret = azure_kusto_load_ingestion_resources(ctx, config); + if (ret != 0) { + flb_plg_error(ctx->ins, "cannot load ingestion resources"); + goto flush_error; + } + + /* Reformat msgpack to JSON payload */ + ret = azure_kusto_format(ctx, event_chunk->tag, tag_len, event_chunk->data, + event_chunk->size, (void **)&json, &json_size); + if (ret != 0) { + flb_plg_error(ctx->ins, "cannot reformat data into json"); + goto flush_error; + } + + ret = azure_kusto_queued_ingestion(ctx, event_chunk->tag, tag_len, json, json_size); + if (ret != 0) { + flb_plg_error(ctx->ins, "cannot perform queued ingestion"); + goto flush_error; + } + + /* Cleanup */ + flb_sds_destroy(json); + + /* Done */ + FLB_OUTPUT_RETURN(FLB_OK); + +flush_error: + if (json) { + flb_sds_destroy(json); + } + FLB_OUTPUT_RETURN(FLB_RETRY); +} + +static int cb_azure_kusto_exit(void *data, struct flb_config *config) +{ + struct flb_azure_kusto *ctx = data; + + if (!ctx) { + return -1; + } + + if (ctx->u) { + flb_upstream_destroy(ctx->u); + ctx->u = NULL; + } + + flb_azure_kusto_conf_destroy(ctx); + + return 0; +} + +static struct flb_config_map config_map[] = { + {FLB_CONFIG_MAP_STR, "tenant_id", (char *)NULL, 0, FLB_TRUE, + offsetof(struct flb_azure_kusto, tenant_id), + "Set the tenant ID of the AAD application used for authentication"}, + {FLB_CONFIG_MAP_STR, "client_id", (char *)NULL, 0, FLB_TRUE, + offsetof(struct flb_azure_kusto, client_id), + "Set the client ID (Application ID) of the AAD application used for authentication"}, + {FLB_CONFIG_MAP_STR, "client_secret", (char *)NULL, 0, FLB_TRUE, + offsetof(struct flb_azure_kusto, client_secret), + "Set the client secret (Application Password) of the AAD application used for " + "authentication"}, + {FLB_CONFIG_MAP_STR, "ingestion_endpoint", (char *)NULL, 0, FLB_TRUE, + offsetof(struct flb_azure_kusto, ingestion_endpoint), + "Set the Kusto cluster's ingestion endpoint URL (e.g. " + "https://ingest-mycluster.eastus.kusto.windows.net)"}, + {FLB_CONFIG_MAP_STR, "database_name", (char *)NULL, 0, FLB_TRUE, + offsetof(struct flb_azure_kusto, database_name), "Set the database name"}, + {FLB_CONFIG_MAP_STR, "table_name", (char *)NULL, 0, FLB_TRUE, + offsetof(struct flb_azure_kusto, table_name), "Set the table name"}, + {FLB_CONFIG_MAP_STR, "ingestion_mapping_reference", (char *)NULL, 0, FLB_TRUE, + offsetof(struct flb_azure_kusto, ingestion_mapping_reference), + "Set the ingestion mapping reference"}, + {FLB_CONFIG_MAP_STR, "log_key", FLB_AZURE_KUSTO_DEFAULT_LOG_KEY, 0, FLB_TRUE, + offsetof(struct flb_azure_kusto, log_key), "The key name of event payload"}, + {FLB_CONFIG_MAP_BOOL, "include_tag_key", "false", 0, FLB_TRUE, + offsetof(struct flb_azure_kusto, include_tag_key), + "If enabled, tag is appended to output. " + "The key name is used 'tag_key' property."}, + {FLB_CONFIG_MAP_STR, "tag_key", FLB_AZURE_KUSTO_DEFAULT_TAG_KEY, 0, FLB_TRUE, + offsetof(struct flb_azure_kusto, tag_key), + "The key name of tag. If 'include_tag_key' is false, " + "This property is ignored"}, + {FLB_CONFIG_MAP_BOOL, "include_time_key", "false", 0, FLB_TRUE, + offsetof(struct flb_azure_kusto, include_time_key), + "If enabled, time is appended to output. " + "The key name is used 'time_key' property."}, + {FLB_CONFIG_MAP_STR, "time_key", FLB_AZURE_KUSTO_DEFAULT_TIME_KEY, 0, FLB_TRUE, + offsetof(struct flb_azure_kusto, time_key), + "The key name of the time. If 'include_time_key' is false, " + "This property is ignored"}, + /* EOF */ + {0}}; + +struct flb_output_plugin out_azure_kusto_plugin = { + .name = "azure_kusto", + .description = "Send events to Kusto (Azure Data Explorer)", + .cb_init = cb_azure_kusto_init, + .cb_flush = cb_azure_kusto_flush, + .cb_exit = cb_azure_kusto_exit, + .config_map = config_map, + /* Plugin flags */ + .flags = FLB_OUTPUT_NET | FLB_IO_TLS, +}; diff --git a/plugins/out_azure_kusto/azure_kusto.h b/plugins/out_azure_kusto/azure_kusto.h new file mode 100644 index 00000000000..ac4eedfd0a8 --- /dev/null +++ b/plugins/out_azure_kusto/azure_kusto.h @@ -0,0 +1,110 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_OUT_AZURE_KUSTO +#define FLB_OUT_AZURE_KUSTO + +#include +#include +#include +#include +#include + +/* refresh token every 50 minutes */ +#define FLB_AZURE_KUSTO_TOKEN_REFRESH 3000 + +/* Kusto streaming inserts oauth scope */ +#define FLB_AZURE_KUSTO_SCOPE "https://help.kusto.windows.net/.default" + +/* MSAL authorization URL */ +#define FLB_MSAL_AUTH_URL_TEMPLATE \ + "https://login.microsoftonline.com/%s/oauth2/v2.0/token" + +#define FLB_AZURE_KUSTO_MGMT_URI_PATH "/v1/rest/mgmt" +#define FLB_AZURE_KUSTO_MGMT_BODY_TEMPLATE "{\"csl\":\"%s\", \"db\": \"NetDefaultDB\"}" + +#define FLB_AZURE_KUSTO_DEFAULT_TIME_KEY "timestamp" +#define FLB_AZURE_KUSTO_DEFAULT_TAG_KEY "tag" +#define FLB_AZURE_KUSTO_DEFAULT_LOG_KEY "log" + +#define AZURE_KUSTO_RESOURCE_STORAGE 0 +#define AZURE_KUSTO_RESOURCE_QUEUE 1 + +#define AZURE_KUSTO_RESOURCE_UPSTREAM_URI "uri" +#define AZURE_KUSTO_RESOURCE_UPSTREAM_SAS "sas" + +#define FLB_AZURE_KUSTO_RESOURCES_LOAD_INTERVAL_SEC 3600 + +struct flb_azure_kusto_resources { + struct flb_upstream_ha *blob_ha; + struct flb_upstream_ha *queue_ha; + flb_sds_t identity_token; + + /* used to reload resouces after some time */ + time_t load_time; +}; + +struct flb_azure_kusto { + /* azure_kusto configuration */ + flb_sds_t tenant_id; + flb_sds_t client_id; + flb_sds_t client_secret; + flb_sds_t ingestion_endpoint; + flb_sds_t database_name; + flb_sds_t table_name; + flb_sds_t ingestion_mapping_reference; + + /* records configuration */ + flb_sds_t log_key; + int include_tag_key; + flb_sds_t tag_key; + int include_time_key; + flb_sds_t time_key; + + /* --- internal data --- */ + + flb_sds_t ingestion_mgmt_endpoint; + + /* oauth2 context */ + flb_sds_t oauth_url; + struct flb_oauth2 *o; + + /* mutex for acquiring oauth tokens */ + pthread_mutex_t token_mutex; + + /* ingestion resources */ + struct flb_azure_kusto_resources *resources; + + /* mutex for loading reosurces */ + pthread_mutex_t resources_mutex; + + /* Upstream connection to the backend server */ + struct flb_upstream *u; + + /* Fluent Bit context */ + struct flb_config *config; + + /* Plugin output instance reference */ + struct flb_output_instance *ins; +}; + +flb_sds_t get_azure_kusto_token(struct flb_azure_kusto *ctx); +flb_sds_t execute_ingest_csl_command(struct flb_azure_kusto *ctx, const char *csl); + +#endif diff --git a/plugins/out_azure_kusto/azure_kusto_conf.c b/plugins/out_azure_kusto/azure_kusto_conf.c new file mode 100644 index 00000000000..ca4931461af --- /dev/null +++ b/plugins/out_azure_kusto/azure_kusto_conf.c @@ -0,0 +1,638 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "azure_kusto.h" +#include "azure_kusto_conf.h" + +static struct flb_upstream_node *flb_upstream_node_create_url(struct flb_azure_kusto *ctx, + struct flb_config *config, + const char *url) +{ + int ret; + char *prot = NULL; + char *host = NULL; + char *port = NULL; + char *uri = NULL; + char *tmp; + struct flb_hash *kv; + struct flb_upstream_node *node = NULL; + int uri_length; + int sas_length; + + /* Parse and split URL */ + ret = flb_utils_url_split(url, &prot, &host, &port, &uri); + if (ret == -1) { + flb_plg_error(ctx->ins, "invalid URL: %s", url); + goto upstream_node_create_url_out; + } + + /* find sas token in query */ + tmp = strchr(uri, '?'); + if (!tmp) { + flb_plg_error(ctx->ins, "uri has no sas token query: %s", uri); + goto upstream_node_create_url_out; + } + uri_length = tmp - uri; + sas_length = strnlen(tmp + 1, 256); + + /* kv that will hold base uri, and sas token */ + kv = flb_hash_create(FLB_HASH_EVICT_NONE, 2, 2); + if (!kv) { + flb_plg_error(ctx->ins, "error creating upstream node hash table"); + goto upstream_node_create_url_out; + } + + ret = flb_hash_add(kv, AZURE_KUSTO_RESOURCE_UPSTREAM_URI, 3, uri, uri_length); + if (ret == -1) { + flb_plg_error(ctx->ins, "error storing resource uri"); + goto upstream_node_create_url_out; + } + + ret = flb_hash_add(kv, AZURE_KUSTO_RESOURCE_UPSTREAM_SAS, 3, tmp + 1, sas_length); + if (ret == -1) { + flb_plg_error(ctx->ins, "error storing resource sas token"); + goto upstream_node_create_url_out; + } + + node = flb_upstream_node_create(NULL, host, port, FLB_TRUE, ctx->ins->tls->verify, + ctx->ins->tls->debug, ctx->ins->tls->vhost, NULL, + NULL, NULL, NULL, NULL, kv, config); + if (!node) { + flb_plg_error(ctx->ins, "error creating resource upstream node"); + goto upstream_node_create_url_out; + } + + flb_free(prot); + flb_free(host); + flb_free(port); + flb_free(uri); + + return node; + +upstream_node_create_url_out: + if (prot) { + flb_free(prot); + } + if (host) { + flb_free(host); + } + if (port) { + flb_free(port); + } + if (uri) { + flb_free(uri); + } + if (kv) { + flb_hash_destroy(kv); + } + return node; +} + +static int flb_azure_kusto_resources_clear(struct flb_azure_kusto_resources *resources) +{ + if (!resources) { + return -1; + } + + if (resources->blob_ha) { + flb_upstream_ha_destroy(resources->blob_ha); + resources->blob_ha = NULL; + } + + if (resources->queue_ha) { + flb_upstream_ha_destroy(resources->queue_ha); + resources->queue_ha = NULL; + } + + if (resources->identity_token) { + flb_sds_destroy(resources->identity_token); + resources->identity_token = NULL; + } + + resources->load_time = 0; + + return 0; +} + +/** + * Parses ".get ingestion resources" response into HA upstreams of the queue & blob + * resources in the response. + * + * @param ctx Pointer to the plugin's context + * @param config Pointer to the config + * @param response sds string containing the response body + * @param blob_ha Pointer to an HA upstream for the blob resources, that would be + * allocated here. + * @param queue_ha Pointer to an HA upstream for the queue resources, that would be + * allocated here. + * @return int 0 on success, -1 on failure + */ +static int parse_storage_resources(struct flb_azure_kusto *ctx, struct flb_config *config, + flb_sds_t response, struct flb_upstream_ha *blob_ha, + struct flb_upstream_ha *queue_ha) +{ + jsmn_parser parser; + jsmntok_t *t; + jsmntok_t *tokens; + int tok_size = 100; + int ret; + int i; + int blob_count = 0; + int queue_count = 0; + char *token_str; + int token_str_len; + int resource_type; + struct flb_upstream_node *node; + struct flb_upstream_ha *ha; + flb_sds_t resource_uri = flb_sds_create(NULL); + + /* Response is a json in the form of + * { + * "Tables": [ + * { + * "TableName": "Table_0", + * "Columns": [...], + * "Rows": [ + * [ + * ("TempStorage" | "SecuredReadyForAggregationQueue" | + * "SuccessfulIngestionsQueue" | "FailedIngestionsQueue" | "IngestionsStatusTable"), + * + * ], + * ... + * ] + * } + * ] + * } + */ + + jsmn_init(&parser); + tokens = flb_calloc(1, sizeof(jsmntok_t) * tok_size); + if (!tokens) { + flb_plg_error(ctx->ins, "error allocating tokens"); + goto load_storage_resources_error; + } + + ret = jsmn_parse(&parser, response, flb_sds_len(response), tokens, tok_size); + if (ret <= 0) { + flb_plg_error(ctx->ins, "error parsing JSON response: %s", response); + goto load_storage_resources_error; + } + + /* skip all tokens until we reach "Rows" */ + for (i = 0; i < ret - 1; i++) { + t = &tokens[i]; + + if (t->type != JSMN_STRING) { + continue; + } + + token_str = response + t->start; + token_str_len = (t->end - t->start); + + /** + * if we found the Rows key, skipping this token and the next one (key and + * wrapping array value) + */ + if (token_str_len == 4 && strncmp(token_str, "Rows", 4) == 0) { + i += 2; + break; + } + } + + /* iterating rows, each row will have 3 tokens: the array holding the column values, + * the first value containing the resource type, and the second value containing the + * resource uri */ + for (; i < ret; i++) { + t = &tokens[i]; + + /** + * each token should be an array with 2 strings: + * First will be the resource type (TempStorage, SecuredReadyForAggregationQueue, + * etc...) Second will be the SAS URI + */ + if (t->type != JSMN_ARRAY) { + break; + } + + /* move to the next token, first item in the array - resource type */ + i++; + t = &tokens[i]; + if (t->type != JSMN_STRING) { + break; + } + + token_str = response + t->start; + token_str_len = (t->end - t->start); + + flb_plg_debug(ctx->ins, "found resource of type: %.*s ", t->end - t->start, + response + t->start); + + if (token_str_len == 11 && strncmp(token_str, "TempStorage", 11) == 0) { + resource_type = AZURE_KUSTO_RESOURCE_STORAGE; + } + else if (token_str_len == 31 && + strncmp(token_str, "SecuredReadyForAggregationQueue", 31) == 0) { + resource_type = AZURE_KUSTO_RESOURCE_QUEUE; + } + /* we don't care about other resources so we just skip the next token and move + on to the next pair */ + else { + i++; + continue; + } + + /* move to the next token, second item in the array - resource URI */ + i++; + t = &tokens[i]; + + if (t->type != JSMN_STRING) { + break; + } + + token_str = response + t->start; + token_str_len = (t->end - t->start); + + resource_uri = flb_sds_copy(resource_uri, token_str, token_str_len); + if (resource_type == AZURE_KUSTO_RESOURCE_QUEUE) { + ha = queue_ha; + queue_count++; + } + else { + ha = blob_ha; + blob_count++; + } + + if (!ha) { + flb_plg_error(ctx->ins, "error creating HA upstream"); + goto load_storage_resources_error; + } + + node = flb_upstream_node_create_url(ctx, config, resource_uri); + if (!node) { + flb_plg_error(ctx->ins, "error creating HA upstream node"); + goto load_storage_resources_error; + } + + flb_upstream_ha_node_add(ha, node); + } + + if (!queue_count || !blob_count) { + flb_plg_error(ctx->ins, "error parsing resources: missing resources"); + goto load_storage_resources_error; + } + + flb_sds_destroy(resource_uri); + flb_free(tokens); + + flb_plg_debug(ctx->ins, "parsed %d blob resources and %d queue resources", blob_count, + queue_count); + + return 0; + +load_storage_resources_error: + + if (tokens) { + flb_free(tokens); + } + + if (resource_uri) { + flb_sds_destroy(resource_uri); + } + + return -1; +} + +/** + * Parses ".get kusto identity token" response and returns the token as an sds string + * + * @param ctx Pointer to the plugin's context + * @param response sds string containing the response body + * @return flb_sds_t The parsed token + */ +static flb_sds_t parse_ingestion_identity_token(struct flb_azure_kusto *ctx, + flb_sds_t response) +{ + flb_sds_t identity_token; + int tok_size = 19; + jsmn_parser parser; + jsmntok_t *t; + jsmntok_t *tokens; + int ret; + char *token_str; + int token_str_len; + + /** + * Response is a json in the form of + * { + * "Tables": [ + * { + * "TableName": "Table_0", + * "Columns": [{ + * "ColumnName": "AuthorizationContext", + * "DataType": "String", + * "ColumnType": "string" + * }], + * "Rows": [ + * [ + * , + * ] + * ] + * } + * ] + * } + * i.e. only one row and one column is expected (exactly 13 tokens) and the value + * should be the last + */ + + jsmn_init(&parser); + tokens = flb_calloc(1, sizeof(jsmntok_t) * tok_size); + if (!tokens) { + flb_plg_error(ctx->ins, "error allocating tokens"); + return NULL; + } + + ret = jsmn_parse(&parser, response, flb_sds_len(response), tokens, tok_size); + if (ret <= 0) { + flb_plg_error(ctx->ins, "error parsing JSON response: %s", response); + flb_free(tokens); + return NULL; + } + + t = &tokens[tok_size - 1]; + if (t->type != JSMN_STRING) { + flb_plg_error(ctx->ins, "unexpected JSON response: %s", response); + flb_free(tokens); + return NULL; + } + + token_str = response + t->start; + token_str_len = (t->end - t->start); + + identity_token = flb_sds_create_len(token_str, token_str_len); + + flb_plg_debug(ctx->ins, "parsed kusto identity token: '%s'", identity_token); + + flb_free(tokens); + + return identity_token; +} + +int azure_kusto_load_ingestion_resources(struct flb_azure_kusto *ctx, + struct flb_config *config) +{ + int ret; + flb_sds_t response; + flb_sds_t identity_token = NULL; + struct flb_upstream_ha *blob_ha = NULL; + struct flb_upstream_ha *queue_ha = NULL; + time_t now; + + if (pthread_mutex_lock(&ctx->resources_mutex)) { + flb_plg_error(ctx->ins, "error locking mutex"); + return -1; + } + + now = time(NULL); + + /* check if we have all resources and they are not stale */ + if (ctx->resources->blob_ha && ctx->resources->queue_ha && + ctx->resources->identity_token && + now - ctx->resources->load_time < FLB_AZURE_KUSTO_RESOURCES_LOAD_INTERVAL_SEC) { + if (pthread_mutex_unlock(&ctx->resources_mutex)) { + flb_plg_error(ctx->ins, "error unlocking mutex"); + return -1; + } + + flb_plg_debug(ctx->ins, "resources are already loaded and are not stale"); + return 0; + } + + flb_plg_info(ctx->ins, "loading kusto ingestion resourcs"); + + response = execute_ingest_csl_command(ctx, ".get ingestion resources"); + if (!response) { + flb_plg_error(ctx->ins, "error getting ingestion storage resources"); + goto load_resources_error; + } + + queue_ha = flb_upstream_ha_create("azure_kusto_queue_ha"); + blob_ha = flb_upstream_ha_create("azure_kusto_blob_ha"); + if (!queue_ha || !blob_ha) { + flb_plg_error(ctx->ins, "error creating storage resources upstreams"); + goto load_resources_error; + } + + ret = parse_storage_resources(ctx, config, response, blob_ha, queue_ha); + if (ret != 0) { + flb_plg_error(ctx->ins, "error parsing ingestion storage resources"); + goto load_resources_error; + } + + flb_sds_destroy(response); + + response = execute_ingest_csl_command(ctx, ".get kusto identity token"); + if (!response) { + flb_plg_error(ctx->ins, "error getting kusto identity token"); + goto load_resources_error; + } + + identity_token = parse_ingestion_identity_token(ctx, response); + if (!identity_token) { + flb_plg_error(ctx->ins, "error parsing ingestion identity token"); + goto load_resources_error; + } + + flb_sds_destroy(response); + + /* free old resources and point to newly allocated */ + flb_azure_kusto_resources_clear(ctx->resources); + ctx->resources->blob_ha = blob_ha; + ctx->resources->queue_ha = queue_ha; + ctx->resources->identity_token = identity_token; + ctx->resources->load_time = now; + + if (pthread_mutex_unlock(&ctx->resources_mutex)) { + flb_plg_error(ctx->ins, "error unlocking mutex"); + return -1; + } + + return 0; + +load_resources_error: + pthread_mutex_unlock(&ctx->resources_mutex); + + if (response) { + flb_sds_destroy(response); + } + + if (blob_ha) { + flb_upstream_ha_destroy(blob_ha); + } + + if (queue_ha) { + flb_upstream_ha_destroy(queue_ha); + } + + if (identity_token) { + flb_sds_destroy(identity_token); + } + + return -1; +} + +static int flb_azure_kusto_resources_destroy(struct flb_azure_kusto_resources *resources) +{ + if (!resources) { + return -1; + } + + flb_azure_kusto_resources_clear(resources); + + flb_free(resources); + + return 0; +} + +struct flb_azure_kusto *flb_azure_kusto_conf_create(struct flb_output_instance *ins, + struct flb_config *config) +{ + int ret; + struct flb_azure_kusto *ctx; + + /* Allocate config context */ + ctx = flb_calloc(1, sizeof(struct flb_azure_kusto)); + if (!ctx) { + flb_errno(); + return NULL; + } + ctx->ins = ins; + ctx->config = config; + + ret = flb_output_config_map_set(ins, (void *)ctx); + if (ret == -1) { + flb_plg_error(ins, "unable to load configuration"); + flb_free(ctx); + return NULL; + } + + /* config: 'tenant_id' */ + if (ctx->tenant_id == NULL) { + flb_plg_error(ctx->ins, "property 'tenant_id' is not defined."); + flb_azure_kusto_conf_destroy(ctx); + return NULL; + } + + /* config: 'client_id' */ + if (ctx->client_id == NULL) { + flb_plg_error(ctx->ins, "property 'client_id' is not defined"); + flb_azure_kusto_conf_destroy(ctx); + return NULL; + } + + /* config: 'client_secret' */ + if (ctx->client_secret == NULL) { + flb_plg_error(ctx->ins, "property 'client_secret' is not defined"); + flb_azure_kusto_conf_destroy(ctx); + return NULL; + } + + /* config: 'ingestion_endpoint' */ + if (ctx->ingestion_endpoint == NULL) { + flb_plg_error(ctx->ins, "property 'ingestion_endpoint' is not defined"); + flb_azure_kusto_conf_destroy(ctx); + return NULL; + } + + /* config: 'database_name' */ + if (ctx->database_name == NULL) { + flb_plg_error(ctx->ins, "property 'database_name' is not defined"); + flb_azure_kusto_conf_destroy(ctx); + return NULL; + } + + /* config: 'table_name' */ + if (ctx->table_name == NULL) { + flb_plg_error(ctx->ins, "property 'table_name' is not defined"); + flb_azure_kusto_conf_destroy(ctx); + return NULL; + } + + /* Create the auth URL */ + ctx->oauth_url = flb_sds_create_size(sizeof(FLB_MSAL_AUTH_URL_TEMPLATE) - 2 + + flb_sds_len(ctx->tenant_id)); + if (!ctx->oauth_url) { + flb_errno(); + flb_azure_kusto_conf_destroy(ctx); + return NULL; + } + ctx->oauth_url = + flb_sds_printf(&ctx->oauth_url, FLB_MSAL_AUTH_URL_TEMPLATE, ctx->tenant_id); + + ctx->resources = flb_calloc(1, sizeof(struct flb_azure_kusto_resources)); + if (!ctx->resources) { + flb_errno(); + flb_azure_kusto_conf_destroy(ctx); + return NULL; + } + + flb_plg_info(ctx->ins, "endpoint='%s', database='%s', table='%s'", + ctx->ingestion_endpoint, ctx->database_name, ctx->table_name); + + return ctx; +} + +int flb_azure_kusto_conf_destroy(struct flb_azure_kusto *ctx) +{ + if (!ctx) { + return -1; + } + + if (ctx->oauth_url) { + flb_sds_destroy(ctx->oauth_url); + ctx->oauth_url = NULL; + } + + if (ctx->o) { + flb_oauth2_destroy(ctx->o); + ctx->o = NULL; + } + + if (ctx->resources) { + flb_azure_kusto_resources_destroy(ctx->resources); + ctx->resources = NULL; + } + + flb_free(ctx); + return 0; +} diff --git a/plugins/out_azure_kusto/azure_kusto_conf.h b/plugins/out_azure_kusto/azure_kusto_conf.h new file mode 100644 index 00000000000..b4b2e3a3972 --- /dev/null +++ b/plugins/out_azure_kusto/azure_kusto_conf.h @@ -0,0 +1,31 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_OUT_AZURE_KUSTO_CONF_H +#define FLB_OUT_AZURE_KUSTO_CONF_H + +#include "azure_kusto.h" + +int azure_kusto_load_ingestion_resources(struct flb_azure_kusto *ctx, + struct flb_config *config); +struct flb_azure_kusto *flb_azure_kusto_conf_create(struct flb_output_instance *ins, + struct flb_config *config); +int flb_azure_kusto_conf_destroy(struct flb_azure_kusto *ctx); + +#endif diff --git a/plugins/out_azure_kusto/azure_kusto_ingest.c b/plugins/out_azure_kusto/azure_kusto_ingest.c new file mode 100644 index 00000000000..db5b3695834 --- /dev/null +++ b/plugins/out_azure_kusto/azure_kusto_ingest.c @@ -0,0 +1,430 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "azure_kusto_ingest.h" + +/* not really uuid but a random string in the form 00000000-0000-0000-0000-000000000000 */ +static char *generate_uuid() +{ + char *chars = "0123456789abcdef"; + char *uuid; + int i; + uint64_t rand; + + uuid = flb_malloc(36); + if (!uuid) { + flb_errno(); + return NULL; + } + + for (i = 0; i < 36; i++) { + if (i == 8 || i == 13 || i == 18 || i == 23) { + uuid[i] = '-'; + continue; + } + + if (flb_random_bytes((unsigned char *)&rand, sizeof(uint64_t))) { + rand = time(NULL); + } + uuid[i] = chars[rand % 16]; + } + + return uuid; +} + +static char *base64_encode(flb_sds_t s, size_t len, size_t *out_len) +{ + char *b64; + int ret; + + *out_len = (4 * ceil(((double)len / 3) + 1)); + b64 = flb_malloc(*out_len); + if (!b64) { + flb_errno(); + return NULL; + } + ret = mbedtls_base64_encode((unsigned char *)b64, *out_len, out_len, + (unsigned char *)s, len); + if (ret != 0) { + flb_error("cannot encode string %s into base64", s); + flb_free(b64); + return NULL; + } + + return b64; +} + +static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t blob_id, + flb_sds_t payload, size_t payload_size) +{ + int ret; + char *blob_uri; + size_t blob_uri_size; + char *blob_sas; + size_t blob_sas_size; + flb_sds_t uri; + flb_sds_t full_uri; + struct flb_upstream_node *u_node; + struct flb_upstream_conn *u_conn; + struct flb_http_client *c; + size_t resp_size; + time_t now; + struct tm tm; + char tmp[64]; + int len; + + now = time(NULL); + gmtime_r(&now, &tm); + len = strftime(tmp, sizeof(tmp) - 1, "%a, %d %b %Y %H:%M:%S GMT", &tm); + + u_node = flb_upstream_ha_node_get(ctx->resources->blob_ha); + if (!u_node) { + flb_plg_error(ctx->ins, "error getting blob upstream"); + return NULL; + } + + ret = flb_hash_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_URI, 3, + (void **)&blob_uri, &blob_uri_size); + if (ret == -1) { + flb_plg_error(ctx->ins, "error getting blob uri"); + return NULL; + } + ret = flb_hash_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_SAS, 3, + (void **)&blob_sas, &blob_sas_size); + if (ret == -1) { + flb_plg_error(ctx->ins, "error getting blob sas token"); + return NULL; + } + + u_conn = flb_upstream_conn_get(u_node->u); + if (!u_conn) { + flb_plg_error(ctx->ins, "error getting blob container upstream connection"); + return NULL; + } + + /* uri will be /.multijson? */ + uri = flb_sds_create_size(blob_uri_size + blob_sas_size + flb_sds_len(blob_id) + 13); + if (!uri) { + flb_plg_error(ctx->ins, "error creating blob container uri"); + goto create_blob_error; + } + uri = flb_sds_printf(&uri, "%s/%s.multijson?%s", blob_uri, blob_id, blob_sas); + + flb_plg_debug(ctx->ins, "uploading payload to blob uri: %s", uri); + + c = flb_http_client(u_conn, FLB_HTTP_PUT, uri, payload, payload_size, NULL, 0, NULL, + 0); + if (!c) { + flb_plg_error(ctx->ins, "cannot create HTTP client context for blob container"); + goto create_blob_error; + } + + flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); + flb_http_add_header(c, "Content-Type", 12, "application/json", 16); + flb_http_add_header(c, "x-ms-blob-type", 14, "BlockBlob", 9); + flb_http_add_header(c, "x-ms-date", 9, tmp, len); + flb_http_add_header(c, "x-ms-version", 12, "2019-12-12", 10); + + ret = flb_http_do(c, &resp_size); + flb_plg_debug(ctx->ins, "kusto blob upload request http_do=%i, HTTP Status: %i", ret, + c->resp.status); + if (ret != 0) { + flb_plg_error(ctx->ins, "cannot send HTTP request"); + goto create_blob_error; + } + + /* Validate return status and HTTP status if set */ + if (c->resp.status != 201) { + if (c->resp.payload_size > 0) { + flb_plg_debug(ctx->ins, "Request failed and returned: \n%s", c->resp.payload); + } + else { + flb_plg_debug(ctx->ins, "Request failed"); + } + goto create_blob_error; + } + + /* generate full blob uri & return it */ + full_uri = flb_sds_create_size(9 + flb_sds_len(u_node->host) + flb_sds_len(uri)); + if (!full_uri) { + flb_plg_error(ctx->ins, "cannot create full blob uri"); + goto create_blob_error; + } + full_uri = flb_sds_printf(&full_uri, "https://%s%s", u_node->host, uri); + + flb_plg_debug(ctx->ins, "created blob %s", full_uri); + + flb_http_client_destroy(c); + flb_upstream_conn_release(u_conn); + flb_sds_destroy(uri); + + return full_uri; + +create_blob_error: + if (c) { + flb_http_client_destroy(c); + } + if (u_conn) { + flb_upstream_conn_release(u_conn); + } + if (uri) { + flb_sds_destroy(uri); + } + return NULL; +} + +static flb_sds_t create_ingestion_message(struct flb_azure_kusto *ctx, flb_sds_t blob_uri, + size_t payload_size) +{ + flb_sds_t message; + char *uuid; + char *message_b64; + size_t b64_len; + + uuid = generate_uuid(); + if (!uuid) { + flb_plg_error(ctx->ins, "error generating unique ingestion UUID"); + return NULL; + } + + message = flb_sds_create(NULL); + if (!message) { + flb_plg_error(ctx->ins, "error creating ingestion message buffer"); + flb_free(uuid); + return NULL; + } + message = flb_sds_printf( + &message, + "{\"Id\": \"%s\", \"BlobPath\": \"%s\", \"RawDataSize\": %lu, \"DatabaseName\": " + "\"%s\", \"TableName\": \"%s\"," + "\"AdditionalProperties\": { \"format\": \"multijson\", " + "\"authorizationContext\": " + "\"%s\", \"jsonMappingReference\": \"%s\" }}", + uuid, blob_uri, payload_size, ctx->database_name, ctx->table_name, + ctx->resources->identity_token, + ctx->ingestion_mapping_reference == NULL ? "" : ctx->ingestion_mapping_reference); + + flb_plg_debug(ctx->ins, "created ingestion message:\n%s", message); + + message_b64 = base64_encode(message, flb_sds_len(message), &b64_len); + if (!message_b64) { + flb_plg_error(ctx->ins, "error encoding ingestion message to base64"); + flb_free(uuid); + flb_sds_destroy(message); + return NULL; + } + + flb_sds_len_set(message, 0); + message = flb_sds_printf(&message, + "%s", + message_b64); + + flb_free(uuid); + flb_free(message_b64); + + return message; +} + +static int azure_kusto_enqueue_ingestion(struct flb_azure_kusto *ctx, flb_sds_t blob_uri, + size_t payload_size) +{ + int ret; + char *queue_uri; + size_t queue_uri_size; + char *queue_sas; + size_t queue_sas_size; + struct flb_upstream_node *u_node; + struct flb_upstream_conn *u_conn; + struct flb_http_client *c; + flb_sds_t uri; + flb_sds_t payload; + size_t resp_size; + time_t now; + struct tm tm; + char tmp[64]; + int len; + + now = time(NULL); + gmtime_r(&now, &tm); + len = strftime(tmp, sizeof(tmp) - 1, "%a, %d %b %Y %H:%M:%S GMT", &tm); + + u_node = flb_upstream_ha_node_get(ctx->resources->queue_ha); + if (!u_node) { + flb_plg_error(ctx->ins, "error getting queue upstream"); + return -1; + } + + ret = flb_hash_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_URI, 3, + (void **)&queue_uri, &queue_uri_size); + if (ret == -1) { + flb_plg_error(ctx->ins, "error getting queue uri"); + return -1; + } + ret = flb_hash_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_SAS, 3, + (void **)&queue_sas, &queue_sas_size); + if (ret == -1) { + flb_plg_error(ctx->ins, "error getting queue sas token"); + return -1; + } + + u_conn = flb_upstream_conn_get(u_node->u); + if (!u_conn) { + flb_plg_error(ctx->ins, "error getting queue upstream connection"); + return -1; + } + + /* uri will be /messages? */ + uri = flb_sds_create_size(queue_uri_size + queue_sas_size + 11); + if (!uri) { + flb_plg_error(ctx->ins, "error creating queue container uri"); + goto enqueue_ingestion_error; + } + uri = flb_sds_printf(&uri, "%s/messages?%s", queue_uri, queue_sas); + + payload = create_ingestion_message(ctx, blob_uri, payload_size); + if (!payload) { + flb_plg_error(ctx->ins, "error creating payload buffer"); + goto enqueue_ingestion_error; + } + + c = flb_http_client(u_conn, FLB_HTTP_POST, uri, payload, flb_sds_len(payload), NULL, + 0, NULL, 0); + if (!c) { + flb_plg_error(ctx->ins, "cannot create HTTP client context for blob container"); + goto enqueue_ingestion_error; + } + + flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); + flb_http_add_header(c, "Content-Type", 12, "application/atom+xml", 20); + flb_http_add_header(c, "x-ms-date", 9, tmp, len); + flb_http_add_header(c, "x-ms-version", 12, "2019-12-12", 10); + + ret = flb_http_do(c, &resp_size); + flb_plg_debug(ctx->ins, "kusto queue request http_do=%i, HTTP Status: %i", ret, + c->resp.status); + if (ret != 0) { + flb_plg_error(ctx->ins, "cannot send HTTP request"); + goto enqueue_ingestion_error; + } + + /* Validate return status and HTTP status if set */ + if (c->resp.status != 201) { + if (c->resp.payload_size > 0) { + flb_plg_debug(ctx->ins, "Request failed and returned: \n%s", c->resp.payload); + } + else { + flb_plg_debug(ctx->ins, "Request failed"); + } + goto enqueue_ingestion_error; + } + + flb_upstream_conn_release(u_conn); + flb_http_client_destroy(c); + flb_sds_destroy(payload); + + return 0; + +enqueue_ingestion_error: + if (u_conn) { + flb_upstream_conn_release(u_conn); + } + if (c) { + flb_http_client_destroy(c); + } + if (payload) { + flb_sds_destroy(payload); + } + return -1; +} + +int azure_kusto_queued_ingestion(struct flb_azure_kusto *ctx, flb_sds_t tag, + size_t tag_len, flb_sds_t payload, size_t payload_size) +{ + int ret; + struct flb_time tm; + char *b64tag; + size_t b64_len; + uint64_t ms; + flb_sds_t blob_id; + flb_sds_t blob_uri; + + flb_time_get(&tm); + ms = ((tm.tm.tv_sec * 1000) + (tm.tm.tv_nsec / 1000000)); + + b64tag = base64_encode(tag, tag_len, &b64_len); + /* remove trailing '=' */ + while (b64_len && b64tag[b64_len - 1] == '=') { + b64tag[b64_len - 1] = '\0'; + b64_len--; + } + + /* flb________ */ + blob_id = flb_sds_create_size(flb_sds_len(ctx->database_name) + + flb_sds_len(ctx->table_name) + b64_len + 24); + if (!blob_id) { + flb_plg_error(ctx->ins, "cannot create blob id"); + goto queued_ingestion_error; + } + blob_id = flb_sds_printf(&blob_id, "flb__%s__%s__%s__%lu", ctx->database_name, + ctx->table_name, b64tag, ms); + + blob_uri = azure_kusto_create_blob(ctx, blob_id, payload, payload_size); + if (!blob_uri) { + flb_plg_error(ctx->ins, "failed to create payload blob"); + goto queued_ingestion_error; + } + + ret = azure_kusto_enqueue_ingestion(ctx, blob_uri, payload_size); + if (ret != 0) { + flb_plg_error(ctx->ins, "failed to enqueue ingestion blob to queue"); + goto queued_ingestion_error; + } + + /* Cleanup */ + + flb_free(b64tag); + flb_sds_destroy(blob_id); + flb_sds_destroy(blob_uri); + + return 0; + +queued_ingestion_error: + + if (b64tag) { + flb_free(b64tag); + } + if (blob_id) { + flb_sds_destroy(blob_id); + } + if (blob_uri) { + flb_sds_destroy(blob_uri); + } + + return -1; +} \ No newline at end of file diff --git a/plugins/out_azure_kusto/azure_kusto_ingest.h b/plugins/out_azure_kusto/azure_kusto_ingest.h new file mode 100644 index 00000000000..60613919a99 --- /dev/null +++ b/plugins/out_azure_kusto/azure_kusto_ingest.h @@ -0,0 +1,28 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_OUT_AZURE_KUSTO_INGEST_H +#define FLB_OUT_AZURE_KUSTO_INGEST_H + +#include "azure_kusto.h" + +int azure_kusto_queued_ingestion(struct flb_azure_kusto *ctx, flb_sds_t tag, + size_t tag_len, flb_sds_t payload, size_t payload_size); + +#endif \ No newline at end of file From 2c0610d726bfb1e02dcdec1e237d2a8b829fe9d9 Mon Sep 17 00:00:00 2001 From: Gershon Papiashvili Date: Mon, 27 Jun 2022 03:30:56 +0300 Subject: [PATCH 02/18] flb_upstream_node: fix tls free before upstream destroy Signed-off-by: Gershon Papiashvili --- src/flb_upstream_node.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/flb_upstream_node.c b/src/flb_upstream_node.c index 747c6f0fe95..956ddd04439 100644 --- a/src/flb_upstream_node.c +++ b/src/flb_upstream_node.c @@ -189,6 +189,11 @@ void flb_upstream_node_destroy(struct flb_upstream_node *node) flb_sds_destroy(node->host); flb_sds_destroy(node->port); + flb_hash_table_destroy(node->ht); + if (node->u) { + flb_upstream_destroy(node->u); + } + #ifdef FLB_HAVE_TLS flb_sds_destroy(node->tls_ca_path); flb_sds_destroy(node->tls_ca_file); @@ -200,11 +205,6 @@ void flb_upstream_node_destroy(struct flb_upstream_node *node) } #endif - flb_hash_table_destroy(node->ht); - if (node->u) { - flb_upstream_destroy(node->u); - } - /* note: node link must be handled by the caller before this call */ flb_free(node); } From 9f1fbcb479d90133c3a814d94784326999c114bc Mon Sep 17 00:00:00 2001 From: Gershon Papiashvili Date: Mon, 27 Jun 2022 11:02:18 +0300 Subject: [PATCH 03/18] out_azure_kusto: fix build Signed-off-by: Gershon Papiashvili --- plugins/out_azure_kusto/azure_kusto_conf.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/plugins/out_azure_kusto/azure_kusto_conf.c b/plugins/out_azure_kusto/azure_kusto_conf.c index ca4931461af..c2de039d64c 100644 --- a/plugins/out_azure_kusto/azure_kusto_conf.c +++ b/plugins/out_azure_kusto/azure_kusto_conf.c @@ -28,10 +28,6 @@ #include #include -#include -#include -#include - #include "azure_kusto.h" #include "azure_kusto_conf.h" From 96ad45ea33232945f92e2e5e60f9114f4ae18295 Mon Sep 17 00:00:00 2001 From: Gershon Papiashvili Date: Fri, 8 Jul 2022 17:31:37 +0300 Subject: [PATCH 04/18] out_azure_kusto: normalize flb_oauth2_payload_append constant lengths Signed-off-by: Gershon Papiashvili --- plugins/out_azure_kusto/azure_kusto.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index 38ca6f5d1e9..dda144b517a 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -43,19 +43,19 @@ static int azure_kusto_get_oauth2_token(struct flb_azure_kusto *ctx) return -1; } - ret = flb_oauth2_payload_append(ctx->o, "scope", 5, FLB_AZURE_KUSTO_SCOPE, -1); + ret = flb_oauth2_payload_append(ctx->o, "scope", 5, FLB_AZURE_KUSTO_SCOPE, 39); if (ret == -1) { flb_plg_error(ctx->ins, "error appending oauth2 params"); return -1; } - ret = flb_oauth2_payload_append(ctx->o, "client_id", -1, ctx->client_id, -1); + ret = flb_oauth2_payload_append(ctx->o, "client_id", 9, ctx->client_id, -1); if (ret == -1) { flb_plg_error(ctx->ins, "error appending oauth2 params"); return -1; } - ret = flb_oauth2_payload_append(ctx->o, "client_secret", -1, ctx->client_secret, -1); + ret = flb_oauth2_payload_append(ctx->o, "client_secret", 13, ctx->client_secret, -1); if (ret == -1) { flb_plg_error(ctx->ins, "error appending oauth2 params"); return -1; From 0197153fd2e71fb5690338c2e67705ac7e13b62f Mon Sep 17 00:00:00 2001 From: Gershon Papiashvili Date: Fri, 8 Jul 2022 17:37:06 +0300 Subject: [PATCH 05/18] out_azure_kusto: check token sds create succeed Signed-off-by: Gershon Papiashvili --- plugins/out_azure_kusto/azure_kusto.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index dda144b517a..b8d08ef900b 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -88,6 +88,10 @@ flb_sds_t get_azure_kusto_token(struct flb_azure_kusto *ctx) /* Copy string to prevent race conditions (get_oauth2 can free the string) */ if (ret == 0) { output = flb_sds_create(ctx->o->token_type); + if (!output) { + flb_plg_error(ctx->ins, "error creating token"); + return NULL; + } flb_sds_printf(&output, " %s", ctx->o->access_token); } From cd8a21cdc2422d3ab34649777527238870cd7c48 Mon Sep 17 00:00:00 2001 From: Gershon Papiashvili Date: Fri, 8 Jul 2022 17:46:37 +0300 Subject: [PATCH 06/18] out_azure_kusto: remove blocking checks on plugin init Signed-off-by: Gershon Papiashvili --- plugins/out_azure_kusto/azure_kusto.c | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index b8d08ef900b..ad4e86ab498 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -217,9 +217,7 @@ flb_sds_t execute_ingest_csl_command(struct flb_azure_kusto *ctx, const char *cs static int cb_azure_kusto_init(struct flb_output_instance *ins, struct flb_config *config, void *data) { - char *token; int io_flags = FLB_IO_TLS; - int ret; struct flb_azure_kusto *ctx; /* Create config context */ @@ -260,22 +258,6 @@ static int cb_azure_kusto_init(struct flb_output_instance *ins, struct flb_confi } flb_output_upstream_set(ctx->u, ins); - /* Get or renew the OAuth2 token */ - token = get_azure_kusto_token(ctx); - - if (!token) { - flb_plg_warn(ctx->ins, "token retrieval failed"); - } - else { - flb_sds_destroy(token); - - /* load or refresh ingestion resources */ - ret = azure_kusto_load_ingestion_resources(ctx, config); - if (ret != 0) { - flb_plg_warn(ctx->ins, "ingestion resources loading failed"); - } - } - return 0; } From 04a5cf24b2dc67d2444deced9183b335a9672045 Mon Sep 17 00:00:00 2001 From: Gershon Papiashvili Date: Fri, 8 Jul 2022 17:52:29 +0300 Subject: [PATCH 07/18] out_azure_kusto: debug on msgpack parsing issues Signed-off-by: Gershon Papiashvili --- plugins/out_azure_kusto/azure_kusto.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index ad4e86ab498..d1058ea69fe 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -301,9 +301,12 @@ static int azure_kusto_format(struct flb_azure_kusto *ctx, const char *tag, int root = result.data; /* Each array must have two entries: time and record */ if (root.type != MSGPACK_OBJECT_ARRAY) { + flb_plg_debug(ctx->ins, "unexpected msgpack object type: %d", root.type); continue; } if (root.via.array.size != 2) { + flb_plg_debug(ctx->ins, "unexpected msgpack array size: %d", + root.via.array.size); continue; } From dae7717cf52a59d0abf0df94715b22b837e91836 Mon Sep 17 00:00:00 2001 From: Gershon Papiashvili Date: Fri, 8 Jul 2022 18:19:12 +0300 Subject: [PATCH 08/18] out_azure_kusto: avoid goto when possible Signed-off-by: Gershon Papiashvili --- plugins/out_azure_kusto/azure_kusto.c | 56 +++++----- plugins/out_azure_kusto/azure_kusto_conf.c | 44 ++++---- plugins/out_azure_kusto/azure_kusto_ingest.c | 102 +++++++++---------- 3 files changed, 95 insertions(+), 107 deletions(-) diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index d1058ea69fe..7607b0aba11 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -133,7 +133,8 @@ flb_sds_t execute_ingest_csl_command(struct flb_azure_kusto *ctx, const char *cs token = get_azure_kusto_token(ctx); if (!token) { flb_plg_error(ctx->ins, "cannot retrieve oauth2 token"); - goto execute_error; + flb_upstream_conn_release(u_conn); + return NULL; } /* Compose request body */ @@ -141,7 +142,9 @@ flb_sds_t execute_ingest_csl_command(struct flb_azure_kusto *ctx, const char *cs flb_sds_create_size(sizeof(FLB_AZURE_KUSTO_MGMT_BODY_TEMPLATE) - 2 + strlen(csl)); if (!body) { flb_plg_error(ctx->ins, "cannot construct request body"); - goto execute_error; + flb_upstream_conn_release(u_conn); + flb_sds_destroy(token); + return NULL; } body = flb_sds_printf(&body, FLB_AZURE_KUSTO_MGMT_BODY_TEMPLATE, csl); @@ -151,7 +154,10 @@ flb_sds_t execute_ingest_csl_command(struct flb_azure_kusto *ctx, const char *cs flb_sds_len(body), NULL, 0, NULL, 0); if (!c) { flb_plg_error(ctx->ins, "cannot create HTTP client context"); - goto execute_error; + flb_upstream_conn_release(u_conn); + flb_sds_destroy(token); + flb_sds_destroy(body); + return NULL; } /* Add headers */ @@ -169,7 +175,11 @@ flb_sds_t execute_ingest_csl_command(struct flb_azure_kusto *ctx, const char *cs if (ret != 0) { flb_plg_error(ctx->ins, "cannot send HTTP request"); - goto execute_error; + flb_upstream_conn_release(u_conn); + flb_sds_destroy(token); + flb_sds_destroy(body); + flb_http_client_destroy(c); + return NULL; } /* Validate return status and HTTP status if set */ @@ -180,7 +190,11 @@ flb_sds_t execute_ingest_csl_command(struct flb_azure_kusto *ctx, const char *cs else { flb_plg_debug(ctx->ins, "Request failed"); } - goto execute_error; + flb_upstream_conn_release(u_conn); + flb_sds_destroy(token); + flb_sds_destroy(body); + flb_http_client_destroy(c); + return NULL; } ret = c->resp.payload_size; @@ -193,25 +207,6 @@ flb_sds_t execute_ingest_csl_command(struct flb_azure_kusto *ctx, const char *cs flb_sds_destroy(body); return resp; - -execute_error: - if (c) { - flb_http_client_destroy(c); - } - - if (u_conn) { - flb_upstream_conn_release(u_conn); - } - - if (token) { - flb_sds_destroy(token); - } - - if (body) { - flb_sds_destroy(body); - } - - return NULL; } static int cb_azure_kusto_init(struct flb_output_instance *ins, struct flb_config *config, @@ -395,7 +390,7 @@ static void cb_azure_kusto_flush(struct flb_event_chunk *event_chunk, ret = azure_kusto_load_ingestion_resources(ctx, config); if (ret != 0) { flb_plg_error(ctx->ins, "cannot load ingestion resources"); - goto flush_error; + FLB_OUTPUT_RETURN(FLB_RETRY); } /* Reformat msgpack to JSON payload */ @@ -403,13 +398,14 @@ static void cb_azure_kusto_flush(struct flb_event_chunk *event_chunk, event_chunk->size, (void **)&json, &json_size); if (ret != 0) { flb_plg_error(ctx->ins, "cannot reformat data into json"); - goto flush_error; + FLB_OUTPUT_RETURN(FLB_RETRY); } ret = azure_kusto_queued_ingestion(ctx, event_chunk->tag, tag_len, json, json_size); if (ret != 0) { flb_plg_error(ctx->ins, "cannot perform queued ingestion"); - goto flush_error; + flb_sds_destroy(json); + FLB_OUTPUT_RETURN(FLB_RETRY); } /* Cleanup */ @@ -417,12 +413,6 @@ static void cb_azure_kusto_flush(struct flb_event_chunk *event_chunk, /* Done */ FLB_OUTPUT_RETURN(FLB_OK); - -flush_error: - if (json) { - flb_sds_destroy(json); - } - FLB_OUTPUT_RETURN(FLB_RETRY); } static int cb_azure_kusto_exit(void *data, struct flb_config *config) diff --git a/plugins/out_azure_kusto/azure_kusto_conf.c b/plugins/out_azure_kusto/azure_kusto_conf.c index c2de039d64c..7674759115a 100644 --- a/plugins/out_azure_kusto/azure_kusto_conf.c +++ b/plugins/out_azure_kusto/azure_kusto_conf.c @@ -41,7 +41,7 @@ static struct flb_upstream_node *flb_upstream_node_create_url(struct flb_azure_k char *port = NULL; char *uri = NULL; char *tmp; - struct flb_hash *kv; + struct flb_hash *kv = NULL; struct flb_upstream_node *node = NULL; int uri_length; int sas_length; @@ -171,7 +171,7 @@ static int parse_storage_resources(struct flb_azure_kusto *ctx, struct flb_confi int resource_type; struct flb_upstream_node *node; struct flb_upstream_ha *ha; - flb_sds_t resource_uri = flb_sds_create(NULL); + flb_sds_t resource_uri; /* Response is a json in the form of * { @@ -192,17 +192,26 @@ static int parse_storage_resources(struct flb_azure_kusto *ctx, struct flb_confi * } */ + resource_uri = flb_sds_create(NULL); + if (!resource_uri) { + flb_plg_error(ctx->ins, "error allocating resource uri buffer"); + return -1; + } + jsmn_init(&parser); tokens = flb_calloc(1, sizeof(jsmntok_t) * tok_size); if (!tokens) { flb_plg_error(ctx->ins, "error allocating tokens"); - goto load_storage_resources_error; + flb_sds_destroy(resource_uri); + return -1; } ret = jsmn_parse(&parser, response, flb_sds_len(response), tokens, tok_size); if (ret <= 0) { flb_plg_error(ctx->ins, "error parsing JSON response: %s", response); - goto load_storage_resources_error; + flb_sds_destroy(resource_uri); + flb_free(tokens); + return -1; } /* skip all tokens until we reach "Rows" */ @@ -291,13 +300,17 @@ static int parse_storage_resources(struct flb_azure_kusto *ctx, struct flb_confi if (!ha) { flb_plg_error(ctx->ins, "error creating HA upstream"); - goto load_storage_resources_error; + flb_sds_destroy(resource_uri); + flb_free(tokens); + return -1; } node = flb_upstream_node_create_url(ctx, config, resource_uri); if (!node) { flb_plg_error(ctx->ins, "error creating HA upstream node"); - goto load_storage_resources_error; + flb_sds_destroy(resource_uri); + flb_free(tokens); + return -1; } flb_upstream_ha_node_add(ha, node); @@ -305,7 +318,9 @@ static int parse_storage_resources(struct flb_azure_kusto *ctx, struct flb_confi if (!queue_count || !blob_count) { flb_plg_error(ctx->ins, "error parsing resources: missing resources"); - goto load_storage_resources_error; + flb_sds_destroy(resource_uri); + flb_free(tokens); + return -1; } flb_sds_destroy(resource_uri); @@ -315,18 +330,6 @@ static int parse_storage_resources(struct flb_azure_kusto *ctx, struct flb_confi queue_count); return 0; - -load_storage_resources_error: - - if (tokens) { - flb_free(tokens); - } - - if (resource_uri) { - flb_sds_destroy(resource_uri); - } - - return -1; } /** @@ -408,7 +411,7 @@ int azure_kusto_load_ingestion_resources(struct flb_azure_kusto *ctx, struct flb_config *config) { int ret; - flb_sds_t response; + flb_sds_t response = NULL; flb_sds_t identity_token = NULL; struct flb_upstream_ha *blob_ha = NULL; struct flb_upstream_ha *queue_ha = NULL; @@ -456,6 +459,7 @@ int azure_kusto_load_ingestion_resources(struct flb_azure_kusto *ctx, } flb_sds_destroy(response); + response = NULL; response = execute_ingest_csl_command(ctx, ".get kusto identity token"); if (!response) { diff --git a/plugins/out_azure_kusto/azure_kusto_ingest.c b/plugins/out_azure_kusto/azure_kusto_ingest.c index db5b3695834..7b488a4cb1b 100644 --- a/plugins/out_azure_kusto/azure_kusto_ingest.c +++ b/plugins/out_azure_kusto/azure_kusto_ingest.c @@ -132,7 +132,8 @@ static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t uri = flb_sds_create_size(blob_uri_size + blob_sas_size + flb_sds_len(blob_id) + 13); if (!uri) { flb_plg_error(ctx->ins, "error creating blob container uri"); - goto create_blob_error; + flb_upstream_conn_release(u_conn); + return NULL; } uri = flb_sds_printf(&uri, "%s/%s.multijson?%s", blob_uri, blob_id, blob_sas); @@ -142,7 +143,9 @@ static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t 0); if (!c) { flb_plg_error(ctx->ins, "cannot create HTTP client context for blob container"); - goto create_blob_error; + flb_upstream_conn_release(u_conn); + flb_sds_destroy(uri); + return NULL; } flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); @@ -156,7 +159,10 @@ static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t c->resp.status); if (ret != 0) { flb_plg_error(ctx->ins, "cannot send HTTP request"); - goto create_blob_error; + flb_upstream_conn_release(u_conn); + flb_sds_destroy(uri); + flb_http_client_destroy(c); + return NULL; } /* Validate return status and HTTP status if set */ @@ -167,36 +173,30 @@ static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t else { flb_plg_debug(ctx->ins, "Request failed"); } - goto create_blob_error; + flb_upstream_conn_release(u_conn); + flb_sds_destroy(uri); + flb_http_client_destroy(c); + return NULL; } /* generate full blob uri & return it */ full_uri = flb_sds_create_size(9 + flb_sds_len(u_node->host) + flb_sds_len(uri)); if (!full_uri) { flb_plg_error(ctx->ins, "cannot create full blob uri"); - goto create_blob_error; + flb_upstream_conn_release(u_conn); + flb_sds_destroy(uri); + flb_http_client_destroy(c); + return NULL; } full_uri = flb_sds_printf(&full_uri, "https://%s%s", u_node->host, uri); flb_plg_debug(ctx->ins, "created blob %s", full_uri); - flb_http_client_destroy(c); flb_upstream_conn_release(u_conn); flb_sds_destroy(uri); + flb_http_client_destroy(c); return full_uri; - -create_blob_error: - if (c) { - flb_http_client_destroy(c); - } - if (u_conn) { - flb_upstream_conn_release(u_conn); - } - if (uri) { - flb_sds_destroy(uri); - } - return NULL; } static flb_sds_t create_ingestion_message(struct flb_azure_kusto *ctx, flb_sds_t blob_uri, @@ -303,21 +303,27 @@ static int azure_kusto_enqueue_ingestion(struct flb_azure_kusto *ctx, flb_sds_t uri = flb_sds_create_size(queue_uri_size + queue_sas_size + 11); if (!uri) { flb_plg_error(ctx->ins, "error creating queue container uri"); - goto enqueue_ingestion_error; + flb_upstream_conn_release(u_conn); + return -1; } uri = flb_sds_printf(&uri, "%s/messages?%s", queue_uri, queue_sas); payload = create_ingestion_message(ctx, blob_uri, payload_size); if (!payload) { flb_plg_error(ctx->ins, "error creating payload buffer"); - goto enqueue_ingestion_error; + flb_upstream_conn_release(u_conn); + flb_sds_destroy(uri); + return -1; } c = flb_http_client(u_conn, FLB_HTTP_POST, uri, payload, flb_sds_len(payload), NULL, 0, NULL, 0); if (!c) { flb_plg_error(ctx->ins, "cannot create HTTP client context for blob container"); - goto enqueue_ingestion_error; + flb_upstream_conn_release(u_conn); + flb_sds_destroy(uri); + flb_sds_destroy(payload); + return -1; } flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); @@ -330,7 +336,11 @@ static int azure_kusto_enqueue_ingestion(struct flb_azure_kusto *ctx, flb_sds_t c->resp.status); if (ret != 0) { flb_plg_error(ctx->ins, "cannot send HTTP request"); - goto enqueue_ingestion_error; + flb_upstream_conn_release(u_conn); + flb_sds_destroy(uri); + flb_sds_destroy(payload); + flb_http_client_destroy(c); + return -1; } /* Validate return status and HTTP status if set */ @@ -341,26 +351,19 @@ static int azure_kusto_enqueue_ingestion(struct flb_azure_kusto *ctx, flb_sds_t else { flb_plg_debug(ctx->ins, "Request failed"); } - goto enqueue_ingestion_error; + flb_upstream_conn_release(u_conn); + flb_sds_destroy(uri); + flb_sds_destroy(payload); + flb_http_client_destroy(c); + return -1; } flb_upstream_conn_release(u_conn); - flb_http_client_destroy(c); + flb_sds_destroy(uri); flb_sds_destroy(payload); + flb_http_client_destroy(c); return 0; - -enqueue_ingestion_error: - if (u_conn) { - flb_upstream_conn_release(u_conn); - } - if (c) { - flb_http_client_destroy(c); - } - if (payload) { - flb_sds_destroy(payload); - } - return -1; } int azure_kusto_queued_ingestion(struct flb_azure_kusto *ctx, flb_sds_t tag, @@ -389,7 +392,8 @@ int azure_kusto_queued_ingestion(struct flb_azure_kusto *ctx, flb_sds_t tag, flb_sds_len(ctx->table_name) + b64_len + 24); if (!blob_id) { flb_plg_error(ctx->ins, "cannot create blob id"); - goto queued_ingestion_error; + flb_free(b64tag); + return -1; } blob_id = flb_sds_printf(&blob_id, "flb__%s__%s__%s__%lu", ctx->database_name, ctx->table_name, b64tag, ms); @@ -397,34 +401,24 @@ int azure_kusto_queued_ingestion(struct flb_azure_kusto *ctx, flb_sds_t tag, blob_uri = azure_kusto_create_blob(ctx, blob_id, payload, payload_size); if (!blob_uri) { flb_plg_error(ctx->ins, "failed to create payload blob"); - goto queued_ingestion_error; + flb_free(b64tag); + flb_sds_destroy(blob_id); + return -1; } ret = azure_kusto_enqueue_ingestion(ctx, blob_uri, payload_size); if (ret != 0) { flb_plg_error(ctx->ins, "failed to enqueue ingestion blob to queue"); - goto queued_ingestion_error; + flb_free(b64tag); + flb_sds_destroy(blob_id); + flb_sds_destroy(blob_uri); + return -1; } /* Cleanup */ - flb_free(b64tag); flb_sds_destroy(blob_id); flb_sds_destroy(blob_uri); return 0; - -queued_ingestion_error: - - if (b64tag) { - flb_free(b64tag); - } - if (blob_id) { - flb_sds_destroy(blob_id); - } - if (blob_uri) { - flb_sds_destroy(blob_uri); - } - - return -1; } \ No newline at end of file From 8e11d993f6aa777507d27adc73183ab5d798c179 Mon Sep 17 00:00:00 2001 From: Gershon Papiashvili Date: Fri, 8 Jul 2022 18:27:59 +0300 Subject: [PATCH 09/18] out_azure_kusto: fix uuid malloc Signed-off-by: Gershon Papiashvili --- plugins/out_azure_kusto/azure_kusto_ingest.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/out_azure_kusto/azure_kusto_ingest.c b/plugins/out_azure_kusto/azure_kusto_ingest.c index 7b488a4cb1b..5a3ebff91f0 100644 --- a/plugins/out_azure_kusto/azure_kusto_ingest.c +++ b/plugins/out_azure_kusto/azure_kusto_ingest.c @@ -37,7 +37,7 @@ static char *generate_uuid() int i; uint64_t rand; - uuid = flb_malloc(36); + uuid = flb_malloc(37); if (!uuid) { flb_errno(); return NULL; @@ -54,6 +54,7 @@ static char *generate_uuid() } uuid[i] = chars[rand % 16]; } + uuid[36] = '\0'; return uuid; } From 20c498a6144afc9cbde0a01aea6a3f902befa8a1 Mon Sep 17 00:00:00 2001 From: Gershon Papiashvili Date: Fri, 8 Jul 2022 18:43:56 +0300 Subject: [PATCH 10/18] out_azure_kusto: better msgpack inits and cleanups Signed-off-by: Gershon Papiashvili --- plugins/out_azure_kusto/azure_kusto.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index 7607b0aba11..c75bf6ea0bd 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -278,16 +278,17 @@ static int azure_kusto_format(struct flb_azure_kusto *ctx, const char *tag, int /* output buffer */ flb_sds_t out_buf; - /* Create temporary msgpack buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - /* Create array for all records */ records = flb_mp_count(data, bytes); if (records <= 0) { - msgpack_sbuffer_destroy(&mp_sbuf); + flb_plg_error(ctx->ins, "error counting msgpack entries"); return -1; } + + /* Create temporary msgpack buffer */ + msgpack_sbuffer_init(&mp_sbuf); + msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); + msgpack_pack_array(&mp_pck, records); off = 0; @@ -351,20 +352,19 @@ static int azure_kusto_format(struct flb_azure_kusto *ctx, const char *tag, int /* Convert from msgpack to JSON */ out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + + /* Cleanup */ msgpack_sbuffer_destroy(&mp_sbuf); + msgpack_unpacked_destroy(&result); if (!out_buf) { flb_plg_error(ctx->ins, "error formatting JSON payload"); - msgpack_unpacked_destroy(&result); return -1; } *out_data = out_buf; *out_size = flb_sds_len(out_buf); - /* Cleanup */ - msgpack_unpacked_destroy(&result); - return 0; } From c86b9342b42eef523ee29641a6ba05493c59f3f5 Mon Sep 17 00:00:00 2001 From: Gershon Papiashvili Date: Fri, 8 Jul 2022 18:44:37 +0300 Subject: [PATCH 11/18] out_azure_kusto: move casting to after declarations Signed-off-by: Gershon Papiashvili --- plugins/out_azure_kusto/azure_kusto.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index c75bf6ea0bd..f3e909b8f3b 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -373,15 +373,15 @@ static void cb_azure_kusto_flush(struct flb_event_chunk *event_chunk, struct flb_input_instance *i_ins, void *out_context, struct flb_config *config) { - (void)i_ins; - (void)config; int ret; flb_sds_t json; size_t json_size; size_t tag_len; - struct flb_azure_kusto *ctx = out_context; + (void)i_ins; + (void)config; + flb_plg_trace(ctx->ins, "flushing bytes %zu", event_chunk->size); tag_len = flb_sds_len(event_chunk->tag); From 423f6ba104a4643612f6e9dea6312b55ffc57cb7 Mon Sep 17 00:00:00 2001 From: Gershon Papiashvili Date: Fri, 8 Jul 2022 19:01:46 +0300 Subject: [PATCH 12/18] out_azure_kusto: use flb_base64_encode Signed-off-by: Gershon Papiashvili --- plugins/out_azure_kusto/azure_kusto_ingest.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/out_azure_kusto/azure_kusto_ingest.c b/plugins/out_azure_kusto/azure_kusto_ingest.c index 5a3ebff91f0..384011115f8 100644 --- a/plugins/out_azure_kusto/azure_kusto_ingest.c +++ b/plugins/out_azure_kusto/azure_kusto_ingest.c @@ -17,6 +17,7 @@ * limitations under the License. */ +#include #include #include #include @@ -63,15 +64,16 @@ static char *base64_encode(flb_sds_t s, size_t len, size_t *out_len) { char *b64; int ret; + size_t buffer_len = 4 * ceil(((double)len / 3) + 1); - *out_len = (4 * ceil(((double)len / 3) + 1)); - b64 = flb_malloc(*out_len); + b64 = flb_malloc(buffer_len); if (!b64) { flb_errno(); return NULL; } - ret = mbedtls_base64_encode((unsigned char *)b64, *out_len, out_len, - (unsigned char *)s, len); + + ret = flb_base64_encode((unsigned char *)b64, buffer_len, out_len, (unsigned char *)s, + len); if (ret != 0) { flb_error("cannot encode string %s into base64", s); flb_free(b64); From 44beae0eedfc4acf44955f889bbce3dbbf669e24 Mon Sep 17 00:00:00 2001 From: Gershon Papiashvili Date: Fri, 8 Jul 2022 19:12:46 +0300 Subject: [PATCH 13/18] out_azure_kusto: remove goto in flb_upstream_node_create_url Signed-off-by: Gershon Papiashvili --- plugins/out_azure_kusto/azure_kusto_conf.c | 53 ++++++++++++---------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/plugins/out_azure_kusto/azure_kusto_conf.c b/plugins/out_azure_kusto/azure_kusto_conf.c index 7674759115a..ca8ec1525e1 100644 --- a/plugins/out_azure_kusto/azure_kusto_conf.c +++ b/plugins/out_azure_kusto/azure_kusto_conf.c @@ -50,14 +50,18 @@ static struct flb_upstream_node *flb_upstream_node_create_url(struct flb_azure_k ret = flb_utils_url_split(url, &prot, &host, &port, &uri); if (ret == -1) { flb_plg_error(ctx->ins, "invalid URL: %s", url); - goto upstream_node_create_url_out; + return NULL; } /* find sas token in query */ tmp = strchr(uri, '?'); if (!tmp) { flb_plg_error(ctx->ins, "uri has no sas token query: %s", uri); - goto upstream_node_create_url_out; + flb_free(prot); + flb_free(host); + flb_free(port); + flb_free(uri); + return NULL; } uri_length = tmp - uri; sas_length = strnlen(tmp + 1, 256); @@ -66,19 +70,33 @@ static struct flb_upstream_node *flb_upstream_node_create_url(struct flb_azure_k kv = flb_hash_create(FLB_HASH_EVICT_NONE, 2, 2); if (!kv) { flb_plg_error(ctx->ins, "error creating upstream node hash table"); - goto upstream_node_create_url_out; + flb_free(prot); + flb_free(host); + flb_free(port); + flb_free(uri); + return NULL; } ret = flb_hash_add(kv, AZURE_KUSTO_RESOURCE_UPSTREAM_URI, 3, uri, uri_length); if (ret == -1) { flb_plg_error(ctx->ins, "error storing resource uri"); - goto upstream_node_create_url_out; + flb_free(prot); + flb_free(host); + flb_free(port); + flb_free(uri); + flb_hash_destroy(kv); + return NULL; } ret = flb_hash_add(kv, AZURE_KUSTO_RESOURCE_UPSTREAM_SAS, 3, tmp + 1, sas_length); if (ret == -1) { flb_plg_error(ctx->ins, "error storing resource sas token"); - goto upstream_node_create_url_out; + flb_free(prot); + flb_free(host); + flb_free(port); + flb_free(uri); + flb_hash_destroy(kv); + return NULL; } node = flb_upstream_node_create(NULL, host, port, FLB_TRUE, ctx->ins->tls->verify, @@ -86,7 +104,12 @@ static struct flb_upstream_node *flb_upstream_node_create_url(struct flb_azure_k NULL, NULL, NULL, NULL, kv, config); if (!node) { flb_plg_error(ctx->ins, "error creating resource upstream node"); - goto upstream_node_create_url_out; + flb_free(prot); + flb_free(host); + flb_free(port); + flb_free(uri); + flb_hash_destroy(kv); + return NULL; } flb_free(prot); @@ -95,24 +118,6 @@ static struct flb_upstream_node *flb_upstream_node_create_url(struct flb_azure_k flb_free(uri); return node; - -upstream_node_create_url_out: - if (prot) { - flb_free(prot); - } - if (host) { - flb_free(host); - } - if (port) { - flb_free(port); - } - if (uri) { - flb_free(uri); - } - if (kv) { - flb_hash_destroy(kv); - } - return node; } static int flb_azure_kusto_resources_clear(struct flb_azure_kusto_resources *resources) From 70a391189bd4382992840b48443b2b3dcda65cb2 Mon Sep 17 00:00:00 2001 From: Gershon Papiashvili Date: Mon, 11 Jul 2022 19:56:39 +0300 Subject: [PATCH 14/18] out_azure_kusto: change defaults Signed-off-by: Gershon Papiashvili --- plugins/out_azure_kusto/azure_kusto.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index f3e909b8f3b..d511cbc5d52 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -457,7 +457,7 @@ static struct flb_config_map config_map[] = { "Set the ingestion mapping reference"}, {FLB_CONFIG_MAP_STR, "log_key", FLB_AZURE_KUSTO_DEFAULT_LOG_KEY, 0, FLB_TRUE, offsetof(struct flb_azure_kusto, log_key), "The key name of event payload"}, - {FLB_CONFIG_MAP_BOOL, "include_tag_key", "false", 0, FLB_TRUE, + {FLB_CONFIG_MAP_BOOL, "include_tag_key", "true", 0, FLB_TRUE, offsetof(struct flb_azure_kusto, include_tag_key), "If enabled, tag is appended to output. " "The key name is used 'tag_key' property."}, @@ -465,7 +465,7 @@ static struct flb_config_map config_map[] = { offsetof(struct flb_azure_kusto, tag_key), "The key name of tag. If 'include_tag_key' is false, " "This property is ignored"}, - {FLB_CONFIG_MAP_BOOL, "include_time_key", "false", 0, FLB_TRUE, + {FLB_CONFIG_MAP_BOOL, "include_time_key", "true", 0, FLB_TRUE, offsetof(struct flb_azure_kusto, include_time_key), "If enabled, time is appended to output. " "The key name is used 'time_key' property."}, From cbac86b9428ed1822c8238b6373944109d7f3a64 Mon Sep 17 00:00:00 2001 From: Gershon Papiashvili Date: Fri, 15 Jul 2022 15:06:48 +0300 Subject: [PATCH 15/18] out_azure_kusto: use snprintf Signed-off-by: Gershon Papiashvili --- plugins/out_azure_kusto/azure_kusto.c | 13 +++--- plugins/out_azure_kusto/azure_kusto_conf.c | 6 +-- plugins/out_azure_kusto/azure_kusto_ingest.c | 47 +++++++++++++------- 3 files changed, 41 insertions(+), 25 deletions(-) diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index d511cbc5d52..09a249648d5 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -87,12 +87,14 @@ flb_sds_t get_azure_kusto_token(struct flb_azure_kusto *ctx) /* Copy string to prevent race conditions (get_oauth2 can free the string) */ if (ret == 0) { - output = flb_sds_create(ctx->o->token_type); + output = flb_sds_create_size(flb_sds_len(ctx->o->token_type) + + flb_sds_len(ctx->o->access_token) + 2); if (!output) { - flb_plg_error(ctx->ins, "error creating token"); + flb_plg_error(ctx->ins, "error creating token buffer"); return NULL; } - flb_sds_printf(&output, " %s", ctx->o->access_token); + flb_sds_snprintf(&output, flb_sds_alloc(output), "%s %s", ctx->o->token_type, + ctx->o->access_token); } if (pthread_mutex_unlock(&ctx->token_mutex)) { @@ -139,15 +141,14 @@ flb_sds_t execute_ingest_csl_command(struct flb_azure_kusto *ctx, const char *cs /* Compose request body */ body = - flb_sds_create_size(sizeof(FLB_AZURE_KUSTO_MGMT_BODY_TEMPLATE) - 2 + strlen(csl)); + flb_sds_create_size(sizeof(FLB_AZURE_KUSTO_MGMT_BODY_TEMPLATE) - 1 + strlen(csl)); if (!body) { flb_plg_error(ctx->ins, "cannot construct request body"); flb_upstream_conn_release(u_conn); flb_sds_destroy(token); return NULL; } - - body = flb_sds_printf(&body, FLB_AZURE_KUSTO_MGMT_BODY_TEMPLATE, csl); + flb_sds_snprintf(&body, flb_sds_alloc(body), FLB_AZURE_KUSTO_MGMT_BODY_TEMPLATE, csl); /* Compose HTTP Client request */ c = flb_http_client(u_conn, FLB_HTTP_POST, FLB_AZURE_KUSTO_MGMT_URI_PATH, body, diff --git a/plugins/out_azure_kusto/azure_kusto_conf.c b/plugins/out_azure_kusto/azure_kusto_conf.c index ca8ec1525e1..4a25301225f 100644 --- a/plugins/out_azure_kusto/azure_kusto_conf.c +++ b/plugins/out_azure_kusto/azure_kusto_conf.c @@ -594,15 +594,15 @@ struct flb_azure_kusto *flb_azure_kusto_conf_create(struct flb_output_instance * } /* Create the auth URL */ - ctx->oauth_url = flb_sds_create_size(sizeof(FLB_MSAL_AUTH_URL_TEMPLATE) - 2 + + ctx->oauth_url = flb_sds_create_size(sizeof(FLB_MSAL_AUTH_URL_TEMPLATE) - 1 + flb_sds_len(ctx->tenant_id)); if (!ctx->oauth_url) { flb_errno(); flb_azure_kusto_conf_destroy(ctx); return NULL; } - ctx->oauth_url = - flb_sds_printf(&ctx->oauth_url, FLB_MSAL_AUTH_URL_TEMPLATE, ctx->tenant_id); + flb_sds_snprintf(&ctx->oauth_url, flb_sds_alloc(ctx->oauth_url), + FLB_MSAL_AUTH_URL_TEMPLATE, ctx->tenant_id); ctx->resources = flb_calloc(1, sizeof(struct flb_azure_kusto_resources)); if (!ctx->resources) { diff --git a/plugins/out_azure_kusto/azure_kusto_ingest.c b/plugins/out_azure_kusto/azure_kusto_ingest.c index 384011115f8..425b2f519d1 100644 --- a/plugins/out_azure_kusto/azure_kusto_ingest.c +++ b/plugins/out_azure_kusto/azure_kusto_ingest.c @@ -134,11 +134,12 @@ static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t /* uri will be /.multijson? */ uri = flb_sds_create_size(blob_uri_size + blob_sas_size + flb_sds_len(blob_id) + 13); if (!uri) { - flb_plg_error(ctx->ins, "error creating blob container uri"); + flb_plg_error(ctx->ins, "error creating blob container uri buffer"); flb_upstream_conn_release(u_conn); return NULL; } - uri = flb_sds_printf(&uri, "%s/%s.multijson?%s", blob_uri, blob_id, blob_sas); + flb_sds_snprintf(&uri, flb_sds_alloc(uri), "%s/%s.multijson?%s", blob_uri, blob_id, + blob_sas); flb_plg_debug(ctx->ins, "uploading payload to blob uri: %s", uri); @@ -185,13 +186,14 @@ static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t /* generate full blob uri & return it */ full_uri = flb_sds_create_size(9 + flb_sds_len(u_node->host) + flb_sds_len(uri)); if (!full_uri) { - flb_plg_error(ctx->ins, "cannot create full blob uri"); + flb_plg_error(ctx->ins, "cannot create full blob uri buffer"); flb_upstream_conn_release(u_conn); flb_sds_destroy(uri); flb_http_client_destroy(c); return NULL; } - full_uri = flb_sds_printf(&full_uri, "https://%s%s", u_node->host, uri); + flb_sds_snprintf(&full_uri, flb_sds_alloc(full_uri), "https://%s%s", u_node->host, + uri); flb_plg_debug(ctx->ins, "created blob %s", full_uri); @@ -209,6 +211,7 @@ static flb_sds_t create_ingestion_message(struct flb_azure_kusto *ctx, flb_sds_t char *uuid; char *message_b64; size_t b64_len; + size_t message_len; uuid = generate_uuid(); if (!uuid) { @@ -222,8 +225,9 @@ static flb_sds_t create_ingestion_message(struct flb_azure_kusto *ctx, flb_sds_t flb_free(uuid); return NULL; } - message = flb_sds_printf( - &message, + + message_len = flb_sds_snprintf( + &message, 0, "{\"Id\": \"%s\", \"BlobPath\": \"%s\", \"RawDataSize\": %lu, \"DatabaseName\": " "\"%s\", \"TableName\": \"%s\"," "\"AdditionalProperties\": { \"format\": \"multijson\", " @@ -232,10 +236,16 @@ static flb_sds_t create_ingestion_message(struct flb_azure_kusto *ctx, flb_sds_t uuid, blob_uri, payload_size, ctx->database_name, ctx->table_name, ctx->resources->identity_token, ctx->ingestion_mapping_reference == NULL ? "" : ctx->ingestion_mapping_reference); + if (message_len == -1) { + flb_plg_error(ctx->ins, "error creating ingestion message"); + flb_free(uuid); + flb_sds_destroy(message); + return NULL; + } flb_plg_debug(ctx->ins, "created ingestion message:\n%s", message); - message_b64 = base64_encode(message, flb_sds_len(message), &b64_len); + message_b64 = base64_encode(message, message_len, &b64_len); if (!message_b64) { flb_plg_error(ctx->ins, "error encoding ingestion message to base64"); flb_free(uuid); @@ -243,10 +253,15 @@ static flb_sds_t create_ingestion_message(struct flb_azure_kusto *ctx, flb_sds_t return NULL; } - flb_sds_len_set(message, 0); - message = flb_sds_printf(&message, - "%s", - message_b64); + message_len = flb_sds_snprintf( + &message, flb_sds_alloc(message), + "%s%c", message_b64, 0); + if (message_len == -1) { + flb_plg_error(ctx->ins, "error creating ingestion queue message"); + flb_free(uuid); + flb_sds_destroy(message); + return NULL; + } flb_free(uuid); flb_free(message_b64); @@ -305,11 +320,11 @@ static int azure_kusto_enqueue_ingestion(struct flb_azure_kusto *ctx, flb_sds_t /* uri will be /messages? */ uri = flb_sds_create_size(queue_uri_size + queue_sas_size + 11); if (!uri) { - flb_plg_error(ctx->ins, "error creating queue container uri"); + flb_plg_error(ctx->ins, "error creating queue container uri buffer"); flb_upstream_conn_release(u_conn); return -1; } - uri = flb_sds_printf(&uri, "%s/messages?%s", queue_uri, queue_sas); + flb_sds_snprintf(&uri, flb_sds_alloc(uri), "%s/messages?%s", queue_uri, queue_sas); payload = create_ingestion_message(ctx, blob_uri, payload_size); if (!payload) { @@ -394,12 +409,12 @@ int azure_kusto_queued_ingestion(struct flb_azure_kusto *ctx, flb_sds_t tag, blob_id = flb_sds_create_size(flb_sds_len(ctx->database_name) + flb_sds_len(ctx->table_name) + b64_len + 24); if (!blob_id) { - flb_plg_error(ctx->ins, "cannot create blob id"); + flb_plg_error(ctx->ins, "cannot create blob id buffer"); flb_free(b64tag); return -1; } - blob_id = flb_sds_printf(&blob_id, "flb__%s__%s__%s__%lu", ctx->database_name, - ctx->table_name, b64tag, ms); + flb_sds_snprintf(&blob_id, flb_sds_alloc(blob_id), "flb__%s__%s__%s__%lu", + ctx->database_name, ctx->table_name, b64tag, ms); blob_uri = azure_kusto_create_blob(ctx, blob_id, payload, payload_size); if (!blob_uri) { From 8553a510d5c367fc7e3520d95b3af6854023f50a Mon Sep 17 00:00:00 2001 From: Gershon Papiashvili Date: Fri, 15 Jul 2022 19:06:18 +0300 Subject: [PATCH 16/18] out_azure_kusto: avoid reprtitive free Signed-off-by: Gershon Papiashvili --- plugins/out_azure_kusto/azure_kusto.c | 147 +++-- plugins/out_azure_kusto/azure_kusto_conf.c | 550 ++++++++++--------- plugins/out_azure_kusto/azure_kusto_ingest.c | 537 ++++++++++-------- 3 files changed, 650 insertions(+), 584 deletions(-) diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index 09a249648d5..b0ba04c4c0e 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -123,89 +123,84 @@ flb_sds_t execute_ingest_csl_command(struct flb_azure_kusto *ctx, const char *cs int ret; struct flb_upstream_conn *u_conn; struct flb_http_client *c; - flb_sds_t resp; + flb_sds_t resp = NULL; /* Get upstream connection */ u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - return NULL; - } - - /* Get or renew Token */ - token = get_azure_kusto_token(ctx); - if (!token) { - flb_plg_error(ctx->ins, "cannot retrieve oauth2 token"); - flb_upstream_conn_release(u_conn); - return NULL; - } - - /* Compose request body */ - body = - flb_sds_create_size(sizeof(FLB_AZURE_KUSTO_MGMT_BODY_TEMPLATE) - 1 + strlen(csl)); - if (!body) { - flb_plg_error(ctx->ins, "cannot construct request body"); - flb_upstream_conn_release(u_conn); - flb_sds_destroy(token); - return NULL; - } - flb_sds_snprintf(&body, flb_sds_alloc(body), FLB_AZURE_KUSTO_MGMT_BODY_TEMPLATE, csl); - - /* Compose HTTP Client request */ - c = flb_http_client(u_conn, FLB_HTTP_POST, FLB_AZURE_KUSTO_MGMT_URI_PATH, body, - flb_sds_len(body), NULL, 0, NULL, 0); - if (!c) { - flb_plg_error(ctx->ins, "cannot create HTTP client context"); - flb_upstream_conn_release(u_conn); - flb_sds_destroy(token); - flb_sds_destroy(body); - return NULL; - } - - /* Add headers */ - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - flb_http_add_header(c, "Content-Type", 12, "application/json", 16); - flb_http_add_header(c, "Accept", 6, "application/json", 16); - flb_http_add_header(c, "Authorization", 13, token, flb_sds_len(token)); - - flb_http_buffer_size(c, FLB_HTTP_DATA_SIZE_MAX * 10); - - /* Send HTTP request */ - ret = flb_http_do(c, &b_sent); - flb_plg_debug(ctx->ins, "Kusto ingestion command request http_do=%i, HTTP Status: %i", - ret, c->resp.status); - - if (ret != 0) { - flb_plg_error(ctx->ins, "cannot send HTTP request"); - flb_upstream_conn_release(u_conn); - flb_sds_destroy(token); - flb_sds_destroy(body); - flb_http_client_destroy(c); - return NULL; - } - /* Validate return status and HTTP status if set */ - if (c->resp.status != 200) { - if (c->resp.payload_size > 0) { - flb_plg_debug(ctx->ins, "Request failed and returned: \n%s", c->resp.payload); + if (u_conn) { + token = get_azure_kusto_token(ctx); + + if (token) { + /* Compose request body */ + body = flb_sds_create_size(sizeof(FLB_AZURE_KUSTO_MGMT_BODY_TEMPLATE) - 1 + + strlen(csl)); + + if (body) { + flb_sds_snprintf(&body, flb_sds_alloc(body), + FLB_AZURE_KUSTO_MGMT_BODY_TEMPLATE, csl); + + /* Compose HTTP Client request */ + c = flb_http_client(u_conn, FLB_HTTP_POST, FLB_AZURE_KUSTO_MGMT_URI_PATH, + body, flb_sds_len(body), NULL, 0, NULL, 0); + + if (c) { + /* Add headers */ + flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); + flb_http_add_header(c, "Content-Type", 12, "application/json", 16); + flb_http_add_header(c, "Accept", 6, "application/json", 16); + flb_http_add_header(c, "Authorization", 13, token, + flb_sds_len(token)); + flb_http_buffer_size(c, FLB_HTTP_DATA_SIZE_MAX * 10); + + /* Send HTTP request */ + ret = flb_http_do(c, &b_sent); + flb_plg_debug( + ctx->ins, + "Kusto ingestion command request http_do=%i, HTTP Status: %i", + ret, c->resp.status); + + if (ret == 0) { + if (c->resp.status == 200) { + /* Copy payload response to the response param */ + resp = + flb_sds_create_len(c->resp.payload, c->resp.payload_size); + } + else if (c->resp.payload_size > 0) { + flb_plg_debug(ctx->ins, "Request failed and returned: \n%s", + c->resp.payload); + } + else { + flb_plg_debug(ctx->ins, "Request failed"); + } + } + else { + flb_plg_error(ctx->ins, "cannot send HTTP request"); + } + + flb_http_client_destroy(c); + } + else { + flb_plg_error(ctx->ins, "cannot create HTTP client context"); + } + + flb_sds_destroy(body); + } + else { + flb_plg_error(ctx->ins, "cannot construct request body"); + } + + flb_sds_destroy(token); } else { - flb_plg_debug(ctx->ins, "Request failed"); + flb_plg_error(ctx->ins, "cannot retrieve oauth2 token"); } + flb_upstream_conn_release(u_conn); - flb_sds_destroy(token); - flb_sds_destroy(body); - flb_http_client_destroy(c); - return NULL; } - - ret = c->resp.payload_size; - /* Copy payload response to the response param */ - resp = flb_sds_create_len(c->resp.payload, c->resp.payload_size); - - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - flb_sds_destroy(token); - flb_sds_destroy(body); + else { + flb_plg_error(ctx->ins, "cannot create upstream connection"); + } return resp; } @@ -230,8 +225,8 @@ static int cb_azure_kusto_init(struct flb_output_instance *ins, struct flb_confi io_flags |= FLB_IO_IPV6; } - /* Create mutex for acquiring oauth tokens and getting ingestion resources (they are - * shared across flush coroutines) + /* Create mutex for acquiring oauth tokens and getting ingestion resources (they + * are shared across flush coroutines) */ pthread_mutex_init(&ctx->token_mutex, NULL); pthread_mutex_init(&ctx->resources_mutex, NULL); diff --git a/plugins/out_azure_kusto/azure_kusto_conf.c b/plugins/out_azure_kusto/azure_kusto_conf.c index 4a25301225f..50cc4dff376 100644 --- a/plugins/out_azure_kusto/azure_kusto_conf.c +++ b/plugins/out_azure_kusto/azure_kusto_conf.c @@ -55,61 +55,50 @@ static struct flb_upstream_node *flb_upstream_node_create_url(struct flb_azure_k /* find sas token in query */ tmp = strchr(uri, '?'); - if (!tmp) { - flb_plg_error(ctx->ins, "uri has no sas token query: %s", uri); - flb_free(prot); - flb_free(host); - flb_free(port); - flb_free(uri); - return NULL; - } - uri_length = tmp - uri; - sas_length = strnlen(tmp + 1, 256); - - /* kv that will hold base uri, and sas token */ - kv = flb_hash_create(FLB_HASH_EVICT_NONE, 2, 2); - if (!kv) { - flb_plg_error(ctx->ins, "error creating upstream node hash table"); - flb_free(prot); - flb_free(host); - flb_free(port); - flb_free(uri); - return NULL; - } - - ret = flb_hash_add(kv, AZURE_KUSTO_RESOURCE_UPSTREAM_URI, 3, uri, uri_length); - if (ret == -1) { - flb_plg_error(ctx->ins, "error storing resource uri"); - flb_free(prot); - flb_free(host); - flb_free(port); - flb_free(uri); - flb_hash_destroy(kv); - return NULL; - } - ret = flb_hash_add(kv, AZURE_KUSTO_RESOURCE_UPSTREAM_SAS, 3, tmp + 1, sas_length); - if (ret == -1) { - flb_plg_error(ctx->ins, "error storing resource sas token"); - flb_free(prot); - flb_free(host); - flb_free(port); - flb_free(uri); - flb_hash_destroy(kv); - return NULL; + if (tmp) { + uri_length = tmp - uri; + sas_length = strnlen(tmp + 1, 256); + + /* kv that will hold base uri, and sas token */ + kv = flb_hash_create(FLB_HASH_EVICT_NONE, 2, 2); + + if (kv) { + ret = flb_hash_add(kv, AZURE_KUSTO_RESOURCE_UPSTREAM_URI, 3, uri, uri_length); + + if (ret != -1) { + ret = flb_hash_add(kv, AZURE_KUSTO_RESOURCE_UPSTREAM_SAS, 3, tmp + 1, + sas_length); + + if (ret != -1) { + node = flb_upstream_node_create( + NULL, host, port, FLB_TRUE, ctx->ins->tls->verify, + ctx->ins->tls->debug, ctx->ins->tls->vhost, NULL, NULL, NULL, + NULL, NULL, kv, config); + + if (!node) { + flb_plg_error(ctx->ins, "error creating resource upstream node"); + } + } + else { + flb_plg_error(ctx->ins, "error storing resource sas token"); + } + } + else { + flb_plg_error(ctx->ins, "error storing resource uri"); + } + + /* avoid destorying if function is successful */ + if (!node) { + flb_hash_destroy(kv); + } + } + else { + flb_plg_error(ctx->ins, "error creating upstream node hash table"); + } } - - node = flb_upstream_node_create(NULL, host, port, FLB_TRUE, ctx->ins->tls->verify, - ctx->ins->tls->debug, ctx->ins->tls->vhost, NULL, - NULL, NULL, NULL, NULL, kv, config); - if (!node) { - flb_plg_error(ctx->ins, "error creating resource upstream node"); - flb_free(prot); - flb_free(host); - flb_free(port); - flb_free(uri); - flb_hash_destroy(kv); - return NULL; + else { + flb_plg_error(ctx->ins, "uri has no sas token query: %s", uri); } flb_free(prot); @@ -167,7 +156,7 @@ static int parse_storage_resources(struct flb_azure_kusto *ctx, struct flb_confi jsmntok_t *t; jsmntok_t *tokens; int tok_size = 100; - int ret; + int ret = -1; int i; int blob_count = 0; int queue_count = 0; @@ -205,136 +194,139 @@ static int parse_storage_resources(struct flb_azure_kusto *ctx, struct flb_confi jsmn_init(&parser); tokens = flb_calloc(1, sizeof(jsmntok_t) * tok_size); - if (!tokens) { - flb_plg_error(ctx->ins, "error allocating tokens"); - flb_sds_destroy(resource_uri); - return -1; - } - - ret = jsmn_parse(&parser, response, flb_sds_len(response), tokens, tok_size); - if (ret <= 0) { - flb_plg_error(ctx->ins, "error parsing JSON response: %s", response); - flb_sds_destroy(resource_uri); - flb_free(tokens); - return -1; - } - /* skip all tokens until we reach "Rows" */ - for (i = 0; i < ret - 1; i++) { - t = &tokens[i]; - - if (t->type != JSMN_STRING) { - continue; - } - - token_str = response + t->start; - token_str_len = (t->end - t->start); - - /** - * if we found the Rows key, skipping this token and the next one (key and - * wrapping array value) - */ - if (token_str_len == 4 && strncmp(token_str, "Rows", 4) == 0) { - i += 2; - break; - } - } - - /* iterating rows, each row will have 3 tokens: the array holding the column values, - * the first value containing the resource type, and the second value containing the - * resource uri */ - for (; i < ret; i++) { - t = &tokens[i]; - - /** - * each token should be an array with 2 strings: - * First will be the resource type (TempStorage, SecuredReadyForAggregationQueue, - * etc...) Second will be the SAS URI - */ - if (t->type != JSMN_ARRAY) { - break; - } - - /* move to the next token, first item in the array - resource type */ - i++; - t = &tokens[i]; - if (t->type != JSMN_STRING) { - break; - } - - token_str = response + t->start; - token_str_len = (t->end - t->start); - - flb_plg_debug(ctx->ins, "found resource of type: %.*s ", t->end - t->start, - response + t->start); - - if (token_str_len == 11 && strncmp(token_str, "TempStorage", 11) == 0) { - resource_type = AZURE_KUSTO_RESOURCE_STORAGE; + if (tokens) { + ret = jsmn_parse(&parser, response, flb_sds_len(response), tokens, tok_size); + + if (ret > 0) { + /* skip all tokens until we reach "Rows" */ + for (i = 0; i < ret - 1; i++) { + t = &tokens[i]; + + if (t->type != JSMN_STRING) { + continue; + } + + token_str = response + t->start; + token_str_len = (t->end - t->start); + + /** + * if we found the Rows key, skipping this token and the next one (key and + * wrapping array value) + */ + if (token_str_len == 4 && strncmp(token_str, "Rows", 4) == 0) { + i += 2; + break; + } + } + + /* iterating rows, each row will have 3 tokens: the array holding the column + * values, the first value containing the resource type, and the second value + * containing the resource uri */ + for (; i < ret; i++) { + t = &tokens[i]; + + /** + * each token should be an array with 2 strings: + * First will be the resource type (TempStorage, + * SecuredReadyForAggregationQueue, etc...) Second will be the SAS URI + */ + if (t->type != JSMN_ARRAY) { + break; + } + + /* move to the next token, first item in the array - resource type */ + i++; + t = &tokens[i]; + if (t->type != JSMN_STRING) { + break; + } + + token_str = response + t->start; + token_str_len = (t->end - t->start); + + flb_plg_debug(ctx->ins, "found resource of type: %.*s ", + t->end - t->start, response + t->start); + + if (token_str_len == 11 && strncmp(token_str, "TempStorage", 11) == 0) { + resource_type = AZURE_KUSTO_RESOURCE_STORAGE; + } + else if (token_str_len == 31 && + strncmp(token_str, "SecuredReadyForAggregationQueue", 31) == 0) { + resource_type = AZURE_KUSTO_RESOURCE_QUEUE; + } + /* we don't care about other resources so we just skip the next token and + move on to the next pair */ + else { + i++; + continue; + } + + /* move to the next token, second item in the array - resource URI */ + i++; + t = &tokens[i]; + + if (t->type != JSMN_STRING) { + break; + } + + token_str = response + t->start; + token_str_len = (t->end - t->start); + + resource_uri = flb_sds_copy(resource_uri, token_str, token_str_len); + if (resource_type == AZURE_KUSTO_RESOURCE_QUEUE) { + ha = queue_ha; + queue_count++; + } + else { + ha = blob_ha; + blob_count++; + } + + if (!ha) { + flb_plg_error(ctx->ins, "error creating HA upstream"); + ret = -1; + break; + } + + node = flb_upstream_node_create_url(ctx, config, resource_uri); + + if (!node) { + flb_plg_error(ctx->ins, "error creating HA upstream node"); + ret = -1; + break; + } + + flb_upstream_ha_node_add(ha, node); + } + + if (ret != -1) { + if (queue_count > 0 && blob_count > 0) { + flb_plg_debug(ctx->ins, + "parsed %d blob resources and %d queue resources", + blob_count, queue_count); + ret = 0; + } + else { + flb_plg_error(ctx->ins, "error parsing resources: missing resources"); + ret = -1; + } + } } - else if (token_str_len == 31 && - strncmp(token_str, "SecuredReadyForAggregationQueue", 31) == 0) { - resource_type = AZURE_KUSTO_RESOURCE_QUEUE; - } - /* we don't care about other resources so we just skip the next token and move - on to the next pair */ else { - i++; - continue; - } - - /* move to the next token, second item in the array - resource URI */ - i++; - t = &tokens[i]; - - if (t->type != JSMN_STRING) { - break; - } - - token_str = response + t->start; - token_str_len = (t->end - t->start); - - resource_uri = flb_sds_copy(resource_uri, token_str, token_str_len); - if (resource_type == AZURE_KUSTO_RESOURCE_QUEUE) { - ha = queue_ha; - queue_count++; - } - else { - ha = blob_ha; - blob_count++; - } - - if (!ha) { - flb_plg_error(ctx->ins, "error creating HA upstream"); - flb_sds_destroy(resource_uri); - flb_free(tokens); - return -1; - } - - node = flb_upstream_node_create_url(ctx, config, resource_uri); - if (!node) { - flb_plg_error(ctx->ins, "error creating HA upstream node"); - flb_sds_destroy(resource_uri); - flb_free(tokens); - return -1; + flb_plg_error(ctx->ins, "error parsing JSON response: %s", response); + ret = -1; } - - flb_upstream_ha_node_add(ha, node); } - - if (!queue_count || !blob_count) { - flb_plg_error(ctx->ins, "error parsing resources: missing resources"); - flb_sds_destroy(resource_uri); - flb_free(tokens); - return -1; + else { + flb_plg_error(ctx->ins, "error allocating tokens"); + ret = -1; } flb_sds_destroy(resource_uri); flb_free(tokens); - flb_plg_debug(ctx->ins, "parsed %d blob resources and %d queue resources", blob_count, - queue_count); - - return 0; + return ret; } /** @@ -347,7 +339,7 @@ static int parse_storage_resources(struct flb_azure_kusto *ctx, struct flb_confi static flb_sds_t parse_ingestion_identity_token(struct flb_azure_kusto *ctx, flb_sds_t response) { - flb_sds_t identity_token; + flb_sds_t identity_token = NULL; int tok_size = 19; jsmn_parser parser; jsmntok_t *t; @@ -387,26 +379,32 @@ static flb_sds_t parse_ingestion_identity_token(struct flb_azure_kusto *ctx, } ret = jsmn_parse(&parser, response, flb_sds_len(response), tokens, tok_size); - if (ret <= 0) { - flb_plg_error(ctx->ins, "error parsing JSON response: %s", response); - flb_free(tokens); - return NULL; + if (ret > 0) { + t = &tokens[tok_size - 1]; + + if (t->type == JSMN_STRING) { + t = &tokens[tok_size - 1]; + token_str = response + t->start; + token_str_len = (t->end - t->start); + + identity_token = flb_sds_create_len(token_str, token_str_len); + + if (identity_token) { + flb_plg_debug(ctx->ins, "parsed kusto identity token: '%s'", + identity_token); + } + else { + flb_plg_error(ctx->ins, "error parsing kusto identity token"); + } + } + else { + flb_plg_error(ctx->ins, "unexpected JSON response: %s", response); + } } - - t = &tokens[tok_size - 1]; - if (t->type != JSMN_STRING) { - flb_plg_error(ctx->ins, "unexpected JSON response: %s", response); - flb_free(tokens); - return NULL; + else { + flb_plg_error(ctx->ins, "error parsing JSON response: %s", response); } - token_str = response + t->start; - token_str_len = (t->end - t->start); - - identity_token = flb_sds_create_len(token_str, token_str_len); - - flb_plg_debug(ctx->ins, "parsed kusto identity token: '%s'", identity_token); - flb_free(tokens); return identity_token; @@ -415,7 +413,7 @@ static flb_sds_t parse_ingestion_identity_token(struct flb_azure_kusto *ctx, int azure_kusto_load_ingestion_resources(struct flb_azure_kusto *ctx, struct flb_config *config) { - int ret; + int ret = -1; flb_sds_t response = NULL; flb_sds_t identity_token = NULL; struct flb_upstream_ha *blob_ha = NULL; @@ -433,96 +431,114 @@ int azure_kusto_load_ingestion_resources(struct flb_azure_kusto *ctx, if (ctx->resources->blob_ha && ctx->resources->queue_ha && ctx->resources->identity_token && now - ctx->resources->load_time < FLB_AZURE_KUSTO_RESOURCES_LOAD_INTERVAL_SEC) { - if (pthread_mutex_unlock(&ctx->resources_mutex)) { - flb_plg_error(ctx->ins, "error unlocking mutex"); - return -1; - } - flb_plg_debug(ctx->ins, "resources are already loaded and are not stale"); - return 0; - } - - flb_plg_info(ctx->ins, "loading kusto ingestion resourcs"); - - response = execute_ingest_csl_command(ctx, ".get ingestion resources"); - if (!response) { - flb_plg_error(ctx->ins, "error getting ingestion storage resources"); - goto load_resources_error; - } - - queue_ha = flb_upstream_ha_create("azure_kusto_queue_ha"); - blob_ha = flb_upstream_ha_create("azure_kusto_blob_ha"); - if (!queue_ha || !blob_ha) { - flb_plg_error(ctx->ins, "error creating storage resources upstreams"); - goto load_resources_error; - } - - ret = parse_storage_resources(ctx, config, response, blob_ha, queue_ha); - if (ret != 0) { - flb_plg_error(ctx->ins, "error parsing ingestion storage resources"); - goto load_resources_error; - } - - flb_sds_destroy(response); - response = NULL; - - response = execute_ingest_csl_command(ctx, ".get kusto identity token"); - if (!response) { - flb_plg_error(ctx->ins, "error getting kusto identity token"); - goto load_resources_error; - } - - identity_token = parse_ingestion_identity_token(ctx, response); - if (!identity_token) { - flb_plg_error(ctx->ins, "error parsing ingestion identity token"); - goto load_resources_error; + ret = 0; + } + else { + flb_plg_info(ctx->ins, "loading kusto ingestion resourcs"); + response = execute_ingest_csl_command(ctx, ".get ingestion resources"); + + if (response) { + queue_ha = flb_upstream_ha_create("azure_kusto_queue_ha"); + + if (queue_ha) { + blob_ha = flb_upstream_ha_create("azure_kusto_blob_ha"); + + if (blob_ha) { + ret = + parse_storage_resources(ctx, config, response, blob_ha, queue_ha); + + if (ret == 0) { + flb_sds_destroy(response); + response = NULL; + + response = + execute_ingest_csl_command(ctx, ".get kusto identity token"); + + if (response) { + identity_token = + parse_ingestion_identity_token(ctx, response); + + if (identity_token) { + ret = flb_azure_kusto_resources_clear(ctx->resources); + + if (ret != -1) { + ctx->resources->blob_ha = blob_ha; + ctx->resources->queue_ha = queue_ha; + ctx->resources->identity_token = identity_token; + ctx->resources->load_time = now; + + ret = 0; + } + else { + flb_plg_error( + ctx->ins, + "error destroying previous ingestion resources"); + } + } + else { + flb_plg_error(ctx->ins, + "error parsing ingestion identity token"); + ret = -1; + } + } + else { + flb_plg_error(ctx->ins, "error getting kusto identity token"); + ret = -1; + } + } + else { + flb_plg_error(ctx->ins, + "error parsing ingestion storage resources"); + ret = -1; + } + + if (ret == -1) { + flb_upstream_ha_destroy(blob_ha); + } + } + else { + flb_plg_error(ctx->ins, "error creating storage resources upstreams"); + ret = -1; + } + + if (ret == -1) { + flb_upstream_ha_destroy(queue_ha); + } + } + else { + flb_plg_error(ctx->ins, "error creating storage resources upstreams"); + } + + if (response) { + flb_sds_destroy(response); + } + } + if (!response) { + flb_plg_error(ctx->ins, "error getting ingestion storage resources"); + } } - flb_sds_destroy(response); - - /* free old resources and point to newly allocated */ - flb_azure_kusto_resources_clear(ctx->resources); - ctx->resources->blob_ha = blob_ha; - ctx->resources->queue_ha = queue_ha; - ctx->resources->identity_token = identity_token; - ctx->resources->load_time = now; - if (pthread_mutex_unlock(&ctx->resources_mutex)) { flb_plg_error(ctx->ins, "error unlocking mutex"); return -1; } - return 0; - -load_resources_error: - pthread_mutex_unlock(&ctx->resources_mutex); - - if (response) { - flb_sds_destroy(response); - } - - if (blob_ha) { - flb_upstream_ha_destroy(blob_ha); - } - - if (queue_ha) { - flb_upstream_ha_destroy(queue_ha); - } - - if (identity_token) { - flb_sds_destroy(identity_token); - } - - return -1; + return ret; } static int flb_azure_kusto_resources_destroy(struct flb_azure_kusto_resources *resources) { + int ret; + if (!resources) { return -1; } - flb_azure_kusto_resources_clear(resources); + ret = flb_azure_kusto_resources_clear(resources); + if (ret != 0) { + return -1; + } flb_free(resources); diff --git a/plugins/out_azure_kusto/azure_kusto_ingest.c b/plugins/out_azure_kusto/azure_kusto_ingest.c index 425b2f519d1..ef842384ec9 100644 --- a/plugins/out_azure_kusto/azure_kusto_ingest.c +++ b/plugins/out_azure_kusto/azure_kusto_ingest.c @@ -83,34 +83,16 @@ static char *base64_encode(flb_sds_t s, size_t len, size_t *out_len) return b64; } -static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t blob_id, - flb_sds_t payload, size_t payload_size) +static flb_sds_t azure_kusto_create_blob_uri(struct flb_azure_kusto *ctx, + struct flb_upstream_node *u_node, + flb_sds_t blob_id) { int ret; + flb_sds_t uri = NULL; char *blob_uri; size_t blob_uri_size; char *blob_sas; size_t blob_sas_size; - flb_sds_t uri; - flb_sds_t full_uri; - struct flb_upstream_node *u_node; - struct flb_upstream_conn *u_conn; - struct flb_http_client *c; - size_t resp_size; - time_t now; - struct tm tm; - char tmp[64]; - int len; - - now = time(NULL); - gmtime_r(&now, &tm); - len = strftime(tmp, sizeof(tmp) - 1, "%a, %d %b %Y %H:%M:%S GMT", &tm); - - u_node = flb_upstream_ha_node_get(ctx->resources->blob_ha); - if (!u_node) { - flb_plg_error(ctx->ins, "error getting blob upstream"); - return NULL; - } ret = flb_hash_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_URI, 3, (void **)&blob_uri, &blob_uri_size); @@ -118,6 +100,7 @@ static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t flb_plg_error(ctx->ins, "error getting blob uri"); return NULL; } + ret = flb_hash_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_SAS, 3, (void **)&blob_sas, &blob_sas_size); if (ret == -1) { @@ -125,158 +108,226 @@ static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t return NULL; } - u_conn = flb_upstream_conn_get(u_node->u); - if (!u_conn) { - flb_plg_error(ctx->ins, "error getting blob container upstream connection"); - return NULL; - } + /* uri will be https:////.multijson? */ + uri = flb_sds_create_size(flb_sds_len(u_node->host) + blob_uri_size + blob_sas_size + + flb_sds_len(blob_id) + 21); - /* uri will be /.multijson? */ - uri = flb_sds_create_size(blob_uri_size + blob_sas_size + flb_sds_len(blob_id) + 13); - if (!uri) { - flb_plg_error(ctx->ins, "error creating blob container uri buffer"); - flb_upstream_conn_release(u_conn); - return NULL; + if (uri) { + flb_sds_snprintf(&uri, flb_sds_alloc(uri), "https://%s%s/%s.multijson?%s", + u_node->host, blob_uri, blob_id, blob_sas); + flb_plg_debug(ctx->ins, "created blob uri %s", uri); + } + else { + flb_plg_error(ctx->ins, "cannot create blob uri buffer"); } - flb_sds_snprintf(&uri, flb_sds_alloc(uri), "%s/%s.multijson?%s", blob_uri, blob_id, - blob_sas); - flb_plg_debug(ctx->ins, "uploading payload to blob uri: %s", uri); + return uri; +} - c = flb_http_client(u_conn, FLB_HTTP_PUT, uri, payload, payload_size, NULL, 0, NULL, - 0); - if (!c) { - flb_plg_error(ctx->ins, "cannot create HTTP client context for blob container"); - flb_upstream_conn_release(u_conn); - flb_sds_destroy(uri); - return NULL; - } +static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t blob_id, + flb_sds_t payload, size_t payload_size) +{ + int ret = -1; + flb_sds_t uri = NULL; + struct flb_upstream_node *u_node; + struct flb_upstream_conn *u_conn; + struct flb_http_client *c; + size_t resp_size; + time_t now; + struct tm tm; + char tmp[64]; + int len; - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - flb_http_add_header(c, "Content-Type", 12, "application/json", 16); - flb_http_add_header(c, "x-ms-blob-type", 14, "BlockBlob", 9); - flb_http_add_header(c, "x-ms-date", 9, tmp, len); - flb_http_add_header(c, "x-ms-version", 12, "2019-12-12", 10); + now = time(NULL); + gmtime_r(&now, &tm); + len = strftime(tmp, sizeof(tmp) - 1, "%a, %d %b %Y %H:%M:%S GMT", &tm); - ret = flb_http_do(c, &resp_size); - flb_plg_debug(ctx->ins, "kusto blob upload request http_do=%i, HTTP Status: %i", ret, - c->resp.status); - if (ret != 0) { - flb_plg_error(ctx->ins, "cannot send HTTP request"); - flb_upstream_conn_release(u_conn); - flb_sds_destroy(uri); - flb_http_client_destroy(c); + u_node = flb_upstream_ha_node_get(ctx->resources->blob_ha); + if (!u_node) { + flb_plg_error(ctx->ins, "error getting blob upstream"); return NULL; } - /* Validate return status and HTTP status if set */ - if (c->resp.status != 201) { - if (c->resp.payload_size > 0) { - flb_plg_debug(ctx->ins, "Request failed and returned: \n%s", c->resp.payload); + u_conn = flb_upstream_conn_get(u_node->u); + + if (u_conn) { + uri = azure_kusto_create_blob_uri(ctx, u_node, blob_id); + + if (uri) { + flb_plg_debug(ctx->ins, "uploading payload to blob uri: %s", uri); + c = flb_http_client(u_conn, FLB_HTTP_PUT, uri, payload, payload_size, NULL, 0, + NULL, 0); + + if (c) { + flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); + flb_http_add_header(c, "Content-Type", 12, "application/json", 16); + flb_http_add_header(c, "x-ms-blob-type", 14, "BlockBlob", 9); + flb_http_add_header(c, "x-ms-date", 9, tmp, len); + flb_http_add_header(c, "x-ms-version", 12, "2019-12-12", 10); + + ret = flb_http_do(c, &resp_size); + flb_plg_debug(ctx->ins, + "kusto blob upload request http_do=%i, HTTP Status: %i", + ret, c->resp.status); + + if (ret == 0) { + /* Validate return status and HTTP status if set */ + if (c->resp.status != 201) { + ret = -1; + + if (c->resp.payload_size > 0) { + flb_plg_debug(ctx->ins, "Request failed and returned: \n%s", + c->resp.payload); + } + else { + flb_plg_debug(ctx->ins, "Request failed"); + } + } + } + else { + flb_plg_error(ctx->ins, "cannot send HTTP request"); + } + + flb_http_client_destroy(c); + } + else { + flb_plg_error(ctx->ins, + "cannot create HTTP client context for blob container"); + } + + if (ret != 0) { + flb_sds_destroy(uri); + uri = NULL; + } } else { - flb_plg_debug(ctx->ins, "Request failed"); + flb_plg_error(ctx->ins, "error creating blob container uri buffer"); } - flb_upstream_conn_release(u_conn); - flb_sds_destroy(uri); - flb_http_client_destroy(c); - return NULL; - } - /* generate full blob uri & return it */ - full_uri = flb_sds_create_size(9 + flb_sds_len(u_node->host) + flb_sds_len(uri)); - if (!full_uri) { - flb_plg_error(ctx->ins, "cannot create full blob uri buffer"); flb_upstream_conn_release(u_conn); - flb_sds_destroy(uri); - flb_http_client_destroy(c); - return NULL; } - flb_sds_snprintf(&full_uri, flb_sds_alloc(full_uri), "https://%s%s", u_node->host, - uri); - - flb_plg_debug(ctx->ins, "created blob %s", full_uri); - - flb_upstream_conn_release(u_conn); - flb_sds_destroy(uri); - flb_http_client_destroy(c); + else { + flb_plg_error(ctx->ins, "error getting blob container upstream connection"); + } - return full_uri; + return uri; } static flb_sds_t create_ingestion_message(struct flb_azure_kusto *ctx, flb_sds_t blob_uri, size_t payload_size) { - flb_sds_t message; + flb_sds_t message = NULL; + int ret = 0; char *uuid; char *message_b64; size_t b64_len; size_t message_len; uuid = generate_uuid(); - if (!uuid) { - flb_plg_error(ctx->ins, "error generating unique ingestion UUID"); - return NULL; - } + if (uuid) { + message = flb_sds_create(NULL); + + if (message) { + message_len = + flb_sds_snprintf(&message, 0, + "{\"Id\": \"%s\", \"BlobPath\": \"%s\", " + "\"RawDataSize\": %lu, \"DatabaseName\": " + "\"%s\", \"TableName\": \"%s\"," + "\"AdditionalProperties\": { \"format\": \"multijson\", " + "\"authorizationContext\": " + "\"%s\", \"jsonMappingReference\": \"%s\" }}", + uuid, blob_uri, payload_size, ctx->database_name, + ctx->table_name, ctx->resources->identity_token, + ctx->ingestion_mapping_reference == NULL + ? "" + : ctx->ingestion_mapping_reference); + + if (message_len != -1) { + flb_plg_debug(ctx->ins, "created ingestion message:\n%s", message); + message_b64 = base64_encode(message, message_len, &b64_len); + + if (message_b64) { + ret = flb_sds_snprintf( + &message, flb_sds_alloc(message), + "%s%c", + message_b64, 0); + + if (ret == -1) { + flb_plg_error(ctx->ins, "error creating ingestion queue message"); + } + + flb_free(message_b64); + } + else { + flb_plg_error(ctx->ins, "error encoding ingestion message to base64"); + } + } + else { + flb_plg_error(ctx->ins, "error creating ingestion message"); + ret = -1; + } + + if (ret == -1) { + flb_sds_destroy(message); + message = NULL; + } + } + else { + flb_plg_error(ctx->ins, "error creating ingestion message buffer"); + } - message = flb_sds_create(NULL); - if (!message) { - flb_plg_error(ctx->ins, "error creating ingestion message buffer"); flb_free(uuid); - return NULL; } - - message_len = flb_sds_snprintf( - &message, 0, - "{\"Id\": \"%s\", \"BlobPath\": \"%s\", \"RawDataSize\": %lu, \"DatabaseName\": " - "\"%s\", \"TableName\": \"%s\"," - "\"AdditionalProperties\": { \"format\": \"multijson\", " - "\"authorizationContext\": " - "\"%s\", \"jsonMappingReference\": \"%s\" }}", - uuid, blob_uri, payload_size, ctx->database_name, ctx->table_name, - ctx->resources->identity_token, - ctx->ingestion_mapping_reference == NULL ? "" : ctx->ingestion_mapping_reference); - if (message_len == -1) { - flb_plg_error(ctx->ins, "error creating ingestion message"); - flb_free(uuid); - flb_sds_destroy(message); - return NULL; + else { + flb_plg_error(ctx->ins, "error generating unique ingestion UUID"); } - flb_plg_debug(ctx->ins, "created ingestion message:\n%s", message); + return message; +} - message_b64 = base64_encode(message, message_len, &b64_len); - if (!message_b64) { - flb_plg_error(ctx->ins, "error encoding ingestion message to base64"); - flb_free(uuid); - flb_sds_destroy(message); +static flb_sds_t azure_kusto_create_queue_uri(struct flb_azure_kusto *ctx, + struct flb_upstream_node *u_node) +{ + int ret; + flb_sds_t uri = NULL; + char *queue_uri; + size_t queue_uri_size; + char *queue_sas; + size_t queue_sas_size; + + ret = flb_hash_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_URI, 3, + (void **)&queue_uri, &queue_uri_size); + if (ret == -1) { + flb_plg_error(ctx->ins, "error getting queue uri"); return NULL; } - message_len = flb_sds_snprintf( - &message, flb_sds_alloc(message), - "%s%c", message_b64, 0); - if (message_len == -1) { - flb_plg_error(ctx->ins, "error creating ingestion queue message"); - flb_free(uuid); - flb_sds_destroy(message); + ret = flb_hash_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_SAS, 3, + (void **)&queue_sas, &queue_sas_size); + if (ret == -1) { + flb_plg_error(ctx->ins, "error getting queue sas token"); return NULL; } - flb_free(uuid); - flb_free(message_b64); + /* uri will be /messages? */ + uri = flb_sds_create_size(queue_uri_size + queue_sas_size + 11); - return message; + if (uri) { + flb_sds_snprintf(&uri, flb_sds_alloc(uri), "%s/messages?%s", queue_uri, + queue_sas); + flb_plg_debug(ctx->ins, "created queue uri %s", uri); + } + else { + flb_plg_error(ctx->ins, "cannot create queue uri buffer"); + } + + return uri; } static int azure_kusto_enqueue_ingestion(struct flb_azure_kusto *ctx, flb_sds_t blob_uri, size_t payload_size) { - int ret; - char *queue_uri; - size_t queue_uri_size; - char *queue_sas; - size_t queue_sas_size; + int ret = -1; struct flb_upstream_node *u_node; struct flb_upstream_conn *u_conn; struct flb_http_client *c; @@ -298,145 +349,149 @@ static int azure_kusto_enqueue_ingestion(struct flb_azure_kusto *ctx, flb_sds_t return -1; } - ret = flb_hash_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_URI, 3, - (void **)&queue_uri, &queue_uri_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "error getting queue uri"); - return -1; - } - ret = flb_hash_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_SAS, 3, - (void **)&queue_sas, &queue_sas_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "error getting queue sas token"); - return -1; - } - u_conn = flb_upstream_conn_get(u_node->u); - if (!u_conn) { - flb_plg_error(ctx->ins, "error getting queue upstream connection"); - return -1; - } - /* uri will be /messages? */ - uri = flb_sds_create_size(queue_uri_size + queue_sas_size + 11); - if (!uri) { - flb_plg_error(ctx->ins, "error creating queue container uri buffer"); - flb_upstream_conn_release(u_conn); - return -1; - } - flb_sds_snprintf(&uri, flb_sds_alloc(uri), "%s/messages?%s", queue_uri, queue_sas); + if (u_conn) { + uri = azure_kusto_create_queue_uri(ctx, u_node); + + if (uri) { + payload = create_ingestion_message(ctx, blob_uri, payload_size); + + if (payload) { + c = flb_http_client(u_conn, FLB_HTTP_POST, uri, payload, + flb_sds_len(payload), NULL, 0, NULL, 0); + + if (c) { + flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); + flb_http_add_header(c, "Content-Type", 12, "application/atom+xml", + 20); + flb_http_add_header(c, "x-ms-date", 9, tmp, len); + flb_http_add_header(c, "x-ms-version", 12, "2019-12-12", 10); + + ret = flb_http_do(c, &resp_size); + flb_plg_debug(ctx->ins, + "kusto queue request http_do=%i, HTTP Status: %i", ret, + c->resp.status); + + if (ret == 0) { + /* Validate return status and HTTP status if set */ + if (c->resp.status != 201) { + ret = -1; + + if (c->resp.payload_size > 0) { + flb_plg_debug(ctx->ins, + "Request failed and returned: \n%s", + c->resp.payload); + } + else { + flb_plg_debug(ctx->ins, "Request failed"); + } + } + } + else { + flb_plg_error(ctx->ins, "cannot send HTTP request"); + } + + flb_http_client_destroy(c); + } + else { + flb_plg_error(ctx->ins, + "cannot create HTTP client context for queue"); + } + + flb_sds_destroy(payload); + } + else { + flb_plg_error(ctx->ins, "error creating payload buffer"); + } + + flb_sds_destroy(uri); + } + else { + flb_plg_error(ctx->ins, "error creating queue uri buffer"); + } - payload = create_ingestion_message(ctx, blob_uri, payload_size); - if (!payload) { - flb_plg_error(ctx->ins, "error creating payload buffer"); flb_upstream_conn_release(u_conn); - flb_sds_destroy(uri); - return -1; } - - c = flb_http_client(u_conn, FLB_HTTP_POST, uri, payload, flb_sds_len(payload), NULL, - 0, NULL, 0); - if (!c) { - flb_plg_error(ctx->ins, "cannot create HTTP client context for blob container"); - flb_upstream_conn_release(u_conn); - flb_sds_destroy(uri); - flb_sds_destroy(payload); - return -1; + else { + flb_plg_error(ctx->ins, "error getting queue upstream connection"); } - flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); - flb_http_add_header(c, "Content-Type", 12, "application/atom+xml", 20); - flb_http_add_header(c, "x-ms-date", 9, tmp, len); - flb_http_add_header(c, "x-ms-version", 12, "2019-12-12", 10); + return ret; +} - ret = flb_http_do(c, &resp_size); - flb_plg_debug(ctx->ins, "kusto queue request http_do=%i, HTTP Status: %i", ret, - c->resp.status); - if (ret != 0) { - flb_plg_error(ctx->ins, "cannot send HTTP request"); - flb_upstream_conn_release(u_conn); - flb_sds_destroy(uri); - flb_sds_destroy(payload); - flb_http_client_destroy(c); - return -1; - } +static flb_sds_t azure_kusto_create_blob_id(struct flb_azure_kusto *ctx, flb_sds_t tag, + size_t tag_len) +{ + flb_sds_t blob_id = NULL; + struct flb_time tm; + uint64_t ms; + char *b64tag; + size_t b64_len; + + flb_time_get(&tm); + ms = ((tm.tm.tv_sec * 1000) + (tm.tm.tv_nsec / 1000000)); + + b64tag = base64_encode(tag, tag_len, &b64_len); + + if (b64tag) { + /* remove trailing '=' */ + while (b64_len && b64tag[b64_len - 1] == '=') { + b64tag[b64_len - 1] = '\0'; + b64_len--; + } - /* Validate return status and HTTP status if set */ - if (c->resp.status != 201) { - if (c->resp.payload_size > 0) { - flb_plg_debug(ctx->ins, "Request failed and returned: \n%s", c->resp.payload); + blob_id = flb_sds_create_size(flb_sds_len(ctx->database_name) + + flb_sds_len(ctx->table_name) + b64_len + 24); + if (blob_id) { + flb_sds_snprintf(&blob_id, flb_sds_alloc(blob_id), "flb__%s__%s__%s__%lu", + ctx->database_name, ctx->table_name, b64tag, ms); } else { - flb_plg_debug(ctx->ins, "Request failed"); + flb_plg_error(ctx->ins, "cannot create blob id buffer"); } - flb_upstream_conn_release(u_conn); - flb_sds_destroy(uri); - flb_sds_destroy(payload); - flb_http_client_destroy(c); - return -1; - } - flb_upstream_conn_release(u_conn); - flb_sds_destroy(uri); - flb_sds_destroy(payload); - flb_http_client_destroy(c); + flb_free(b64tag); + } + else { + flb_plg_error(ctx->ins, "error encoding tag '%s' to base64", tag); + } - return 0; + return blob_id; } int azure_kusto_queued_ingestion(struct flb_azure_kusto *ctx, flb_sds_t tag, size_t tag_len, flb_sds_t payload, size_t payload_size) { - int ret; - struct flb_time tm; - char *b64tag; - size_t b64_len; - uint64_t ms; + int ret = -1; flb_sds_t blob_id; flb_sds_t blob_uri; - flb_time_get(&tm); - ms = ((tm.tm.tv_sec * 1000) + (tm.tm.tv_nsec / 1000000)); + /* flb____
____ */ + blob_id = azure_kusto_create_blob_id(ctx, tag, tag_len); - b64tag = base64_encode(tag, tag_len, &b64_len); - /* remove trailing '=' */ - while (b64_len && b64tag[b64_len - 1] == '=') { - b64tag[b64_len - 1] = '\0'; - b64_len--; - } + if (blob_id) { + blob_uri = azure_kusto_create_blob(ctx, blob_id, payload, payload_size); - /* flb____
____ */ - blob_id = flb_sds_create_size(flb_sds_len(ctx->database_name) + - flb_sds_len(ctx->table_name) + b64_len + 24); - if (!blob_id) { - flb_plg_error(ctx->ins, "cannot create blob id buffer"); - flb_free(b64tag); - return -1; - } - flb_sds_snprintf(&blob_id, flb_sds_alloc(blob_id), "flb__%s__%s__%s__%lu", - ctx->database_name, ctx->table_name, b64tag, ms); + if (blob_uri) { + ret = azure_kusto_enqueue_ingestion(ctx, blob_uri, payload_size); - blob_uri = azure_kusto_create_blob(ctx, blob_id, payload, payload_size); - if (!blob_uri) { - flb_plg_error(ctx->ins, "failed to create payload blob"); - flb_free(b64tag); - flb_sds_destroy(blob_id); - return -1; - } + if (ret != 0) { + flb_plg_error(ctx->ins, "failed to enqueue ingestion blob to queue"); + ret = -1; + } + + flb_sds_destroy(blob_uri); + } + else { + flb_plg_error(ctx->ins, "failed to create payload blob uri"); + } - ret = azure_kusto_enqueue_ingestion(ctx, blob_uri, payload_size); - if (ret != 0) { - flb_plg_error(ctx->ins, "failed to enqueue ingestion blob to queue"); - flb_free(b64tag); flb_sds_destroy(blob_id); - flb_sds_destroy(blob_uri); - return -1; + } + else { + flb_plg_error(ctx->ins, "cannot create blob id"); } - /* Cleanup */ - flb_free(b64tag); - flb_sds_destroy(blob_id); - flb_sds_destroy(blob_uri); - - return 0; + return ret; } \ No newline at end of file From 567ccd872df369f9569986de7306dda13150a71d Mon Sep 17 00:00:00 2001 From: Gershon Papiashvili Date: Fri, 5 Aug 2022 02:29:31 +0300 Subject: [PATCH 17/18] out_azure_kusto: fix ingestion message snprintf Signed-off-by: Gershon Papiashvili --- plugins/out_azure_kusto/azure_kusto_ingest.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/out_azure_kusto/azure_kusto_ingest.c b/plugins/out_azure_kusto/azure_kusto_ingest.c index ef842384ec9..fc4f8e22cde 100644 --- a/plugins/out_azure_kusto/azure_kusto_ingest.c +++ b/plugins/out_azure_kusto/azure_kusto_ingest.c @@ -235,12 +235,12 @@ static flb_sds_t create_ingestion_message(struct flb_azure_kusto *ctx, flb_sds_t "\"%s\", \"TableName\": \"%s\"," "\"AdditionalProperties\": { \"format\": \"multijson\", " "\"authorizationContext\": " - "\"%s\", \"jsonMappingReference\": \"%s\" }}", + "\"%s\", \"jsonMappingReference\": \"%s\" }}%c", uuid, blob_uri, payload_size, ctx->database_name, ctx->table_name, ctx->resources->identity_token, ctx->ingestion_mapping_reference == NULL ? "" - : ctx->ingestion_mapping_reference); + : ctx->ingestion_mapping_reference, 0); if (message_len != -1) { flb_plg_debug(ctx->ins, "created ingestion message:\n%s", message); From 9ed4f0c02eb3e96d00f6235eeb615ac638018870 Mon Sep 17 00:00:00 2001 From: Gershon Papiashvili Date: Wed, 10 Aug 2022 17:24:06 +0300 Subject: [PATCH 18/18] out_azure_kusto: use new name for hash table interface Signed-off-by: Gershon Papiashvili --- plugins/out_azure_kusto/azure_kusto_conf.c | 10 +++++----- plugins/out_azure_kusto/azure_kusto_ingest.c | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/plugins/out_azure_kusto/azure_kusto_conf.c b/plugins/out_azure_kusto/azure_kusto_conf.c index 50cc4dff376..a5e0e9eb435 100644 --- a/plugins/out_azure_kusto/azure_kusto_conf.c +++ b/plugins/out_azure_kusto/azure_kusto_conf.c @@ -41,7 +41,7 @@ static struct flb_upstream_node *flb_upstream_node_create_url(struct flb_azure_k char *port = NULL; char *uri = NULL; char *tmp; - struct flb_hash *kv = NULL; + struct flb_hash_table *kv = NULL; struct flb_upstream_node *node = NULL; int uri_length; int sas_length; @@ -61,13 +61,13 @@ static struct flb_upstream_node *flb_upstream_node_create_url(struct flb_azure_k sas_length = strnlen(tmp + 1, 256); /* kv that will hold base uri, and sas token */ - kv = flb_hash_create(FLB_HASH_EVICT_NONE, 2, 2); + kv = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 2, 2); if (kv) { - ret = flb_hash_add(kv, AZURE_KUSTO_RESOURCE_UPSTREAM_URI, 3, uri, uri_length); + ret = flb_hash_table_add(kv, AZURE_KUSTO_RESOURCE_UPSTREAM_URI, 3, uri, uri_length); if (ret != -1) { - ret = flb_hash_add(kv, AZURE_KUSTO_RESOURCE_UPSTREAM_SAS, 3, tmp + 1, + ret = flb_hash_table_add(kv, AZURE_KUSTO_RESOURCE_UPSTREAM_SAS, 3, tmp + 1, sas_length); if (ret != -1) { @@ -90,7 +90,7 @@ static struct flb_upstream_node *flb_upstream_node_create_url(struct flb_azure_k /* avoid destorying if function is successful */ if (!node) { - flb_hash_destroy(kv); + flb_hash_table_destroy(kv); } } else { diff --git a/plugins/out_azure_kusto/azure_kusto_ingest.c b/plugins/out_azure_kusto/azure_kusto_ingest.c index fc4f8e22cde..309b6ae63b3 100644 --- a/plugins/out_azure_kusto/azure_kusto_ingest.c +++ b/plugins/out_azure_kusto/azure_kusto_ingest.c @@ -94,14 +94,14 @@ static flb_sds_t azure_kusto_create_blob_uri(struct flb_azure_kusto *ctx, char *blob_sas; size_t blob_sas_size; - ret = flb_hash_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_URI, 3, + ret = flb_hash_table_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_URI, 3, (void **)&blob_uri, &blob_uri_size); if (ret == -1) { flb_plg_error(ctx->ins, "error getting blob uri"); return NULL; } - ret = flb_hash_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_SAS, 3, + ret = flb_hash_table_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_SAS, 3, (void **)&blob_sas, &blob_sas_size); if (ret == -1) { flb_plg_error(ctx->ins, "error getting blob sas token"); @@ -295,14 +295,14 @@ static flb_sds_t azure_kusto_create_queue_uri(struct flb_azure_kusto *ctx, char *queue_sas; size_t queue_sas_size; - ret = flb_hash_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_URI, 3, + ret = flb_hash_table_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_URI, 3, (void **)&queue_uri, &queue_uri_size); if (ret == -1) { flb_plg_error(ctx->ins, "error getting queue uri"); return NULL; } - ret = flb_hash_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_SAS, 3, + ret = flb_hash_table_get(u_node->ht, AZURE_KUSTO_RESOURCE_UPSTREAM_SAS, 3, (void **)&queue_sas, &queue_sas_size); if (ret == -1) { flb_plg_error(ctx->ins, "error getting queue sas token");