Skip to content

opentelemetry: logs: json decoding to msgpack perf improvement #11430

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
edsiper wants to merge 2 commits into
base: master
Choose a base branch
from
Open

opentelemetry: logs: json decoding to msgpack perf improvement #11430

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ad792ae
opentelemetry: logs opentelemetry: encode in-place to avoid extra copy
edsiper Feb 4, 2026
9b47af3
benchmarks: add opentelemetry perf test
edsiper Feb 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions benchmarks/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,9 @@ target_link_libraries(flb-bench-pack_json
${CMAKE_THREAD_LIBS_INIT}
)

add_executable(flb-bench-opentelemetry opentelemetry.c)
target_link_libraries(flb-bench-opentelemetry
fluent-bit-static
${CMAKE_THREAD_LIBS_INIT}
)

164 changes: 164 additions & 0 deletions benchmarks/opentelemetry.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */

/* Fluent Bit
* ==========
* Copyright (C) 2015-2026 The Fluent Bit Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

/*
* OpenTelemetry encoding benchmarks (OTLP logs, metrics, traces).
* Usage: flb-bench-opentelemetry -m <mode> -f <input_file> [-i iterations]
*
* Modes:
* otlp-json-logs OTLP-JSON logs -> Fluent Bit log events
* (flb_opentelemetry_logs_json_to_msgpack)
*/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>

#include <fluent-bit.h>
#include <fluent-bit/flb_log_event_encoder.h>
#include <fluent-bit/flb_opentelemetry.h>

static long diff_ns(struct timespec *s, struct timespec *e)
{
return (e->tv_sec - s->tv_sec) * 1000000000L + (e->tv_nsec - s->tv_nsec);
}

static int run_otlp_json_logs(char *json, size_t len, int iterations)
{
int i;
int ret;
struct timespec ts, te;
struct flb_log_event_encoder enc;
int error_status;
long d_ns;
uint64_t total_bytes;
double mibps;

ret = flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2);
if (ret != FLB_EVENT_ENCODER_SUCCESS) {
fprintf(stderr, "encoder init failed\n");
return -1;
}

ret = flb_opentelemetry_logs_json_to_msgpack(&enc, json, len, NULL, &error_status);
if (ret < 0) {
fprintf(stderr, "OTLP encode failed (error_status=%d), file must be OTLP-JSON (resourceLogs...)\n", error_status);
flb_log_event_encoder_destroy(&enc);
return -1;
}
flb_log_event_encoder_reset(&enc);

printf("Benchmarking OTLP-JSON logs encoding (flb_opentelemetry_logs_json_to_msgpack)\n");
printf("------------------------------------------------------------------------\n\n");
printf("Iterations : %d\n", iterations);
printf("JSON size : %zu bytes\n", len);
printf("------------------------------------------------------------------------\n\n");

clock_gettime(CLOCK_MONOTONIC, &ts);
for (i = 0; i < iterations; i++) {
flb_log_event_encoder_reset(&enc);
ret = flb_opentelemetry_logs_json_to_msgpack(&enc, json, len, NULL, &error_status);
if (ret < 0) {
fprintf(stderr, "OTLP encode failed at iteration %d\n", i);
flb_log_event_encoder_destroy(&enc);
return -1;
}
}
clock_gettime(CLOCK_MONOTONIC, &te);
flb_log_event_encoder_destroy(&enc);

d_ns = diff_ns(&ts, &te);
total_bytes = (uint64_t) len * (uint64_t) iterations;
mibps = (double) total_bytes / d_ns * 1e9 / (1024.0 * 1024.0);

printf("------------------------------------------------------------------------\n");
printf("Total time : %ld ns\n", d_ns);
printf("Per call : %ld ns\n", d_ns / (long) iterations);
printf("Throughput : %.2f MiB/s\n", mibps);
printf("------------------------------------------------------------------------\n");
return 0;
}

static void usage(const char *prog)
{
fprintf(stderr, "Usage: %s -m <mode> -f <input_file> [-i iterations]\n", prog);
fprintf(stderr, "\n");
fprintf(stderr, "Modes:\n");
fprintf(stderr, " otlp-json-logs OTLP-JSON logs encoding\n");
fprintf(stderr, "\n");
}

int main(int argc, char **argv)
{
int ret;
int iterations = 100;
char *input_file = NULL;
char *mode = NULL;
char *json;
size_t len;
int opt;

while ((opt = getopt(argc, argv, "f:i:m:")) != -1) {
switch (opt) {
case 'f':
input_file = optarg;
break;
case 'i':
iterations = atoi(optarg);
if (iterations <= 0) {
fprintf(stderr, "Invalid -i (iterations): %s\n", optarg);
return 1;
}
break;
case 'm':
mode = optarg;
break;
default:
usage(argv[0]);
return 1;
}
}

if (mode == NULL || input_file == NULL) {
fprintf(stderr, "Error: -m and -f are required.\n\n");
usage(argv[0]);
return 1;
}

ret = flb_utils_read_file(input_file, &json, &len);
if (ret != 0) {
fprintf(stderr, "error reading %s\n", input_file);
return 1;
}

if (strcmp(mode, "otlp-json-logs") == 0) {
ret = run_otlp_json_logs(json, len, iterations);
}
else {
fprintf(stderr, "Unknown mode: %s\n", mode);
usage(argv[0]);
free(json);
return 1;
}

free(json);
Comment on lines +158 to +162
Copy link

@coderabbitai coderabbitai bot Feb 4, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
# Search for flb_utils_read_file implementation
rg -n 'int flb_utils_read_file|flb_utils_read_file\s*\(' src/flb_utils.c -A 20 | head -80

Repository: fluent/fluent-bit

Length of output: 1257

🏁 Script executed:

#!/bin/bash
# Also check the function signature and how out_buf is allocated
rg -n 'out_buf\s*=' src/flb_utils.c -B 3 -A 1 | head -50

Repository: fluent/fluent-bit

Length of output: 250

🏁 Script executed:

#!/bin/bash
# Search for how buf is allocated in flb_utils_read_file_offset
rg -n 'flb_utils_read_file_offset' src/flb_utils.c -A 80 | grep -E 'flb_sds|buf\s*=' | head -20

Repository: fluent/fluent-bit

Length of output: 158

🏁 Script executed:

#!/bin/bash
# Search for flb_sds_t type definition and typical deallocation patterns
rg -n 'flb_sds_t|flb_sds_destroy|flb_sds_free' --type-list | head -5 && rg 'flb_sds_destroy|flb_sds_free' -n | head -10

Repository: fluent/fluent-bit

Length of output: 1084

🏁 Script executed:

#!/antml:base64
# Check the opentelemetry.c file to see the full context of how json is used
cat -n benchmarks/opentelemetry.c | sed -n '140,170p'

Repository: fluent/fluent-bit

Length of output: 843

🏁 Script executed:

#!/bin/bash
# Check flb_calloc and flb_free definitions
rg -n 'define flb_calloc|define flb_free|^\s*flb_calloc|^\s*flb_free' src/flb_mem.h -A 2 | head -30

Repository: fluent/fluent-bit

Length of output: 114

🏁 Script executed:

#!/bin/bash
# Check how flb_utils_read_file results are freed in other parts of the codebase
rg -n 'flb_utils_read_file' src -A 5 | grep -E 'flb_utils_read_file|free\(|flb_free\(' | head -30

Repository: fluent/fluent-bit

Length of output: 450

🏁 Script executed:

#!/bin/bash
# Find flb_calloc and flb_free definitions
fd -e h -e c | xargs rg -l 'flb_calloc|flb_free' | head -20

Repository: fluent/fluent-bit

Length of output: 759

🏁 Script executed:

#!/bin/bash
# Search for flb_calloc definition more broadly
rg -n 'flb_calloc' -t c -t h | grep -E 'define|^[^:]*:\s*(static\s+)?(inline\s+)?.*flb_calloc' | head -20

Repository: fluent/fluent-bit

Length of output: 1927

🏁 Script executed:

#!/bin/bash
# Look at include/fluent_bit.h or similar
find . -name "*.h" -path "*/include/*" | head -20

Repository: fluent/fluent-bit

Length of output: 860

🏁 Script executed:

#!/bin/bash
# Search for flb_calloc and flb_free definitions in include directory
rg -n 'flb_calloc|flb_free' include/fluent-bit/ | head -30

Repository: fluent/fluent-bit

Length of output: 2465

🏁 Script executed:

#!/bin/bash
# Look for memory management header file
find include -name "*mem*" -o -name "*alloc*"

Repository: fluent/fluent-bit

Length of output: 89

🏁 Script executed:

#!/antml:base64
# Directly check the main fluent-bit.h header
head -100 include/fluent-bit.h | cat -n

Repository: fluent/fluent-bit

Length of output: 1804

🏁 Script executed:

#!/bin/bash
# View flb_mem.h to see the definitions of flb_calloc and flb_free
cat -n include/fluent-bit/flb_mem.h | sed -n '80,135p'

Repository: fluent/fluent-bit

Length of output: 1407

Use flb_free() for consistency with Fluent Bit's memory management patterns.

The buffer returned by flb_utils_read_file is freed with free() at lines 158 and 162. Since the buffer is allocated internally using flb_calloc (in flb_utils_read_file_offset), it should be freed with flb_free() instead to maintain consistency with Fluent Bit's memory management conventions throughout the codebase.

🤖 Prompt for AI Agents 
In `@benchmarks/opentelemetry.c` around lines 158 - 162, The buffer returned by
flb_utils_read_file (allocated via flb_calloc inside flb_utils_read_file_offset)
is being deallocated with free(json) in the function handling the JSON buffer;
replace both occurrences of free(json) with flb_free(json) to use Fluent Bit's
allocator/deallocator pair and keep memory management consistent (look for the
variable json in the function that calls flb_utils_read_file and update its two
free(json) calls to flb_free(json)).

return ret != 0 ? 1 : 0;
}
Loading
Loading