diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 98d679777..9928a0eae 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -35,15 +35,17 @@ jobs:
packages: "libsqlite3-dev"
run_tests: true
- - name: "Base + Admin API"
- cmake_flags: "-DCMAKE_BUILD_TYPE=Release -DENABLE_ADMIN_API=ON"
+ - name: "Base + Admin API + OTEL"
+ cmake_flags: "-DCMAKE_BUILD_TYPE=Release -DENABLE_ADMIN_API=ON -DWITH_OTEL=ON -DCMAKE_PREFIX_PATH=/opt/opentelemetry-cpp"
packages: ""
run_tests: true
+ with_otel: true
- - name: "YAML + Admin API"
- cmake_flags: "-DCMAKE_BUILD_TYPE=Release -DHAVE_YAML=ON -DENABLE_ADMIN_API=ON"
+ - name: "YAML + Admin API + OTEL"
+ cmake_flags: "-DCMAKE_BUILD_TYPE=Release -DHAVE_YAML=ON -DENABLE_ADMIN_API=ON -DWITH_OTEL=ON -DCMAKE_PREFIX_PATH=/opt/opentelemetry-cpp"
packages: "libyaml-cpp-dev"
run_tests: true
+ with_otel: true
steps:
- uses: actions/checkout@v4
@@ -52,6 +54,7 @@ jobs:
- name: Install base dependencies
run: |
+ sudo add-apt-repository universe -y
sudo apt-get update
sudo apt-get install -y \
build-essential \
@@ -72,6 +75,36 @@ jobs:
run: |
sudo apt-get install -y ${{ matrix.config.packages }}
+ - name: Install opentelemetry-cpp runtime dependencies
+ if: matrix.config.with_otel
+ run: |
+ sudo apt-get install -y libprotobuf-dev
+
+ - name: Cache opentelemetry-cpp
+ if: matrix.config.with_otel
+ id: cache-otel
+ uses: actions/cache@v4
+ with:
+ path: /opt/opentelemetry-cpp
+ key: otel-cpp-1.24.0-ubuntu-x64
+
+ - name: Build opentelemetry-cpp
+ if: matrix.config.with_otel && steps.cache-otel.outputs.cache-hit != 'true'
+ run: |
+ sudo apt-get install -y libgrpc++-dev protobuf-compiler-grpc || true
+ wget -q https://github.com/open-telemetry/opentelemetry-cpp/archive/refs/tags/v1.24.0.tar.gz
+ tar xzf v1.24.0.tar.gz
+ cmake -S opentelemetry-cpp-1.24.0 -B otel-build \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_INSTALL_PREFIX=/opt/opentelemetry-cpp \
+ -DWITH_OTLP_HTTP=ON \
+ -DWITH_OTLP_GRPC=OFF \
+ -DBUILD_TESTING=OFF \
+ -DWITH_BENCHMARK=OFF \
+ -DWITH_EXAMPLES=OFF
+ cmake --build otel-build -j$(nproc)
+ sudo cmake --install otel-build
+
- name: Configure CMake
run: |
mkdir -p build
@@ -271,7 +304,15 @@ jobs:
git
wget
+ - name: Cache Cygwin libraries
+ id: cache-cygwin-libs
+ uses: actions/cache@v4
+ with:
+ path: C:\cygwin\usr\local
+ key: cygwin-gtest-1.14.0
+
- name: Build and install googletest from source
+ if: steps.cache-cygwin-libs.outputs.cache-hit != 'true'
shell: C:\cygwin\bin\bash.exe --login --norc -eo pipefail -o igncr '{0}'
run: |
cd /tmp
@@ -330,7 +371,15 @@ jobs:
git
wget
+ - name: Cache Cygwin libraries
+ id: cache-cygwin-libs
+ uses: actions/cache@v4
+ with:
+ path: C:\cygwin\usr\local
+ key: cygwin-gtest-1.14.0-yaml-0.7.0
+
- name: Build and install googletest from source
+ if: steps.cache-cygwin-libs.outputs.cache-hit != 'true'
shell: C:\cygwin\bin\bash.exe --login --norc -eo pipefail -o igncr '{0}'
run: |
cd /tmp
@@ -343,6 +392,7 @@ jobs:
make install
- name: Build and install yaml-cpp from source
+ if: steps.cache-cygwin-libs.outputs.cache-hit != 'true'
shell: C:\cygwin\bin\bash.exe --login --norc -eo pipefail -o igncr '{0}'
run: |
cd /tmp
diff --git a/CLAUDE.md b/CLAUDE.md
index 1c2cfaf12..642d2d69e 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -351,10 +351,22 @@ Use separate build directories for different CMake configurations to avoid lengt
build/ - default build (without optional features)
build_sqlite/ - build with -DHAVE_SQLITE=ON
build_debug/ - debug build with -DCMAKE_BUILD_TYPE=Debug
+build_otel/ - build with -DWITH_OTEL=ON (requires vcpkg)
build_test/ - test data and converted worlds (not for compilation)
```
**Always warn the user when changing build directories or running cmake/make in a different directory.**
+### OpenTelemetry Build (WITH_OTEL)
+opentelemetry-cpp is installed via vcpkg at `~/repos/vcpkg`. Always pass the toolchain file and prefix path:
+```bash
+cmake -S . -B build_otel \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DWITH_OTEL=ON \
+ -DCMAKE_TOOLCHAIN_FILE=~/repos/vcpkg/scripts/buildsystems/vcpkg.cmake \
+ -DCMAKE_PREFIX_PATH=~/repos/vcpkg/installed/x64-linux
+make -C build_otel -j$(($(nproc)/2))
+```
+
### File Encoding - CRITICAL
**Proper workflow for editing KOI8-R files:**
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4794ba42d..666069074 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -10,7 +10,6 @@ set_property(GLOBAL PROPERTY USE_FOLDERS ON)
set(SOURCES
src/engine/structs/blocking_queue.cpp
- src/engine/db/influxdb.cpp
src/engine/core/heartbeat.cpp
src/engine/core/heartbeat_commands.cpp
src/gameplay/abilities/abilities_rollsystem.cpp
@@ -98,6 +97,8 @@ set(SOURCES
src/utils/levenshtein.cpp
src/gameplay/mechanics/liquid.cpp
src/utils/logger.cpp
+ src/utils/logging/file_log_sender.cpp
+ src/utils/logging/log_manager.cpp
src/gameplay/magic/magic.cpp
src/gameplay/magic/magic_items.cpp
src/gameplay/magic/magic_rooms.cpp
@@ -158,6 +159,8 @@ set(SOURCES
src/engine/structs/flags.hpp
src/utils/id_converter.cpp
src/utils/utils_time.cpp
+ src/utils/tracing/trace_manager.cpp
+ src/engine/observability/otel_trace_sender.cpp
src/utils/thread_pool.cpp
src/gameplay/mechanics/title.cpp
src/gameplay/statistics/top.cpp
@@ -461,7 +464,12 @@ set(SOURCES
src/engine/db/player_index.cpp
src/gameplay/skills/addshot.cpp
src/gameplay/classes/mob_classes_info.cpp
- src/gameplay/classes/recalc_mob_params_by_vnum.cpp)
+ src/gameplay/classes/recalc_mob_params_by_vnum.cpp
+ src/engine/observability/otel_provider.cpp
+ src/engine/observability/otel_traces.cpp
+ src/engine/observability/otel_metrics.cpp
+ src/engine/observability/otel_helpers.cpp
+ src/engine/observability/otel_log_sender.cpp)
@@ -511,7 +519,6 @@ set(HEADERS
src/administration/accounts.h
src/engine/core/action_targeting.h
src/engine/structs/blocking_queue.h
- src/engine/db/influxdb.h
src/engine/core/heartbeat_commands.h
src/gameplay/mechanics/weather.h
src/gameplay/core/game_limits.h
@@ -605,6 +612,9 @@ set(HEADERS
src/utils/levenshtein.h
src/gameplay/mechanics/liquid.h
src/utils/logger.h
+ src/utils/logging/log_sender.h
+ src/utils/logging/file_log_sender.h
+ src/utils/logging/log_manager.h
src/gameplay/magic/magic.h
src/gameplay/magic/magic_items.h
src/gameplay/magic/magic_rooms.h
@@ -662,6 +672,10 @@ set(HEADERS
src/engine/core/sysdep.h
src/engine/network/telnet.h
src/utils/utils_time.h
+ src/utils/tracing/trace_sender.h
+ src/utils/tracing/noop_trace_sender.h
+ src/utils/tracing/trace_manager.h
+ src/engine/observability/otel_trace_sender.h
src/gameplay/mechanics/title.h
src/gameplay/statistics/top.h
src/utils/utils.h
@@ -898,6 +912,12 @@ set(HEADERS
src/gameplay/mechanics/tutelar.h
src/gameplay/skills/addshot.h
src/engine/db/player_index.h
+ src/engine/observability/otel_provider.h
+ src/engine/observability/otel_traces.h
+ src/engine/observability/otel_metrics.h
+ src/engine/observability/otel_helpers.h
+ src/engine/observability/otel_log_sender.h
+ src/utils/logging/log_sender.h
src/gameplay/classes/recalc_mob_params_by_vnum.h)
# Build types
@@ -1204,6 +1224,29 @@ else ()
message(STATUS "SQLite is turned off.")
endif ()
+# OpenTelemetry support
+if (WITH_OTEL)
+ message(STATUS "OpenTelemetry integration: ENABLED")
+
+ # Find OpenTelemetry from vcpkg
+ find_package(opentelemetry-cpp CONFIG REQUIRED)
+
+ # Define WITH_OTEL for preprocessor
+ add_definitions(-DWITH_OTEL)
+
+ # Link OpenTelemetry libraries
+ target_link_libraries(circle.library
+ opentelemetry-cpp::api
+ opentelemetry-cpp::sdk
+ opentelemetry-cpp::ext
+ opentelemetry-cpp::otlp_http_exporter
+ opentelemetry-cpp::otlp_http_metric_exporter
+ opentelemetry-cpp::otlp_http_log_record_exporter
+ opentelemetry-cpp::resources
+ )
+else()
+ message(STATUS "OpenTelemetry integration: DISABLED")
+endif()
# YAML support
if (HAVE_YAML)
# Try to find yaml-cpp via CMake config first
@@ -1330,7 +1373,7 @@ if (UNIX AND NOT CYGWIN)
set(DEFAULT_WITH_ASAN YES)
else ()
set(DEFAULT_WITH_ASAN NO)
-endif ()
+endif()
option(WITH_ASAN "Compile with ASAN" ${DEFAULT_WITH_ASAN})
if (WITH_ASAN)
@@ -1450,6 +1493,9 @@ if (BUILD_TESTS)
add_subdirectory(tests)
endif ()
+option(WITH_OTEL "Enable OpenTelemetry integration" OFF)
+
+# vim: set ts=4 sw=4 ai tw=0 noet syntax=cmake :
# =============================================================================
# Data directories setup for running server from build directory
diff --git a/lib/misc/configuration.xml b/lib/misc/configuration.xml
index af1d6489d..15cd05434 100644
--- a/lib/misc/configuration.xml
+++ b/lib/misc/configuration.xml
@@ -114,4 +114,40 @@
-->
+
+
+ true
+
+
+ http://localhost:4318/v1/metrics
+
+
+ http://localhost:4318/v1/traces
+
+
+ http://localhost:4318/v1/logs
+
+
+
+ bylins-${host}-${port}
+ 1.0.0
+
+
+ duplicate
+
+
\ No newline at end of file
diff --git a/src/engine/core/comm.cpp b/src/engine/core/comm.cpp
index 58e407408..c5bcf3b48 100644
--- a/src/engine/core/comm.cpp
+++ b/src/engine/core/comm.cpp
@@ -37,6 +37,7 @@
#include "engine/db/world_characters.h"
#include "engine/entities/entities_constants.h"
#include "administration/shutdown_parameters.h"
+#include "engine/observability/otel_provider.h"
#include "external_trigger.h"
#include "handler.h"
#include "gameplay/clans/house.h"
@@ -673,7 +674,9 @@ int main_function(int argc, char **argv) {
" -h Print this command line argument help.\n"
" -o Write log to instead of stderr.\n"
" -r Restrict MUD -- no new players allowed.\n"
- " -s Suppress special procedure assignments.\n", argv[0]);
+ " -s Suppress special procedure assignments.\n"
+ "\n"
+ " -S Use SQLite database for world loading.\n", argv[0]);
exit(0);
default: printf("SYSERR: Unknown option -%c in argument string.\n", *(argv[pos] + 1));
@@ -683,7 +686,9 @@ int main_function(int argc, char **argv) {
" -h Print this command line argument help.\n"
" -o Write log to instead of stderr.\n"
" -r Restrict MUD -- no new players allowed.\n"
- " -s Suppress special procedure assignments.\n", argv[0]);
+ " -s Suppress special procedure assignments.\n"
+ "\n"
+ " -S Use SQLite database for world loading.\n", argv[0]);
exit(1);
break;
}
@@ -720,6 +725,7 @@ int main_function(int argc, char **argv) {
// directories are created in the working directory (next to the binary),
// not inside the data directory.
runtime_config.setup_logs();
+ runtime_config.setup_telemetry(port);
logfile = runtime_config.logs(SYSLOG).handle();
if (chdir(dir) < 0) {
perror("\r\nSYSERR: Fatal error changing to data directory");
@@ -759,6 +765,11 @@ void stop_game(ush_int port) {
log("Opening mother connection.");
mother_desc = init_socket(port);
+ if (mother_desc < 0) {
+ log("SYSERR: Failed to bind to port %d. Server cannot start.", port);
+ log("Please check if another instance is running or if you have permission to use this port.");
+ exit(1);
+ }
#ifdef ENABLE_ADMIN_API
if (runtime_config.admin_api_enabled()) {
@@ -826,6 +837,9 @@ void stop_game(ush_int port) {
game_loop(mother_desc);
#endif
+ // Shutdown OTEL providers to flush remaining telemetry
+ observability::OtelProvider::Instance().Shutdown();
+
FlushPlayerIndex();
// храны надо сейвить до Crash_save_all_rent(), иначе будем брать бабло у чара при записи
@@ -1009,9 +1023,9 @@ socket_t init_socket(ush_int port) {
sa.sin_addr = *(get_bind_addr());
if (bind(s, (struct sockaddr *) &sa, sizeof(sa)) < 0) {
- perror("SYSERR: bind");
+ log("SYSERR: bind() failed - port %d is already in use or permission denied", port);
CLOSE_SOCKET(s);
- exit(1);
+ return -1;
}
nonblock(s);
listen(s, 5);
@@ -2178,8 +2192,7 @@ RETSIGTYPE checkpointing(int/* sig*/) {
}
RETSIGTYPE hupsig(int/* sig*/) {
- log("SYSERR: Received SIGHUP, SIGINT, or SIGTERM. Shutting down...");
- exit(1); // perhaps something more elegant should substituted
+ shutdown_parameters.shutdown_now();
}
#endif // CIRCLE_UNIX
diff --git a/src/engine/core/config.cpp b/src/engine/core/config.cpp
index d9a896ad5..3b45b1a33 100644
--- a/src/engine/core/config.cpp
+++ b/src/engine/core/config.cpp
@@ -22,7 +22,12 @@
#include "engine/structs/meta_enum.h"
#if CIRCLE_UNIX
+#ifdef WITH_OTEL
+#include "engine/observability/otel_provider.h"
+#endif
+using ETelemetryLogMode = RuntimeConfiguration::ETelemetryLogMode;
#include
+#include
#endif
#include
@@ -720,6 +725,13 @@ RuntimeConfiguration::RuntimeConfiguration() :
m_msdp_debug(false),
m_changelog_file_name(Boards::constants::CHANGELOG_FILE_NAME),
m_changelog_format(Boards::constants::loader_formats::GIT),
+ m_telemetry_enabled(false),
+ m_telemetry_metrics_endpoint("http://localhost:4318/v1/metrics"),
+ m_telemetry_traces_endpoint("http://localhost:4318/v1/traces"),
+ m_telemetry_logs_endpoint("http://localhost:4318/v1/logs"),
+ m_telemetry_service_name("bylins-mud"),
+ m_telemetry_service_version("1.0.0"),
+ m_telemetry_log_mode(ETelemetryLogMode::kFileOnly),
m_yaml_threads(0) {
}
@@ -740,6 +752,8 @@ void RuntimeConfiguration::load_from_file(const char *filename) {
load_boards_configuration(&root);
load_external_triggers(&root);
load_statistics_configuration(&root);
+ load_telemetry_configuration_impl(&root);
+ load_telemetry_configuration(&root);
load_world_loader_configuration(&root);
#ifdef ENABLE_ADMIN_API
load_admin_api_configuration(&root);
@@ -796,6 +810,111 @@ bool CLogInfo::open() {
RuntimeConfiguration runtime_config;
+void RuntimeConfiguration::load_telemetry_configuration_impl(const pugi::xml_node *root) {
+ const auto telemetry = root->child("telemetry");
+ if (!telemetry) {
+ return;
+ }
+
+ const auto enabled = telemetry.child("enabled");
+ if (enabled) {
+ const std::string value = enabled.child_value();
+ m_telemetry_enabled = (value == "true" || value == "1");
+ }
+
+ const auto otlp = telemetry.child("otlp");
+ if (otlp) {
+ const auto metrics = otlp.child("metrics");
+ if (metrics) {
+ const auto endpoint = metrics.child("endpoint");
+ if (endpoint) {
+ m_telemetry_metrics_endpoint = endpoint.child_value();
+ }
+ }
+
+ const auto traces = otlp.child("traces");
+ if (traces) {
+ const auto endpoint = traces.child("endpoint");
+ if (endpoint) {
+ m_telemetry_traces_endpoint = endpoint.child_value();
+ }
+ }
+
+ const auto logs_otlp = otlp.child("logs_otlp");
+ if (logs_otlp) {
+ const auto endpoint = logs_otlp.child("endpoint");
+ if (endpoint) {
+ m_telemetry_logs_endpoint = endpoint.child_value();
+ }
+ }
+ }
+
+ const auto service = telemetry.child("service");
+ if (service) {
+ const auto name = service.child("name");
+ if (name) {
+ m_telemetry_service_name = name.child_value();
+ }
+ const auto version = service.child("version");
+ if (version) {
+ m_telemetry_service_version = version.child_value();
+ }
+ }
+
+ const auto logs = telemetry.child("logs");
+ if (logs) {
+ const auto mode = logs.child("mode");
+ if (mode) {
+ const std::string mode_str = mode.child_value();
+ if (mode_str == "file-only") {
+ m_telemetry_log_mode = ETelemetryLogMode::kFileOnly;
+ } else if (mode_str == "otel-only") {
+ m_telemetry_log_mode = ETelemetryLogMode::kOtelOnly;
+ } else if (mode_str == "duplicate") {
+ m_telemetry_log_mode = ETelemetryLogMode::kDuplicate;
+ }
+ }
+ }
+}
+
+void RuntimeConfiguration::load_telemetry_configuration(const pugi::xml_node *) {
+ // OtelProvider is initialized later via setup_telemetry(port)
+ // once the game port is known from command-line arguments.
+}
+
+void RuntimeConfiguration::setup_telemetry(int port) {
+#ifdef WITH_OTEL
+ if (!m_telemetry_enabled) {
+ return;
+ }
+
+ // Interpolate variables in service name: ${port}, ${host}, ${version}
+ char hostname[256] = "unknown";
+ gethostname(hostname, sizeof(hostname));
+
+ auto replace_all = [](std::string str, const std::string &var, const std::string &val) {
+ std::string::size_type pos;
+ while ((pos = str.find(var)) != std::string::npos) {
+ str.replace(pos, var.size(), val);
+ }
+ return str;
+ };
+
+ std::string name = m_telemetry_service_name;
+ name = replace_all(name, "${port}", std::to_string(port));
+ name = replace_all(name, "${host}", hostname);
+ name = replace_all(name, "${version}", m_telemetry_service_version);
+
+ observability::OtelProvider::Instance().Initialize(
+ m_telemetry_metrics_endpoint,
+ m_telemetry_traces_endpoint,
+ m_telemetry_logs_endpoint,
+ name,
+ m_telemetry_service_version);
+#else
+ (void)port;
+#endif
+}
// vim: ts=4 sw=4 tw=0 noet syntax=cpp :
#ifdef ENABLE_ADMIN_API
diff --git a/src/engine/core/config.h b/src/engine/core/config.h
index fab67821b..b93d19d58 100644
--- a/src/engine/core/config.h
+++ b/src/engine/core/config.h
@@ -161,6 +161,7 @@ class RuntimeConfiguration {
auto output_queue_size() const { return m_output_queue_size; }
void setup_logs();
+ void setup_telemetry(int port);
auto syslog_converter() const { return m_syslog_converter; }
void enable_logging() { m_logging_enabled = true; }
@@ -177,6 +178,16 @@ class RuntimeConfiguration {
const auto &statistics() const { return m_statistics; }
+ bool telemetry_enabled() const { return m_telemetry_enabled; }
+ const std::string &telemetry_metrics_endpoint() const { return m_telemetry_metrics_endpoint; }
+ const std::string &telemetry_traces_endpoint() const { return m_telemetry_traces_endpoint; }
+ const std::string &telemetry_logs_endpoint() const { return m_telemetry_logs_endpoint; }
+ const std::string &telemetry_service_name() const { return m_telemetry_service_name; }
+ const std::string &telemetry_service_version() const { return m_telemetry_service_version; }
+ enum class ETelemetryLogMode { kFileOnly, kOtelOnly, kDuplicate, kUndefined };
+ ETelemetryLogMode telemetry_log_mode() const { return m_telemetry_log_mode; }
+
+ void load_telemetry_configuration(const pugi::xml_node *root);
size_t yaml_threads() const { return m_yaml_threads; }
#ifdef ENABLE_ADMIN_API
@@ -203,6 +214,7 @@ class RuntimeConfiguration {
void load_boards_configuration(const pugi::xml_node *root);
void load_external_triggers(const pugi::xml_node *root);
void load_statistics_configuration(const pugi::xml_node *root);
+ void load_telemetry_configuration_impl(const pugi::xml_node *root);
void load_world_loader_configuration(const pugi::xml_node *root);
#ifdef ENABLE_ADMIN_API
void load_admin_api_configuration(const pugi::xml_node *root);
@@ -222,6 +234,14 @@ class RuntimeConfiguration {
std::string m_external_reboot_trigger_file_name;
StatisticsConfiguration m_statistics;
+
+ bool m_telemetry_enabled;
+ std::string m_telemetry_metrics_endpoint;
+ std::string m_telemetry_traces_endpoint;
+ std::string m_telemetry_logs_endpoint;
+ std::string m_telemetry_service_name;
+ std::string m_telemetry_service_version;
+ ETelemetryLogMode m_telemetry_log_mode;
size_t m_yaml_threads;
diff --git a/src/engine/core/heartbeat.cpp b/src/engine/core/heartbeat.cpp
index ad781dbf1..3e1ed030e 100644
--- a/src/engine/core/heartbeat.cpp
+++ b/src/engine/core/heartbeat.cpp
@@ -26,6 +26,8 @@
#include "gameplay/mechanics/corpse.h"
#include "engine/db/global_objects.h"
#include "engine/ui/cmd_god/do_set_all.h"
+#include "engine/observability/otel_traces.h"
+#include "engine/observability/otel_metrics.h"
#include "gameplay/statistics/money_drop.h"
#include "gameplay/mechanics/weather.h"
#include "utils/utils_time.h"
@@ -33,6 +35,7 @@
#include "gameplay/communication/check_invoice.h"
#include "gameplay/mechanics/depot.h"
#include "gameplay/statistics/spell_usage.h"
+#include "utils/tracing/trace_manager.h"
#if defined WITH_SCRIPTING
#include "scripting.hpp"
@@ -549,6 +552,15 @@ Heartbeat::Heartbeat() :
void Heartbeat::operator()(const int missed_pulses) {
pulse_label_t label;
+ // Capture current pulse numbers BEFORE advance
+ const auto current_heartbeat_number = global_pulse_number();
+ const auto current_pulse_number = pulse_number();
+
+ // Create trace span for this pulse
+ char span_name[64];
+ snprintf(span_name, sizeof(span_name), "Heartbeat #%lu pulse #%d", current_heartbeat_number, current_pulse_number);
+ auto pulse_span = tracing::TraceManager::Instance().StartSpan(span_name);
+
utils::CExecutionTimer timer;
pulse(missed_pulses, label);
const auto execution_time = timer.delta();
@@ -561,12 +573,42 @@ void Heartbeat::operator()(const int missed_pulses) {
mudlog(tmpbuf, LGH, kLvlImmortal, SYSLOG, true);
}
m_measurements.add(label, pulse_number(), execution_time.count());
- if (GlobalObjects::stats_sender().ready()) {
- influxdb::Record record("heartbeat");
- record.add_tag("pulse", pulse_number());
- record.add_field("duration", execution_time.count());
- GlobalObjects::stats_sender().send(record);
+
+#ifdef WITH_OTEL
+ static int debug_counter = 0;
+ if (++debug_counter % 250 == 0) { // Every 10 seconds
+ char debug_buf[256];
+ snprintf(debug_buf, sizeof(debug_buf), "DEBUG: Heartbeat OTEL called, label.size=%zu, pulse_mod=%d",
+ label.size(), pulse_number() % 25);
+ mudlog(debug_buf, CMP, kLvlImmortal, SYSLOG, true);
+ }
+ // 1. Metrics for each executed step
+ for (const auto& [step_index, step_time] : label) {
+ if (step_index < m_steps.size()) {
+ std::map step_attrs;
+ step_attrs["step"] = m_steps[step_index].name();
+ observability::OtelMetrics::RecordHistogram("heartbeat.step.duration", step_time, step_attrs);
+ }
}
+
+ // 2. Total pulse duration with modulo
+ std::map pulse_attrs;
+ pulse_attrs["pulse_mod"] = std::to_string(pulse_number() % 25);
+ observability::OtelMetrics::RecordHistogram("heartbeat.total.duration", execution_time.count(), pulse_attrs);
+
+ // 3. Record missed pulses if any
+ if (missed_pulses > 0) {
+ observability::OtelMetrics::RecordCounter("heartbeat.missed_pulses_total", missed_pulses);
+ }
+#endif
+
+ // Close parent span
+ pulse_span->SetAttribute("heartbeat_number", static_cast(current_heartbeat_number));
+ pulse_span->SetAttribute("pulse_number", static_cast(current_pulse_number));
+ pulse_span->SetAttribute("execution_time_seconds", execution_time.count());
+ pulse_span->SetAttribute("missed_pulses", static_cast(missed_pulses));
+ pulse_span->SetAttribute("steps_executed", static_cast(label.size()));
+ pulse_span->End();
}
long long Heartbeat::period() const {
@@ -618,7 +660,11 @@ void Heartbeat::pulse(const int missed_pulses, pulse_label_t &label) {
if (0 == (m_pulse_number + step.offset()) % step.modulo()) {
utils::CExecutionTimer timer;
-
+
+ // Create child span for this step
+ auto step_span = tracing::TraceManager::Instance().StartSpan(step.name());
+ step_span->SetAttribute("step_index", static_cast(i));
+ step_span->SetAttribute("step_modulo", static_cast(step.modulo()));
step.action()->perform(pulse_number(), missed_pulses);
const auto execution_time = timer.delta().count();
if (step.modulo() >= kSecsPerMudHour * kPassesPerSec) {
@@ -630,6 +676,8 @@ void Heartbeat::pulse(const int missed_pulses, pulse_label_t &label) {
log("HeartBeat memory resize, step:(%s), memory used: virt (%d kB) phys (%d kB)", step.name().c_str(), vmem_used, pmem_used);
// mudlog(buf, CMP, kLvlGreatGod, SYSLOG, true);
}
+ step_span->SetAttribute("execution_time_seconds", execution_time);
+ step_span->End();
label.emplace(i, execution_time);
m_executed_steps.insert(i);
step.add_measurement(i, pulse_number(), execution_time);
diff --git a/src/engine/db/db.cpp b/src/engine/db/db.cpp
index f752d3022..ae4de877f 100644
--- a/src/engine/db/db.cpp
+++ b/src/engine/db/db.cpp
@@ -37,12 +37,17 @@
#include "gameplay/mechanics/noob.h"
#include "obj_prototypes.h"
#include "engine/olc/olc.h"
+#include "engine/observability/otel_helpers.h"
+#include "engine/observability/otel_metrics.h"
+#include "utils/tracing/trace_manager.h"
#include "gameplay/communication/offtop.h"
#include "gameplay/communication/parcel.h"
#include "administration/privilege.h"
#include "gameplay/mechanics/sets_drop.h"
#include "gameplay/mechanics/stable_objs.h"
#include "gameplay/economics/shop_ext.h"
+#include "engine/observability/otel_metrics.h"
+#include "engine/observability/otel_traces.h"
#include "gameplay/mechanics/stuff.h"
#include "gameplay/mechanics/title.h"
#include "gameplay/statistics/top.h"
@@ -1932,6 +1937,11 @@ void ZoneUpdate() {
struct reset_q_element *update_u, *temp;
static int timer = 0;
utils::CExecutionTimer timer_count;
+ // OpenTelemetry: Track zone updates
+ auto zone_span = tracing::TraceManager::Instance().StartSpan("Zone Update");
+ observability::ScopedMetric zone_metric("zone.update.duration");
+
+ int zones_reset_count = 0;
if (((++timer * kPulseZone) / kPassesPerSec) >= 60) // one minute has passed
{
/*
@@ -1992,6 +2002,14 @@ void ZoneUpdate() {
ss << zone_table[it].vnum << " ";
if (zone_table[it].vnum < dungeons::kZoneStartDungeons) {
ResetZone(it);
+ zones_reset_count++;
+
+ // OpenTelemetry: Record zone reset
+ std::map attrs;
+ attrs["zone_vnum"] = std::to_string(zone_table[it].vnum);
+ attrs["reset_mode"] = std::to_string(zone_table[it].reset_mode);
+
+ observability::OtelMetrics::RecordCounter("zone.reset.total", 1, attrs);
} else {
log("Закрываю брошенный dungeon %d", it);
dungeons::DungeonReset(it);
@@ -2017,6 +2035,9 @@ void ZoneUpdate() {
if (k >= kZonesReset)
break;
}
+
+ // OpenTelemetry: Record total zones reset
+ zone_span->SetAttribute("zones_reset_count", static_cast(zones_reset_count));
}
bool CanBeReset(ZoneRnum zone) {
@@ -2307,18 +2328,15 @@ class ZoneReset {
void ZoneReset::Reset() {
utils::CExecutionTimer timer;
- if (GlobalObjects::stats_sender().ready()) {
- ResetZoneEssential();
- const auto execution_time = timer.delta();
+ ResetZoneEssential();
+ const auto execution_time = timer.delta();
- influxdb::Record record("zone_reset");
- record.add_tag("pulse", GlobalObjects::heartbeat().pulse_number());
- record.add_tag("zone", zone_table[m_zone_rnum].vnum);
- record.add_field("duration", execution_time.count());
- GlobalObjects::stats_sender().send(record);
- } else {
- ResetZoneEssential();
- }
+#ifdef WITH_OTEL
+ std::map attrs;
+ attrs["pulse"] = std::to_string(GlobalObjects::heartbeat().pulse_number());
+ attrs["zone"] = std::to_string(zone_table[m_zone_rnum].vnum);
+ observability::OtelMetrics::RecordHistogram("zone.reset.duration", execution_time.count(), attrs);
+#endif
}
bool ZoneReset::HandleZoneCmdQ(const MobRnum rnum) const {
@@ -2345,18 +2363,13 @@ bool ZoneReset::HandleZoneCmdQ(const MobRnum rnum) const {
const auto execution_time = overall_timer.delta();
- if (GlobalObjects::stats_sender().ready()) {
- influxdb::Record record("Q_command");
-
- record.add_tag("pulse", GlobalObjects::heartbeat().pulse_number());
- record.add_tag("zone", zone_table[m_zone_rnum].vnum);
- record.add_tag("rnum", rnum);
-
- record.add_field("duration", execution_time.count());
- record.add_field("extract", extract_time.count());
- record.add_field("get_mobs", get_mobs_time.count());
- GlobalObjects::stats_sender().send(record);
- }
+#ifdef WITH_OTEL
+ std::map attrs;
+ attrs["pulse"] = std::to_string(GlobalObjects::heartbeat().pulse_number());
+ attrs["zone"] = std::to_string(zone_table[m_zone_rnum].vnum);
+ attrs["rnum"] = std::to_string(rnum);
+ observability::OtelMetrics::RecordHistogram("zone.command.Q.duration", execution_time.count(), attrs);
+#endif
return extracted;
}
@@ -2912,6 +2925,12 @@ void SetGodSkills(CharData *ch) {
// по умолчанию reboot = 0 (пользуется только при ребуте)
int LoadPlayerCharacter(const char *name, CharData *char_element, int load_flags) {
const auto player_i = char_element->load_char_ascii(name, load_flags);
+ // OpenTelemetry: Track player loading
+ auto load_span = tracing::TraceManager::Instance().StartSpan("Load Player");
+ load_span->SetAttribute("character_name", std::string(name));
+
+ observability::ScopedMetric load_metric("player.load.duration");
+
if (player_i > -1) {
char_element->set_pfilepos(player_i);
}
diff --git a/src/engine/db/global_objects.cpp b/src/engine/db/global_objects.cpp
index 8f95fdc06..b2f2ddce2 100644
--- a/src/engine/db/global_objects.cpp
+++ b/src/engine/db/global_objects.cpp
@@ -3,6 +3,7 @@
#include
#include "administration/ban.h"
+#include "utils/logging/log_manager.h"
namespace {
// This struct defines order of creating and destroying global objects
@@ -12,6 +13,9 @@ struct GlobalObjectsStorage {
/// This object should be destroyed last because it serves all output operations. So I define it first.
std::shared_ptr output_thread;
+ /// LogManager intentionally leaked (never destroyed) to serve log() calls from destructors
+ logging::LogManager* log_manager;
+
celebrates::CelebrateList mono_celebrates;
celebrates::CelebrateList poly_celebrates;
celebrates::CelebrateList real_celebrates;
@@ -46,7 +50,6 @@ struct GlobalObjectsStorage {
InspectRequestDeque inspect_request_deque;
BanList *ban;
Heartbeat heartbeat;
- std::shared_ptr stats_sender;
ZoneTable zone_table;
DailyQuest::DailyQuestMap daily_quests;
Strengthening strengthening;
@@ -55,7 +58,9 @@ struct GlobalObjectsStorage {
};
GlobalObjectsStorage::GlobalObjectsStorage() :
+ log_manager(new logging::LogManager()),
ban(nullptr) {
+ // log_manager intentionally never deleted - will leak at shutdown
}
// This function ensures that global objects will be created at the moment of getting access to them
@@ -170,13 +175,8 @@ Heartbeat &GlobalObjects::heartbeat() {
return global_objects().heartbeat;
}
-influxdb::Sender &GlobalObjects::stats_sender() {
- if (!global_objects().stats_sender) {
- global_objects().stats_sender = std::make_shared(
- runtime_config.statistics().host(), runtime_config.statistics().port());
- }
-
- return *global_objects().stats_sender;
+observability::OtelProvider &GlobalObjects::otel_provider() {
+ return observability::OtelProvider::Instance();
}
OutputThread &GlobalObjects::output_thread() {
@@ -251,6 +251,9 @@ obj2triggers_t &GlobalObjects::obj_triggers() {
return global_objects().obj2triggers;
}
+logging::LogManager &GlobalObjects::log_manager() {
+ return *global_objects().log_manager;
+}
RoomDescriptions &GlobalObjects::descriptions() {
return global_objects().room_descriptions;
}
diff --git a/src/engine/db/global_objects.h b/src/engine/db/global_objects.h
index 37dafacb2..d105ff4ff 100644
--- a/src/engine/db/global_objects.h
+++ b/src/engine/db/global_objects.h
@@ -15,9 +15,11 @@
#include "engine/ui/cmd_god/do_inspect.h"
#include "engine/scripting/dg_event.h"
#include "gameplay/economics/shops_implementation.h"
+#include "engine/observability/otel_provider.h"
+#include "utils/logging/log_manager.h"
#include "world_objects.h"
#include "world_characters.h"
-#include "influxdb.h"
+#include "engine/observability/otel_provider.h"
#include "engine/entities/zone.h"
#include "gameplay/quests/daily_quest.h"
#include "gameplay/skills/skills_info.h"
@@ -70,8 +72,9 @@ class GlobalObjects {
static SetAllInspReqListType &setall_inspect_list();
static BanList *&ban();
static Heartbeat &heartbeat();
- static influxdb::Sender &stats_sender();
+ static observability::OtelProvider &otel_provider();
static OutputThread &output_thread();
+ static logging::LogManager &log_manager();
static ZoneTable &zone_table();
static RunestoneRoster &Runestones();
diff --git a/src/engine/db/influxdb.cpp b/src/engine/db/influxdb.cpp
deleted file mode 100644
index 3ca21092b..000000000
--- a/src/engine/db/influxdb.cpp
+++ /dev/null
@@ -1,124 +0,0 @@
-#include "influxdb.h"
-
-#include "utils/logger.h"
-
-#include
-
-#ifndef WIN32
-#include
-#include
-#include
-#include
-
-constexpr int INVALID_SOCKET = -1;
-constexpr int SOCKET_ERROR = -1;
-#endif
-
-namespace influxdb {
-class SenderImpl {
- public:
- SenderImpl(const std::string &host, const unsigned short port);
-
- bool ready() const { return !m_host.empty(); }
- bool send(const std::string &data);
-
- private:
- std::string m_host;
- int m_port;
-
- socket_t m_socket;
- struct sockaddr_in m_addr;
-};
-
-SenderImpl::SenderImpl(const std::string &host, const unsigned short port) :
- m_host(host),
- m_port(port),
- m_socket(INVALID_SOCKET) {
- memset(&m_addr, 0, sizeof(m_addr));
-
- if (m_host.empty()) {
- return;
- }
-
- m_addr.sin_family = AF_INET;
- m_addr.sin_port = htons(port);
-
- struct hostent *hp = gethostbyname(m_host.c_str());
- if (hp) {
- in_addr *server_address = reinterpret_cast(hp->h_addr_list[0]);
- log("Statistics server has been resolved to '%s'.\n",
- inet_ntoa(*server_address));
- memcpy(&m_addr.sin_addr, server_address, hp->h_length);
- } else {
- log("SYSERR: failed to resolve server name '%s'. Turning sending statistics off.\n", m_host.c_str());
- m_host.clear();
- }
-
- m_socket = socket(AF_INET, SOCK_DGRAM, 0);
- if (INVALID_SOCKET == m_socket) {
- log("SYSERR: Couldn't create UDP socket. Turning sending statistics off.\n");
- m_host.clear();
- }
-}
-
-bool SenderImpl::send(const std::string &data) {
- if (INVALID_SOCKET != m_socket) {
- const int result = sendto(m_socket, data.c_str(), static_cast(data.size()),
- 0, reinterpret_cast(&m_addr), sizeof(m_addr));
-
- return SOCKET_ERROR != result;
- }
-
- return false;
-}
-
-Sender::Sender(const std::string &host, const unsigned short port) :
- m_implementation(new SenderImpl(host, port)) {
-}
-
-Sender::~Sender() {
- delete m_implementation;
-}
-
-bool Sender::ready() const {
- return m_implementation->ready();
-}
-
-bool Sender::send(const Record &record) const {
- std::string data;
- if (!record.get_data(data)) {
- return false;
- }
-
- return m_implementation->send(data);
-}
-
-bool Record::get_data(std::string &data) const {
- if (m_fields.empty()) {
- log("SYSERR: Attempt to send statistics record without any field.\n");
- return false;
- }
-
- std::stringstream ss;
- ss << m_measurement;
- for (const auto &tag : m_tags) {
- ss << "," << tag;
- }
- ss << " ";
-
- bool first = true;
- for (const auto &field : m_fields) {
- ss << (first ? "" : ",") << field;
- first = false;
- }
-
- using namespace std::chrono;
- nanoseconds timestamp = duration_cast(system_clock::now().time_since_epoch());
- ss << " " << timestamp.count();
-
- data = ss.str();
- return true;
-}
-}
-
-// vim: ts=4 sw=4 tw=0 noet syntax=cpp :
diff --git a/src/engine/db/influxdb.h b/src/engine/db/influxdb.h
deleted file mode 100644
index ae45411eb..000000000
--- a/src/engine/db/influxdb.h
+++ /dev/null
@@ -1,56 +0,0 @@
-#ifndef __INFLUX_HPP__
-#define __INFLUX_HPP__
-
-#include
-#include
-#include
-
-namespace influxdb {
-class Record {
- public:
- Record(const std::string &measurement) : m_measurement(measurement) {}
-
- template
- Record &add_tag(const std::string &name, const T &value) { return add_to_list(m_tags, name, value); }
-
- template
- Record &add_field(const std::string &name, const T &value) { return add_to_list(m_fields, name, value); }
-
- bool get_data(std::string &data) const;
-
- private:
- using strings_list_t = std::list;
-
- template
- Record &add_to_list(strings_list_t &list, const std::string &name, const T &value);
-
- std::string m_measurement;
- strings_list_t m_tags;
- strings_list_t m_fields;
-};
-
-template
-Record &Record::add_to_list(strings_list_t &list, const std::string &name, const T &value) {
- std::stringstream ss;
- ss << name << "=" << value;
- list.push_back(ss.str());
-
- return *this;
-}
-
-class Sender {
- public:
- Sender(const std::string &host, const unsigned short port);
- ~Sender();
-
- bool ready() const;
- bool send(const Record &record) const;
-
- private:
- class SenderImpl *m_implementation;
-};
-}
-
-#endif // __INFLUX_HPP__
-
-// vim: ts=4 sw=4 tw=0 noet syntax=cpp :
diff --git a/src/engine/db/obj_save.cpp b/src/engine/db/obj_save.cpp
index 8043ae1b6..bc4c28217 100644
--- a/src/engine/db/obj_save.cpp
+++ b/src/engine/db/obj_save.cpp
@@ -27,6 +27,9 @@
#include "player_index.h"
#include
+#include "engine/observability/otel_helpers.h"
+#include "engine/observability/otel_metrics.h"
+#include "utils/tracing/trace_manager.h"
#include
const int LOC_INVENTORY = 0;
@@ -2435,6 +2438,12 @@ int receptionist(CharData *ch, void *me, int cmd, char *argument) {
void Crash_frac_save_all(int frac_part) {
DescriptorData *d;
+ // OpenTelemetry: Track fractional save
+ auto save_span = tracing::TraceManager::Instance().StartSpan("Player Save (Fractional)");
+ save_span->SetAttribute("save_type", "frac");
+ save_span->SetAttribute("frac_part", static_cast(frac_part));
+
+ int saved_count = 0;
for (d = descriptor_list; d; d = d->next) {
if ((d->state == EConState::kPlaying) && !d->character->IsNpc() && GET_ACTIVITY(d->character) == frac_part) {
@@ -2449,12 +2458,24 @@ void Crash_frac_save_all(int frac_part) {
if (timer1.delta().count() > 0.1)
log("Crash_frac_save_all: save_char, timer %f, save player: %s", timer1.delta().count(), d->character->get_name().c_str());
d->character->UnsetFlag(EPlrFlag::kCrashSave);
+ saved_count++;
+
+ // OpenTelemetry: Record save metrics
+ std::map attrs;
+ attrs["save_type"] = "frac";
+ attrs["character"] = d->character->get_name();
+
+ observability::OtelMetrics::RecordHistogram("player.save.duration", timer.delta().count(), attrs);
+ observability::OtelMetrics::RecordCounter("player.save.total", 1, attrs);
}
}
}
void Crash_save_all(void) {
DescriptorData *d;
+ auto save_span = tracing::TraceManager::Instance().StartSpan("Player Save (Full)");
+ save_span->SetAttribute("save_type", "full");
+
for (d = descriptor_list; d; d = d->next) {
if ((d->state == EConState::kPlaying) && d->character->IsFlagged(EPlrFlag::kCrashSave)) {
Crash_crashsave(d->character.get());
diff --git a/src/engine/entities/char_data.cpp b/src/engine/entities/char_data.cpp
index e79dfb4d5..e22c012e4 100644
--- a/src/engine/entities/char_data.cpp
+++ b/src/engine/entities/char_data.cpp
@@ -2,6 +2,7 @@
// Copyright (c) 2008 Krodo
// Part of Bylins http://www.mud.ru
+#include "engine/observability/otel_helpers.h"
#include "engine/core/handler.h"
#include "administration/privilege.h"
#include "char_player.h"
@@ -17,6 +18,7 @@
#include "gameplay/statistics/money_drop.h"
#include "gameplay/affects/affect_data.h"
#include "gameplay/mechanics/illumination.h"
+#include "utils/tracing/trace_sender.h"
#include "engine/ui/alias.h"
#include
diff --git a/src/engine/entities/char_data.h b/src/engine/entities/char_data.h
index 00e1bf54f..d19146a1d 100644
--- a/src/engine/entities/char_data.h
+++ b/src/engine/entities/char_data.h
@@ -27,6 +27,10 @@
#include
#include
#include