From 702a072f686fcb6b274b258ad78a6463cfb0dbe5 Mon Sep 17 00:00:00 2001 From: Logic Date: Sat, 14 Mar 2026 11:26:02 +0800 Subject: [PATCH 1/4] feat: add MySQL R2DBC query engine support and update documentation --- README.md | 3 +- README_CN.md | 3 +- README_JP.md | 3 +- .../hertzbeat-collector-basic/pom.xml | 6 - .../collect/database/JdbcCommonCollect.java | 267 ++++++++---- .../database/query/JdbcQueryExecutor.java | 30 ++ .../query/JdbcQueryExecutorRegistry.java | 54 +++ .../database/query/JdbcQueryRowSet.java | 33 ++ .../hertzbeat-collector-collector/pom.xml | 31 +- .../mysql/MysqlCollectorProperties.java | 53 +++ .../mysql/MysqlJdbcDriverAvailability.java | 80 ++++ .../mysql/MysqlR2dbcJdbcQueryExecutor.java | 214 ++++++++++ .../strategy/CollectStrategyFactory.java | 10 + .../src/main/resources/application.yml | 7 + ...bcQueryAdapterTemplateIntegrationTest.java | 320 ++++++++++++++ .../MysqlJdbcDriverAvailabilityTest.java | 41 ++ ...ryAdapterCompatibilityIntegrationTest.java | 312 ++++++++++++++ ...bcQueryAdapterTemplateIntegrationTest.java | 322 ++++++++++++++ .../MysqlJdbcQueryParityIntegrationTest.java | 393 ++++++++++++++++++ .../MysqlR2dbcJdbcQueryExecutorTest.java | 143 +++++++ ...anbaseJdbcQueryAdapterIntegrationTest.java | 275 ++++++++++++ .../TidbJdbcQueryAdapterIntegrationTest.java | 240 +++++++++++ .../strategy/CollectStrategyFactoryTest.java | 34 ++ .../hertzbeat-collector-mysql-r2dbc/pom.xml | 77 ++++ .../mysql/r2dbc/MysqlQueryExecutor.java | 33 ++ .../mysql/r2dbc/MysqlR2dbcConfiguration.java | 55 +++ .../MysqlR2dbcConnectionFactoryProvider.java | 67 +++ .../mysql/r2dbc/MysqlR2dbcQueryExecutor.java | 156 +++++++ .../collector/mysql/r2dbc/QueryOptions.java | 60 +++ .../collector/mysql/r2dbc/QueryResult.java | 46 ++ .../mysql/r2dbc/ResultSetMapper.java | 95 +++++ .../collector/mysql/r2dbc/SqlGuard.java | 85 ++++ ...ysqlR2dbcQueryExecutorIntegrationTest.java | 201 +++++++++ .../MysqlSqlTemplateCompatibilityTest.java | 65 +++ .../mysql/r2dbc/ResultSetMapperTest.java | 81 ++++ .../collector/mysql/r2dbc/SqlGuardTest.java | 53 +++ hertzbeat-collector/pom.xml | 14 + .../pom.xml | 124 ++++++ .../AbstractMysqlR2dbcCollectE2eTest.java | 233 +++++++++++ ...MysqlR2dbcCollectCompatibilityE2eTest.java | 61 +++ .../mysql/MysqlR2dbcCollectE2eTest.java | 41 ++ hertzbeat-e2e/pom.xml | 1 + hertzbeat-startup/pom.xml | 18 + .../src/main/resources/application.yml | 7 + .../ReactorNettyCompatibilityTest.java | 54 +++ .../StartupMysqlR2dbcCompatibilityTest.java | 116 ++++++ home/docs/download.md | 2 +- home/docs/help/mariadb.md | 18 +- home/docs/help/mysql.md | 19 +- home/docs/help/oceanbase.md | 17 +- home/docs/help/tidb.md | 16 + home/docs/start/docker-deploy.md | 6 +- home/docs/start/native-collector.md | 8 +- home/docs/start/package-deploy.md | 7 +- home/docs/start/quickstart.md | 4 +- .../current/download.md | 2 +- .../current/help/mariadb.md | 18 +- .../current/help/mysql.md | 18 +- .../current/help/oceanbase.md | 17 +- .../current/help/tidb.md | 16 + .../current/start/docker-deploy.md | 6 +- .../current/start/native-collector.md | 8 +- .../current/start/package-deploy.md | 7 +- .../current/start/quickstart.md | 4 +- material/licenses/NOTICE | 16 + material/licenses/backend/LICENSE | 3 + material/licenses/collector/LICENSE | 3 + material/licenses/collector/NOTICE | 16 + pom.xml | 24 ++ script/application.yml | 7 + script/docker-compose/README.md | 7 +- .../hertzbeat-mysql-iotdb/README.md | 7 +- .../hertzbeat-mysql-iotdb/README_CN.md | 7 +- .../conf/application.yml | 7 + .../hertzbeat-mysql-iotdb/docker-compose.yaml | 1 + .../hertzbeat-mysql-iotdb/ext-lib/README | 7 +- .../hertzbeat-mysql-tdengine/README.md | 7 +- .../hertzbeat-mysql-tdengine/README_CN.md | 7 +- .../conf/application.yml | 7 + .../docker-compose.yaml | 1 + .../hertzbeat-mysql-tdengine/ext-lib/README | 7 +- .../README.md | 7 +- .../README_CN.md | 7 +- .../conf/application.yml | 7 + .../docker-compose.yaml | 1 + .../ext-lib/README | 7 +- .../hertzbeat-postgresql-greptimedb/README.md | 8 +- .../README_CN.md | 7 +- .../conf/application.yml | 7 + .../docker-compose.yaml | 1 + .../ext-lib/README | 7 +- .../README.md | 8 +- .../README_CN.md | 7 +- .../conf/application.yml | 7 + .../docker-compose.yaml | 1 + .../ext-lib/README | 7 +- script/ext-lib/README | 6 +- 97 files changed, 4842 insertions(+), 197 deletions(-) create mode 100644 hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutor.java create mode 100644 hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutorRegistry.java create mode 100644 hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryRowSet.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlCollectorProperties.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailability.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutor.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MariadbJdbcQueryAdapterTemplateIntegrationTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailabilityTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterCompatibilityIntegrationTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterTemplateIntegrationTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryParityIntegrationTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutorTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/OceanbaseJdbcQueryAdapterIntegrationTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/TidbJdbcQueryAdapterIntegrationTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactoryTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/pom.xml create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlQueryExecutor.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConfiguration.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConnectionFactoryProvider.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutor.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryOptions.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryResult.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapper.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuard.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutorIntegrationTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlSqlTemplateCompatibilityTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapperTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuardTest.java create mode 100644 hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/pom.xml create mode 100644 hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/AbstractMysqlR2dbcCollectE2eTest.java create mode 100644 hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectCompatibilityE2eTest.java create mode 100644 hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectE2eTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ReactorNettyCompatibilityTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/StartupMysqlR2dbcCompatibilityTest.java diff --git a/README.md b/README.md index e6b5b7dbf7a..eb8cac645f8 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,8 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache manager-host: ${MANAGER_HOST:127.0.0.1} manager-port: ${MANAGER_PORT:1158} ``` - - If you need MySQL, OceanBase, Oracle, or DB2 monitoring with external JDBC drivers from `ext-lib`, use the JVM collector package. + - If you do not provide JDBC drivers in `ext-lib`, MySQL, MariaDB, and OceanBase can use the built-in query engine and run on the native collector package as well. TiDB follows the same rule for its SQL query metric set. + - If `mysql-connector-j` is present in `ext-lib`, the built-in server collector or JVM collector automatically prefers JDBC after restart for MySQL, MariaDB, and OceanBase. TiDB follows the same rule for its SQL query metric set, while its HTTP metrics are unchanged. Oracle and DB2 still require the JVM collector package because they depend on external JDBC drivers. - Run `$ ./bin/startup.sh ` or `bin/startup.bat` for the JVM collector package. Run `$ ./bin/startup.sh ` for Linux or macOS native collector packages, and `bin\\startup.bat` for the Windows native collector package. - Access `http://localhost:1157` and you will see the registered new collector in dashboard diff --git a/README_CN.md b/README_CN.md index fda8d7ca640..80058e8f7b4 100644 --- a/README_CN.md +++ b/README_CN.md @@ -145,7 +145,8 @@ manager-host: ${MANAGER_HOST:127.0.0.1} manager-port: ${MANAGER_PORT:1158} ``` - - 如果需要通过 `ext-lib` 加载 MySQL、OceanBase、Oracle、DB2 等外置 JDBC 驱动，请使用 JVM 采集器安装包。 + - 如果没有在 `ext-lib` 中提供 JDBC 驱动，MySQL、MariaDB、OceanBase 可以直接使用内置查询引擎，也可以使用 Native 采集器安装包；TiDB 的 SQL 查询指标也遵循同样规则。 + - 如果在 `ext-lib` 中放入了 `mysql-connector-j`，主程序内置采集器或 JVM 采集器会在重启后自动优先走 JDBC；这一点现在适用于 MySQL、MariaDB、OceanBase，TiDB 的 SQL 查询指标也遵循同样规则，而它的 HTTP 指标不受影响。Oracle、DB2 仍然必须使用 JVM 采集器安装包，因为它们依赖外置 JDBC 驱动。 - JVM 采集器安装包使用 `$ ./bin/startup.sh ` 或 `bin/startup.bat` 启动。Linux 或 macOS 的 Native 采集器安装包使用 `$ ./bin/startup.sh ` 启动，Windows 的 Native 采集器安装包使用 `bin\\startup.bat` 启动 - 浏览器访问主 HertzBeat 服务 `http://localhost:1157` 查看概览页面即可看到注册上来的新采集器 diff --git a/README_JP.md b/README_JP.md index f8993253319..4487344ce90 100644 --- a/README_JP.md +++ b/README_JP.md @@ -148,7 +148,8 @@ - `mode: ${MODE:public}`：実行モード(パブリッククラスタまたはプライベートクラウドエッジ)。 - `manager-host: ${MANAGER_HOST:127.0.0.1}`：メインhertzbeatサーバーのIP。 - `manager-port: ${MANAGER_PORT:1158}`：メインhertzbeatサーバポート。 - - `ext-lib` で MySQL、OceanBase、Oracle、DB2 などの外部 JDBC ドライバーを読み込む必要がある場合は、JVM コレクターのインストールパッケージを使用してください。 + - `ext-lib` に JDBC ドライバーを置かない場合、MySQL、MariaDB、OceanBase は組み込みのクエリエンジンを使って Native コレクターパッケージでも監視できます。TiDB も SQL クエリのメトリクスセットについては同じルールです。 + - `ext-lib` に `mysql-connector-j` を置いた場合は、再起動後に組み込みサーバーコレクターまたは JVM コレクターが MySQL、MariaDB、OceanBase で自動的に JDBC を優先します。TiDB も SQL クエリのメトリクスセットについては同じルールで、HTTP メトリクスは影響を受けません。Oracle と DB2 は引き続き外部 JDBC ドライバーに依存するため、JVM コレクターパッケージを使用してください。 - JVM コレクターのインストールパッケージは `$ ./bin/startup.sh` または `bin/startup.bat`、Linux/macOS の Native コレクターパッケージは `$ ./bin/startup.sh`、Windows の Native コレクターパッケージは `bin\\startup.bat` で起動します。 - メインの HertzBeat サービス `http://localhost:1157` にアクセスすると、登録された新しいコレクターを確認できます。 diff --git a/hertzbeat-collector/hertzbeat-collector-basic/pom.xml b/hertzbeat-collector/hertzbeat-collector-basic/pom.xml index de107950e41..63c88e262df 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-basic/pom.xml @@ -67,12 +67,6 @@ commons-net commons-net - - - com.mysql - mysql-connector-j - provided - com.clickhouse diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollect.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollect.java index 42529c72842..76e9421a6a2 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollect.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollect.java @@ -20,7 +20,6 @@ import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.DriverManager; -import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.HashMap; @@ -34,6 +33,9 @@ import org.apache.hertzbeat.collector.collect.common.cache.CacheIdentifier; import org.apache.hertzbeat.collector.collect.common.cache.GlobalConnectionCache; import org.apache.hertzbeat.collector.collect.common.cache.JdbcConnect; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutor; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutorRegistry; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryRowSet; import org.apache.hertzbeat.collector.collect.common.ssh.SshTunnelHelper; import org.apache.hertzbeat.collector.constants.CollectorConstants; import org.apache.hertzbeat.collector.dispatch.DispatchConstants; @@ -205,31 +207,26 @@ public void preCheck(Metrics metrics) throws IllegalArgumentException { public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) { long startTime = System.currentTimeMillis(); JdbcProtocol jdbcProtocol = metrics.getJdbc(); - SshTunnel sshTunnel = jdbcProtocol.getSshTunnel(); - int timeout = CollectUtil.getTimeout(jdbcProtocol.getTimeout()); boolean reuseConnection = Boolean.parseBoolean(jdbcProtocol.getReuseConnection()); - Statement statement = null; - String databaseUrl; try { - if (sshTunnel != null && Boolean.parseBoolean(sshTunnel.getEnable())) { - int localPort = SshTunnelHelper.localPortForward(sshTunnel, jdbcProtocol.getHost(), jdbcProtocol.getPort()); - databaseUrl = constructDatabaseUrl(jdbcProtocol, "localhost", String.valueOf(localPort)); - } else { - databaseUrl = constructDatabaseUrl(jdbcProtocol, jdbcProtocol.getHost(), jdbcProtocol.getPort()); - } - - statement = getConnection(jdbcProtocol.getUsername(), - jdbcProtocol.getPassword(), databaseUrl, timeout, reuseConnection); switch (jdbcProtocol.getQueryType()) { - case QUERY_TYPE_ONE_ROW -> queryOneRow(statement, jdbcProtocol.getSql(), metrics.getAliasFields(), builder, startTime); - case QUERY_TYPE_MULTI_ROW -> queryMultiRow(statement, jdbcProtocol.getSql(), metrics.getAliasFields(), builder, startTime); - case QUERY_TYPE_COLUMNS -> queryOneRowByMatchTwoColumns(statement, jdbcProtocol.getSql(), metrics.getAliasFields(), builder, startTime); - case RUN_SCRIPT -> { - Connection connection = statement.getConnection(); - FileSystemResource rc = new FileSystemResource(jdbcProtocol.getSql()); - ScriptUtils.executeSqlScript(connection, rc); + case QUERY_TYPE_ONE_ROW -> { + try (JdbcQueryRowSet rowSet = executeQuery(metrics, timeout, reuseConnection, 1)) { + queryOneRow(rowSet, metrics.getAliasFields(), builder, startTime); + } + } + case QUERY_TYPE_MULTI_ROW -> { + try (JdbcQueryRowSet rowSet = executeQuery(metrics, timeout, reuseConnection, 1000)) { + queryMultiRow(rowSet, metrics.getAliasFields(), builder, startTime); + } } + case QUERY_TYPE_COLUMNS -> { + try (JdbcQueryRowSet rowSet = executeQuery(metrics, timeout, reuseConnection, 1000)) { + queryOneRowByMatchTwoColumns(rowSet, metrics.getAliasFields(), builder, startTime); + } + } + case RUN_SCRIPT -> runScript(metrics, timeout, reuseConnection); default -> { builder.setCode(CollectRep.Code.FAIL); builder.setMsg("Not support database query type: " + jdbcProtocol.getQueryType()); @@ -261,23 +258,6 @@ public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) { log.error("Jdbc error: {}.", errorMessage, e); builder.setCode(CollectRep.Code.FAIL); builder.setMsg("Query Error: " + errorMessage); - } finally { - if (statement != null) { - Connection connection = null; - try { - connection = statement.getConnection(); - statement.close(); - } catch (Exception e) { - log.error("Jdbc close statement error: {}", e.getMessage()); - } - try { - if (!reuseConnection && connection != null) { - connection.close(); - } - } catch (Exception e) { - log.error("Jdbc close connection error: {}", e.getMessage()); - } - } } } @@ -286,6 +266,52 @@ public String supportProtocol() { return DispatchConstants.PROTOCOL_JDBC; } + private JdbcQueryRowSet executeQuery(Metrics metrics, int timeout, boolean reuseConnection, int maxRows) throws Exception { + Optional executor = JdbcQueryExecutorRegistry.resolve(metrics); + if (executor.isPresent()) { + return executor.get().executeQuery(metrics, timeout, maxRows); + } + return executeJdbcQuery(metrics.getJdbc(), timeout, reuseConnection, maxRows); + } + + private JdbcQueryRowSet executeJdbcQuery(JdbcProtocol jdbcProtocol, int timeout, boolean reuseConnection, + int maxRows) throws Exception { + Statement statement = null; + try { + String databaseUrl = resolveDatabaseUrl(jdbcProtocol); + statement = getConnection(jdbcProtocol.getUsername(), + jdbcProtocol.getPassword(), databaseUrl, timeout, reuseConnection); + statement.setMaxRows(maxRows); + return new ResultSetJdbcQueryRowSet(statement, statement.executeQuery(jdbcProtocol.getSql()), reuseConnection); + } catch (Exception exception) { + closeStatementAndConnection(statement, reuseConnection); + throw exception; + } + } + + private void runScript(Metrics metrics, int timeout, boolean reuseConnection) throws Exception { + JdbcProtocol jdbcProtocol = metrics.getJdbc(); + Statement statement = null; + try { + String databaseUrl = resolveDatabaseUrl(jdbcProtocol); + statement = getConnection(jdbcProtocol.getUsername(), + jdbcProtocol.getPassword(), databaseUrl, timeout, reuseConnection); + Connection connection = statement.getConnection(); + FileSystemResource rc = new FileSystemResource(jdbcProtocol.getSql()); + ScriptUtils.executeSqlScript(connection, rc); + } finally { + closeStatementAndConnection(statement, reuseConnection); + } + } + + private String resolveDatabaseUrl(JdbcProtocol jdbcProtocol) throws Exception { + SshTunnel sshTunnel = jdbcProtocol.getSshTunnel(); + if (sshTunnel != null && Boolean.parseBoolean(sshTunnel.getEnable())) { + int localPort = SshTunnelHelper.localPortForward(sshTunnel, jdbcProtocol.getHost(), jdbcProtocol.getPort()); + return constructDatabaseUrl(jdbcProtocol, "localhost", String.valueOf(localPort)); + } + return constructDatabaseUrl(jdbcProtocol, jdbcProtocol.getHost(), jdbcProtocol.getPort()); + } private Statement getConnection(String username, String password, String url, Integer timeout, boolean reuseConnection) throws Exception { CacheIdentifier identifier = CacheIdentifier.builder() @@ -343,29 +369,25 @@ private Statement getConnection(String username, String password, String url, In * query metrics：one tow three four * query sql：select one, tow, three, four from book limit 1; * - * @param statement statement - * @param sql sql + * @param rowSet row set * @param columns query metrics field list * @throws Exception when error happen */ - private void queryOneRow(Statement statement, String sql, List columns, + private void queryOneRow(JdbcQueryRowSet rowSet, List columns, CollectRep.MetricsData.Builder builder, long startTime) throws Exception { - statement.setMaxRows(1); - try (ResultSet resultSet = statement.executeQuery(sql)) { - if (resultSet.next()) { - CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); - for (String column : columns) { - if (CollectorConstants.RESPONSE_TIME.equals(column)) { - long time = System.currentTimeMillis() - startTime; - valueRowBuilder.addColumn(String.valueOf(time)); - } else { - String value = resultSet.getString(column); - value = value == null ? CommonConstants.NULL_VALUE : value; - valueRowBuilder.addColumn(value); - } + if (rowSet.next()) { + CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); + for (String column : columns) { + if (CollectorConstants.RESPONSE_TIME.equals(column)) { + long time = System.currentTimeMillis() - startTime; + valueRowBuilder.addColumn(String.valueOf(time)); + } else { + String value = rowSet.getString(column); + value = value == null ? CommonConstants.NULL_VALUE : value; + valueRowBuilder.addColumn(value); } - builder.addValueRow(valueRowBuilder.build()); } + builder.addValueRow(valueRowBuilder.build()); } } @@ -380,33 +402,30 @@ private void queryOneRow(Statement statement, String sql, List columns, * three - value3 * four - value4 * - * @param statement statement - * @param sql sql + * @param rowSet row set * @param columns query metrics field list * @throws Exception when error happen */ - private void queryOneRowByMatchTwoColumns(Statement statement, String sql, List columns, + private void queryOneRowByMatchTwoColumns(JdbcQueryRowSet rowSet, List columns, CollectRep.MetricsData.Builder builder, long startTime) throws Exception { - try (ResultSet resultSet = statement.executeQuery(sql)) { - HashMap values = new HashMap<>(columns.size()); - while (resultSet.next()) { - if (resultSet.getString(1) != null) { - values.put(resultSet.getString(1).toLowerCase().trim(), resultSet.getString(2)); - } + HashMap values = new HashMap<>(columns.size()); + while (rowSet.next()) { + if (rowSet.getString(1) != null) { + values.put(rowSet.getString(1).toLowerCase().trim(), rowSet.getString(2)); } - CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); - for (String column : columns) { - if (CollectorConstants.RESPONSE_TIME.equals(column)) { - long time = System.currentTimeMillis() - startTime; - valueRowBuilder.addColumn(String.valueOf(time)); - } else { - String value = values.get(column.toLowerCase()); - value = value == null ? CommonConstants.NULL_VALUE : value; - valueRowBuilder.addColumn(value); - } + } + CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); + for (String column : columns) { + if (CollectorConstants.RESPONSE_TIME.equals(column)) { + long time = System.currentTimeMillis() - startTime; + valueRowBuilder.addColumn(String.valueOf(time)); + } else { + String value = values.get(column.toLowerCase()); + value = value == null ? CommonConstants.NULL_VALUE : value; + valueRowBuilder.addColumn(value); } - builder.addValueRow(valueRowBuilder.build()); } + builder.addValueRow(valueRowBuilder.build()); } /** @@ -416,28 +435,45 @@ private void queryOneRowByMatchTwoColumns(Statement statement, String sql, List< * query sql：select one, tow, three, four from book; * and return multi row record mapping with the metrics * - * @param statement statement - * @param sql sql + * @param rowSet row set * @param columns query metrics field list * @throws Exception when error happen */ - private void queryMultiRow(Statement statement, String sql, List columns, + private void queryMultiRow(JdbcQueryRowSet rowSet, List columns, CollectRep.MetricsData.Builder builder, long startTime) throws Exception { - try (ResultSet resultSet = statement.executeQuery(sql)) { - while (resultSet.next()) { - CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); - for (String column : columns) { - if (CollectorConstants.RESPONSE_TIME.equals(column)) { - long time = System.currentTimeMillis() - startTime; - valueRowBuilder.addColumn(String.valueOf(time)); - } else { - String value = resultSet.getString(column); - value = value == null ? CommonConstants.NULL_VALUE : value; - valueRowBuilder.addColumn(value); - } + while (rowSet.next()) { + CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); + for (String column : columns) { + if (CollectorConstants.RESPONSE_TIME.equals(column)) { + long time = System.currentTimeMillis() - startTime; + valueRowBuilder.addColumn(String.valueOf(time)); + } else { + String value = rowSet.getString(column); + value = value == null ? CommonConstants.NULL_VALUE : value; + valueRowBuilder.addColumn(value); } - builder.addValueRow(valueRowBuilder.build()); } + builder.addValueRow(valueRowBuilder.build()); + } + } + + private void closeStatementAndConnection(Statement statement, boolean reuseConnection) { + if (statement == null) { + return; + } + Connection connection = null; + try { + connection = statement.getConnection(); + statement.close(); + } catch (Exception exception) { + log.error("Jdbc close statement error: {}", exception.getMessage()); + } + try { + if (!reuseConnection && connection != null) { + connection.close(); + } + } catch (Exception exception) { + log.error("Jdbc close connection error: {}", exception.getMessage()); } } @@ -548,4 +584,53 @@ private String constructDatabaseUrl(JdbcProtocol jdbcProtocol, String host, Stri default -> throw new IllegalArgumentException("Not support database platform: " + jdbcProtocol.getPlatform()); }; } + + private static final class ResultSetJdbcQueryRowSet implements JdbcQueryRowSet { + + private final Statement statement; + private final java.sql.ResultSet resultSet; + private final boolean reuseConnection; + + private ResultSetJdbcQueryRowSet(Statement statement, java.sql.ResultSet resultSet, boolean reuseConnection) { + this.statement = statement; + this.resultSet = resultSet; + this.reuseConnection = reuseConnection; + } + + @Override + public boolean next() throws Exception { + return resultSet.next(); + } + + @Override + public String getString(String column) throws Exception { + return resultSet.getString(column); + } + + @Override + public String getString(int index) throws Exception { + return resultSet.getString(index); + } + + @Override + public void close() throws Exception { + Connection connection = null; + try { + connection = statement.getConnection(); + } catch (Exception ignored) { + // ignore + } + try { + resultSet.close(); + } finally { + try { + statement.close(); + } finally { + if (!reuseConnection && connection != null) { + connection.close(); + } + } + } + } + } } diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutor.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutor.java new file mode 100644 index 00000000000..3d3c9076da3 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutor.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.query; + +import org.apache.hertzbeat.common.entity.job.Metrics; + +/** + * Adapter point for replacing only the SQL query execution part of JdbcCommonCollect. + */ +public interface JdbcQueryExecutor { + + boolean supports(Metrics metrics); + + JdbcQueryRowSet executeQuery(Metrics metrics, int timeout, int maxRows) throws Exception; +} diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutorRegistry.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutorRegistry.java new file mode 100644 index 00000000000..bb9f1c0baa6 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutorRegistry.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.query; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; +import org.apache.hertzbeat.common.entity.job.Metrics; + +/** + * Static registry used by JdbcCommonCollect to discover optional query executors. + */ +public final class JdbcQueryExecutorRegistry { + + private static final List EXECUTORS = new CopyOnWriteArrayList<>(); + + private JdbcQueryExecutorRegistry() { + } + + public static void register(JdbcQueryExecutor executor) { + if (executor == null || EXECUTORS.contains(executor)) { + return; + } + EXECUTORS.add(executor); + } + + public static void unregister(JdbcQueryExecutor executor) { + if (executor == null) { + return; + } + EXECUTORS.remove(executor); + } + + public static Optional resolve(Metrics metrics) { + return EXECUTORS.stream() + .filter(executor -> executor.supports(metrics)) + .findFirst(); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryRowSet.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryRowSet.java new file mode 100644 index 00000000000..a93cef11d77 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryRowSet.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.query; + +/** + * A minimal row cursor abstraction shared by JDBC and R2DBC-backed database queries. + */ +public interface JdbcQueryRowSet extends AutoCloseable { + + boolean next() throws Exception; + + String getString(String column) throws Exception; + + String getString(int index) throws Exception; + + @Override + void close() throws Exception; +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/pom.xml b/hertzbeat-collector/hertzbeat-collector-collector/pom.xml index e4f32ac721f..5fcca5107d9 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-collector/pom.xml @@ -48,6 +48,12 @@ ${hertzbeat.version} + + org.apache.hertzbeat + hertzbeat-collector-mysql-r2dbc + ${hertzbeat.version} + + org.apache.hertzbeat @@ -99,6 +105,23 @@ io.micrometer micrometer-registry-prometheus + + + org.springframework.boot + spring-boot-starter-test + test + + + com.mysql + mysql-connector-j + test + + + org.testcontainers + testcontainers + ${testcontainers.version} + test + @@ -365,14 +388,6 @@ apache-hertzbeat-collector-native-${hzb.version} - - - src/main/resources - - META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect - - - org.apache.maven.plugins diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlCollectorProperties.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlCollectorProperties.java new file mode 100644 index 00000000000..ac34a37ab84 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlCollectorProperties.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * Collector-side MySQL query engine routing. + */ +@Getter +@Setter +@ConfigurationProperties(prefix = "hertzbeat.collector.mysql") +public class MysqlCollectorProperties { + + private QueryEngine queryEngine = QueryEngine.AUTO; + + public QueryEngine resolveQueryEngine(boolean mysqlJdbcDriverAvailable) { + if (queryEngine == QueryEngine.AUTO) { + return mysqlJdbcDriverAvailable ? QueryEngine.JDBC : QueryEngine.R2DBC; + } + return queryEngine; + } + + public boolean useR2dbc(boolean mysqlJdbcDriverAvailable) { + return resolveQueryEngine(mysqlJdbcDriverAvailable) == QueryEngine.R2DBC; + } + + /** + * Supported collector-side query engines. + */ + public enum QueryEngine { + AUTO, + JDBC, + R2DBC + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailability.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailability.java new file mode 100644 index 00000000000..8c1466aedeb --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailability.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import java.net.URL; +import java.security.CodeSource; +import java.util.Locale; +import org.springframework.stereotype.Component; +import org.springframework.util.ClassUtils; +import org.springframework.util.StringUtils; + +/** + * Detects whether a MySQL JDBC driver is available from the external ext-lib path. + */ +@Component +public class MysqlJdbcDriverAvailability { + + private static final String[] MYSQL_DRIVER_CLASSES = { + "com.mysql.cj.jdbc.Driver", + "com.mysql.jdbc.Driver" + }; + + public boolean hasMysqlJdbcDriver() { + ClassLoader classLoader = ClassUtils.getDefaultClassLoader(); + for (String driverClass : MYSQL_DRIVER_CLASSES) { + if (!ClassUtils.isPresent(driverClass, classLoader)) { + continue; + } + try { + if (isExternalExtLibDriver(ClassUtils.forName(driverClass, classLoader))) { + return true; + } + } catch (ClassNotFoundException ignored) { + // Race-free enough for runtime detection: keep probing other known driver class names. + } + } + return false; + } + + boolean isExternalExtLibDriver(Class driverClass) { + String location = resolveLocation(driverClass); + return isExtLibLocation(location); + } + + static boolean isExtLibLocation(String location) { + if (!StringUtils.hasText(location)) { + return false; + } + String normalized = location + .replace('\\', '/') + .toLowerCase(Locale.ROOT); + return normalized.contains("/ext-lib/"); + } + + private String resolveLocation(Class driverClass) { + CodeSource codeSource = driverClass.getProtectionDomain().getCodeSource(); + if (codeSource != null && codeSource.getLocation() != null) { + return codeSource.getLocation().toExternalForm(); + } + String resourceName = ClassUtils.convertClassNameToResourcePath(driverClass.getName()) + ".class"; + ClassLoader classLoader = driverClass.getClassLoader(); + URL resource = classLoader != null ? classLoader.getResource(resourceName) : ClassLoader.getSystemResource(resourceName); + return resource != null ? resource.toExternalForm() : null; + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutor.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutor.java new file mode 100644 index 00000000000..55837095c60 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutor.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import java.net.URI; +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import org.apache.hertzbeat.collector.collect.common.ssh.SshTunnelHelper; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutor; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutorRegistry; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryRowSet; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.QueryOptions; +import org.apache.hertzbeat.collector.mysql.r2dbc.QueryResult; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.SshTunnel; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +/** + * MySQL-compatible query-only adapter that lets JdbcCommonCollect execute read-only queries through the built-in + * R2DBC path when no MySQL JDBC driver is present. + */ +@Component +public class MysqlR2dbcJdbcQueryExecutor implements JdbcQueryExecutor, InitializingBean, DisposableBean { + + private static final String QUERY_TYPE_ONE_ROW = "oneRow"; + private static final String QUERY_TYPE_MULTI_ROW = "multiRow"; + private static final String QUERY_TYPE_COLUMNS = "columns"; + + private final MysqlCollectorProperties properties; + private final MysqlQueryExecutor mysqlQueryExecutor; + private final MysqlJdbcDriverAvailability mysqlJdbcDriverAvailability; + + public MysqlR2dbcJdbcQueryExecutor(MysqlCollectorProperties properties, + MysqlQueryExecutor mysqlQueryExecutor, + MysqlJdbcDriverAvailability mysqlJdbcDriverAvailability) { + this.properties = properties; + this.mysqlQueryExecutor = mysqlQueryExecutor; + this.mysqlJdbcDriverAvailability = mysqlJdbcDriverAvailability; + } + + @Override + public boolean supports(Metrics metrics) { + if (metrics == null || metrics.getJdbc() == null) { + return false; + } + JdbcProtocol jdbcProtocol = metrics.getJdbc(); + if (!isMysqlCompatiblePlatform(jdbcProtocol.getPlatform())) { + return false; + } + String queryType = jdbcProtocol.getQueryType(); + return properties.useR2dbc(mysqlJdbcDriverAvailability.hasMysqlJdbcDriver()) + && (QUERY_TYPE_ONE_ROW.equals(queryType) + || QUERY_TYPE_MULTI_ROW.equals(queryType) + || QUERY_TYPE_COLUMNS.equals(queryType)); + } + + private boolean isMysqlCompatiblePlatform(String platform) { + return "mysql".equalsIgnoreCase(platform) || "mariadb".equalsIgnoreCase(platform); + } + + @Override + public JdbcQueryRowSet executeQuery(Metrics metrics, int timeout, int maxRows) { + JdbcProtocol jdbcProtocol = metrics.getJdbc(); + QueryOptions options = buildQueryOptions(jdbcProtocol, timeout, maxRows); + QueryResult queryResult = mysqlQueryExecutor.execute(jdbcProtocol.getSql(), options); + if (queryResult.hasError()) { + throw new IllegalStateException("R2DBC MySQL query failed: " + queryResult.getError()); + } + return new QueryResultRowSet(queryResult); + } + + @Override + public void afterPropertiesSet() { + JdbcQueryExecutorRegistry.register(this); + } + + @Override + public void destroy() { + JdbcQueryExecutorRegistry.unregister(this); + } + + private QueryOptions buildQueryOptions(JdbcProtocol jdbcProtocol, int timeout, int maxRows) { + MysqlTarget target = resolveTarget(jdbcProtocol); + SshTunnel sshTunnel = jdbcProtocol.getSshTunnel(); + String host = target.host(); + int port = target.port(); + if (sshTunnel != null && Boolean.parseBoolean(sshTunnel.getEnable())) { + try { + int localPort = SshTunnelHelper.localPortForward(sshTunnel, host, String.valueOf(port)); + host = "127.0.0.1"; + port = localPort; + } catch (Exception exception) { + throw new IllegalStateException("R2DBC MySQL query adapter failed to establish SSH tunnel", exception); + } + } + return QueryOptions.builder() + .host(host) + .port(port) + .username(jdbcProtocol.getUsername()) + .password(jdbcProtocol.getPassword()) + .database(target.database()) + .schema(target.database()) + .timeout(Duration.ofMillis(timeout)) + .maxRows(maxRows) + .fetchSize(256) + .readOnly(true) + .build(); + } + + private MysqlTarget resolveTarget(JdbcProtocol jdbcProtocol) { + if (StringUtils.hasText(jdbcProtocol.getUrl())) { + return parseJdbcUrl(jdbcProtocol.getUrl(), jdbcProtocol.getDatabase()); + } + if (!StringUtils.hasText(jdbcProtocol.getHost()) || !StringUtils.hasText(jdbcProtocol.getPort())) { + throw new IllegalArgumentException("R2DBC MySQL query adapter requires host/port or a jdbc:mysql URL"); + } + return new MysqlTarget(jdbcProtocol.getHost(), Integer.parseInt(jdbcProtocol.getPort()), jdbcProtocol.getDatabase()); + } + + private MysqlTarget parseJdbcUrl(String url, String fallbackDatabase) { + String trimmed = url.trim(); + if (!(trimmed.startsWith("jdbc:mysql://") || trimmed.startsWith("jdbc:mariadb://"))) { + throw new IllegalArgumentException("R2DBC MySQL query adapter only supports jdbc:mysql:// or jdbc:mariadb:// URLs"); + } + URI uri = URI.create(trimmed.substring("jdbc:".length())); + String host = uri.getHost(); + int port = uri.getPort() > 0 ? uri.getPort() : 3306; + if (!StringUtils.hasText(host)) { + throw new IllegalArgumentException("R2DBC MySQL query adapter URL must include a host"); + } + String path = uri.getPath(); + String database = StringUtils.hasText(path) && path.length() > 1 ? path.substring(1) : fallbackDatabase; + return new MysqlTarget(host, port, database); + } + + private record MysqlTarget(String host, int port, String database) { + } + + private static final class QueryResultRowSet implements JdbcQueryRowSet { + + private final List> rows; + private final Map columnIndexMap; + private int currentIndex = -1; + + private QueryResultRowSet(QueryResult queryResult) { + this.rows = queryResult.getRows(); + this.columnIndexMap = buildColumnIndexMap(queryResult.getColumns()); + } + + @Override + public boolean next() { + currentIndex++; + return currentIndex < rows.size(); + } + + @Override + public String getString(String column) { + Integer index = columnIndexMap.get(column.toLowerCase(Locale.ROOT)); + if (index == null) { + throw new IllegalArgumentException("Column not found in R2DBC MySQL result: " + column); + } + return getString(index + 1); + } + + @Override + public String getString(int index) { + if (currentIndex < 0 || currentIndex >= rows.size()) { + throw new IllegalStateException("R2DBC MySQL result cursor is not positioned on a row"); + } + int zeroBased = index - 1; + List row = rows.get(currentIndex); + if (zeroBased < 0 || zeroBased >= row.size()) { + throw new IllegalArgumentException("Column index out of bounds in R2DBC MySQL result: " + index); + } + return row.get(zeroBased); + } + + @Override + public void close() { + // QueryResult is fully materialized, so there is nothing left to close here. + } + + private static Map buildColumnIndexMap(List columns) { + Map indexMap = new HashMap<>(columns.size()); + for (int index = 0; index < columns.size(); index++) { + indexMap.put(columns.get(index).toLowerCase(Locale.ROOT), index); + } + return indexMap; + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactory.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactory.java index 4c6517ab905..28b9d2db88e 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactory.java +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactory.java @@ -20,6 +20,7 @@ import java.util.ServiceLoader; import java.util.concurrent.ConcurrentHashMap; +import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.collector.collect.AbstractCollect; import org.springframework.boot.CommandLineRunner; import org.springframework.context.annotation.Configuration; @@ -29,6 +30,7 @@ /** * Specific metrics collection factory */ +@Slf4j @Configuration @Order(value = Ordered.HIGHEST_PRECEDENCE + 1) public class CollectStrategyFactory implements CommandLineRunner { @@ -49,10 +51,18 @@ public static AbstractCollect invoke(String protocol) { @Override public void run(String... args) throws Exception { + COLLECT_STRATEGY.clear(); // spi load and registry protocol and collect instance ServiceLoader loader = ServiceLoader.load(AbstractCollect.class, AbstractCollect.class.getClassLoader()); for (AbstractCollect collect : loader) { COLLECT_STRATEGY.put(collect.supportProtocol(), collect); } + if (COLLECT_STRATEGY.isEmpty()) { + throw new IllegalStateException( + "No collect strategies were registered. " + + "Verify META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect " + + "is present on the runtime classpath."); + } + log.info("Registered {} collect strategies: {}", COLLECT_STRATEGY.size(), COLLECT_STRATEGY.keySet()); } } diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml index 9615e547b94..08d98b88717 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml @@ -77,6 +77,13 @@ common: type: netty hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MariadbJdbcQueryAdapterTemplateIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MariadbJdbcQueryAdapterTemplateIntegrationTest.java new file mode 100644 index 00000000000..1ac654969d0 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MariadbJdbcQueryAdapterTemplateIntegrationTest.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MariadbJdbcQueryAdapterTemplateIntegrationTest { + + private static final String TEST_DATABASE = "hzb"; + private static final String TEST_USERNAME = "test"; + private static final String TEST_PASSWORD = "test123"; + private static final String ROOT_PASSWORD = "root123"; + + private GenericContainer container; + private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor; + private List mariadbTemplateMetrics; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + new CollectStrategyFactory().run(); + container = new GenericContainer<>(DockerImageName.parse("mariadb:11.4")) + .withExposedPorts(3306) + .withEnv("MARIADB_DATABASE", TEST_DATABASE) + .withEnv("MARIADB_USER", TEST_USERNAME) + .withEnv("MARIADB_PASSWORD", TEST_PASSWORD) + .withEnv("MARIADB_ROOT_PASSWORD", ROOT_PASSWORD) + .waitingFor(Wait.forListeningPort()); + container.start(); + awaitTcpLoginReady(container, TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE); + initMonitoringData(container); + + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + jdbcQueryExecutor.afterPropertiesSet(); + mariadbTemplateMetrics = loadMariadbTemplate().getMetrics(); + } + + @AfterAll + void tearDown() throws Exception { + if (jdbcQueryExecutor != null) { + jdbcQueryExecutor.destroy(); + } + if (container != null) { + container.stop(); + } + } + + @TestFactory + Stream shouldCollectOfficialMariadbTemplateThroughJdbcQueryAdapter() { + return mariadbTemplateMetrics.stream() + .map(templateMetric -> DynamicTest.dynamicTest(templateMetric.getName(), + () -> verifyTemplateMetric(templateMetric))); + } + + private void verifyTemplateMetric(Metrics templateMetric) throws Exception { + Metrics metric = materializeMetric(templateMetric); + if ("process_state".equals(metric.getName())) { + startBackgroundSleepQuery(container); + } + if ("slow_sql".equals(metric.getName())) { + generateSlowQuery(container); + } + CollectRep.MetricsData metricsData = collect(metric); + assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(), + () -> metric.getName() + " failed: " + metricsData.getMsg()); + assertEquals(metric.getFields().size(), metricsData.getFieldsCount(), + () -> metric.getName() + " fields should still be produced by the original parser"); + if ("columns".equals(metric.getJdbc().getQueryType())) { + assertEquals(1, metricsData.getValuesCount(), () -> metric.getName() + " should keep the original single-row shape"); + } + if ("basic".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0); + assertNotNull(metricsData.getValues().getFirst().getColumns(0)); + assertTrue(!Objects.equals(CommonConstants.NULL_VALUE, metricsData.getValues().getFirst().getColumns(0)), + "basic.version should be collected through the adapted query path"); + } + if ("process_state".equals(metric.getName()) || "slow_sql".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0, () -> metric.getName() + " should return at least one row"); + } + } + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L) + .tenantId(1L) + .app("mariadb") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { + }); + CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch(); + MetricsCollect metricsCollect = new MetricsCollect( + metric, + new StubTimeout(timerTask), + collectDataDispatch, + "collector-test", + List.of()); + metricsCollect.run(); + assertNotNull(collectDataDispatch.metricsData, metric.getName() + " should dispatch metrics data"); + return collectDataDispatch.metricsData; + } + + private Metrics materializeMetric(Metrics templateMetric) { + Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbcProtocol = metric.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306))); + jdbcProtocol.setUsername(TEST_USERNAME); + jdbcProtocol.setPassword(TEST_PASSWORD); + jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis())); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setUrl(null); + jdbcProtocol.setSshTunnel(null); + if (jdbcProtocol.getDatabase() == null || jdbcProtocol.getDatabase().contains("^_^")) { + jdbcProtocol.setDatabase(TEST_DATABASE); + } + if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) { + metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList())); + } + return metric; + } + + private Job loadMariadbTemplate() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mariadb.yml") + .toAbsolutePath() + .normalize(); + Yaml yaml = new Yaml(); + try (Reader reader = Files.newBufferedReader(template)) { + return yaml.loadAs(reader, Job.class); + } + } + + private void initMonitoringData(GenericContainer mariaDb) throws Exception { + execRoot(mariaDb, + "GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';" + + " GRANT PROCESS ON *.* TO '" + TEST_USERNAME + "'@'%';" + + " SET GLOBAL log_output='TABLE';" + + " SET GLOBAL slow_query_log='ON';" + + " SET GLOBAL long_query_time=0;" + + " FLUSH PRIVILEGES;"); + generateSlowQuery(mariaDb); + } + + private void generateSlowQuery(GenericContainer mariaDb) throws Exception { + execUser(mariaDb, TEST_DATABASE, "SELECT SLEEP(0.2);"); + Thread.sleep(300); + } + + private void startBackgroundSleepQuery(GenericContainer mariaDb) throws Exception { + String command = String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "nohup sh -lc", + "'$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + TEST_USERNAME, + "-p" + TEST_PASSWORD, + TEST_DATABASE, + "-e", + "\"SELECT SLEEP(15)\" >/tmp/process-state.log 2>&1'", + ">/dev/null 2>&1 &"); + mariaDb.execInContainer("sh", "-lc", command); + Thread.sleep(500); + } + + private void awaitTcpLoginReady(GenericContainer mariaDb, String username, String password, String database) throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis(); + while (System.currentTimeMillis() < deadline) { + try { + var result = mariaDb.execInContainer("sh", "-lc", mysqlCliCommand(username, password, database, "SELECT 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for the MariaDB entrypoint to finish bootstrapping and switch to the final TCP listener. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MariaDB TCP login to become ready"); + } + + private void execRoot(GenericContainer mariaDb, String sql) throws Exception { + var result = mariaDb.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("root mysql command failed: " + result.getStderr()); + } + } + + private void execUser(GenericContainer mariaDb, String database, String sql) throws Exception { + var result = mariaDb.execInContainer("sh", "-lc", mysqlCliCommand(TEST_USERNAME, TEST_PASSWORD, database, sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("user mysql command failed: " + result.getStderr()); + } + } + + private String mysqlCliCommand(String username, String password, String database, String sql) { + return String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + username, + "-p" + password, + database, + "-e", + "\"" + sql.replace("\"", "\\\"") + "\""); + } + + private static final class CapturingCollectDataDispatch implements CollectDataDispatch { + + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) { + this.metricsData = metricsData; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, List metricsDataList) { + if (metricsDataList != null && !metricsDataList.isEmpty()) { + this.metricsData = metricsDataList.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailabilityTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailabilityTest.java new file mode 100644 index 00000000000..9c5cabe8428 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailabilityTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class MysqlJdbcDriverAvailabilityTest { + + @Test + void shouldTreatOnlyExtLibLocationsAsAutoJdbcSignal() { + assertTrue(MysqlJdbcDriverAvailability.isExtLibLocation("/opt/hertzbeat/ext-lib/mysql-connector-j-9.0.0.jar")); + assertTrue(MysqlJdbcDriverAvailability.isExtLibLocation("file:/C:/hertzbeat/ext-lib/mysql-connector-j-9.0.0.jar")); + assertFalse(MysqlJdbcDriverAvailability.isExtLibLocation("/Users/dev/.m2/repository/com/mysql/mysql-connector-j/9.0.0/mysql-connector-j-9.0.0.jar")); + assertFalse(MysqlJdbcDriverAvailability.isExtLibLocation(null)); + } + + @Test + void shouldIgnoreTestClasspathMysqlDriverWhenItIsNotFromExtLib() { + MysqlJdbcDriverAvailability availability = new MysqlJdbcDriverAvailability(); + + assertFalse(availability.hasMysqlJdbcDriver()); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterCompatibilityIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterCompatibilityIntegrationTest.java new file mode 100644 index 00000000000..38d7265ddb8 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterCompatibilityIntegrationTest.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MysqlJdbcQueryAdapterCompatibilityIntegrationTest { + + private static final String TEST_DATABASE = "hzb"; + private static final String TEST_USERNAME = "test"; + private static final String TEST_PASSWORD = "test123"; + private static final String ROOT_PASSWORD = "root123"; + private static final Set REPRESENTATIVE_TEMPLATE_METRICS = Set.of("basic", "process_state"); + + private List representativeTemplateMetrics; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + new CollectStrategyFactory().run(); + representativeTemplateMetrics = loadMysqlTemplate().getMetrics().stream() + .filter(metric -> REPRESENTATIVE_TEMPLATE_METRICS.contains(metric.getName())) + .collect(Collectors.toList()); + } + + @TestFactory + Stream shouldCollectRepresentativeTemplateMetricsAcrossCompatibilityMatrix() { + return Stream.of( + new DatabaseTarget("mysql-5.7.44", DockerImageName.parse("mysql:5.7.44"), false), + new DatabaseTarget("mysql-8.0.36", DockerImageName.parse("mysql:8.0.36"), false), + new DatabaseTarget("mariadb-11.4", DockerImageName.parse("mariadb:11.4"), true)) + .map(target -> DynamicTest.dynamicTest(target.name(), () -> verifyRepresentativeMetrics(target))); + } + + private void verifyRepresentativeMetrics(DatabaseTarget target) throws Exception { + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + try (GenericContainer container = createContainer(target)) { + jdbcQueryExecutor.afterPropertiesSet(); + container.start(); + awaitTcpLoginReady(container, TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE); + initMonitoringData(container); + + for (Metrics templateMetric : representativeTemplateMetrics) { + Metrics metric = materializeMetric(templateMetric, container); + if ("process_state".equals(metric.getName())) { + startBackgroundSleepQuery(container); + } + CollectRep.MetricsData metricsData = collect(metric); + assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(), + () -> target.name() + " " + metric.getName() + " failed: " + metricsData.getMsg()); + assertEquals(metric.getFields().size(), metricsData.getFieldsCount(), + () -> target.name() + " " + metric.getName() + " should keep the original parser output shape"); + if ("basic".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0, () -> target.name() + " basic should return data"); + assertNotNull(metricsData.getValues().getFirst().getColumns(0)); + assertTrue(!Objects.equals(CommonConstants.NULL_VALUE, metricsData.getValues().getFirst().getColumns(0)), + () -> target.name() + " basic.version should be collected"); + } + if ("process_state".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0, + () -> target.name() + " process_state should return at least one grouped state row"); + } + } + } finally { + jdbcQueryExecutor.destroy(); + } + } + + private GenericContainer createContainer(DatabaseTarget target) { + GenericContainer container = new GenericContainer<>(target.image()) + .withExposedPorts(3306) + .waitingFor(Wait.forListeningPort()); + if (target.mariaDb()) { + return container.withEnv("MARIADB_DATABASE", TEST_DATABASE) + .withEnv("MARIADB_USER", TEST_USERNAME) + .withEnv("MARIADB_PASSWORD", TEST_PASSWORD) + .withEnv("MARIADB_ROOT_PASSWORD", ROOT_PASSWORD); + } + return container.withEnv("MYSQL_DATABASE", TEST_DATABASE) + .withEnv("MYSQL_USER", TEST_USERNAME) + .withEnv("MYSQL_PASSWORD", TEST_PASSWORD) + .withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD); + } + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L) + .tenantId(1L) + .app("mysql") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { + }); + CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch(); + MetricsCollect metricsCollect = new MetricsCollect( + metric, + new StubTimeout(timerTask), + collectDataDispatch, + "collector-test", + List.of()); + metricsCollect.run(); + return collectDataDispatch.metricsData; + } + + private Metrics materializeMetric(Metrics templateMetric, GenericContainer container) { + Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbcProtocol = metric.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306))); + jdbcProtocol.setUsername(TEST_USERNAME); + jdbcProtocol.setPassword(TEST_PASSWORD); + jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis())); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setUrl(null); + jdbcProtocol.setSshTunnel(null); + if (jdbcProtocol.getDatabase() == null || jdbcProtocol.getDatabase().contains("^_^")) { + jdbcProtocol.setDatabase(TEST_DATABASE); + } + if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) { + metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList())); + } + return metric; + } + + private Job loadMysqlTemplate() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mysql.yml") + .toAbsolutePath() + .normalize(); + Yaml yaml = new Yaml(); + try (Reader reader = Files.newBufferedReader(template)) { + return yaml.loadAs(reader, Job.class); + } + } + + private void initMonitoringData(GenericContainer mysql) throws Exception { + execRoot(mysql, + "GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';" + + " GRANT PROCESS ON *.* TO '" + TEST_USERNAME + "'@'%';" + + " SET GLOBAL log_output='TABLE';" + + " SET GLOBAL slow_query_log='ON';" + + " SET GLOBAL long_query_time=0;" + + " FLUSH PRIVILEGES;"); + } + + private void startBackgroundSleepQuery(GenericContainer mysql) throws Exception { + String command = String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "nohup sh -lc", + "'$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + TEST_USERNAME, + "-p" + TEST_PASSWORD, + TEST_DATABASE, + "-e", + "\"SELECT SLEEP(15)\" >/tmp/process-state.log 2>&1'", + ">/dev/null 2>&1 &"); + mysql.execInContainer("sh", "-lc", command); + Thread.sleep(500); + } + + private void awaitTcpLoginReady(GenericContainer mysql, String username, String password, String database) throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis(); + while (System.currentTimeMillis() < deadline) { + try { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand(username, password, database, "SELECT 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for the database entrypoint to finish bootstrapping and switch to the final TCP listener. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MySQL-compatible TCP login to become ready"); + } + + private void execRoot(GenericContainer mysql, String sql) throws Exception { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("root mysql command failed: " + result.getStderr()); + } + } + + private String mysqlCliCommand(String username, String password, String database, String sql) { + return String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + username, + "-p" + password, + database, + "-e", + "\"" + sql.replace("\"", "\\\"") + "\""); + } + + private record DatabaseTarget(String name, DockerImageName image, boolean mariaDb) { + } + + private static final class CapturingCollectDataDispatch implements CollectDataDispatch { + + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) { + this.metricsData = metricsData; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, List metricsDataList) { + if (metricsDataList != null && !metricsDataList.isEmpty()) { + this.metricsData = metricsDataList.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterTemplateIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterTemplateIntegrationTest.java new file mode 100644 index 00000000000..3d91c7107d6 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterTemplateIntegrationTest.java @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestFactory; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MysqlJdbcQueryAdapterTemplateIntegrationTest { + + private static final String TEST_DATABASE = "hzb"; + private static final String TEST_USERNAME = "test"; + private static final String TEST_PASSWORD = "test123"; + private static final String ROOT_PASSWORD = "root123"; + + private GenericContainer container; + private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor; + private List mysqlTemplateMetrics; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + new CollectStrategyFactory().run(); + container = new GenericContainer<>(DockerImageName.parse("mysql:8.0.36")) + .withExposedPorts(3306) + .withEnv("MYSQL_DATABASE", TEST_DATABASE) + .withEnv("MYSQL_USER", TEST_USERNAME) + .withEnv("MYSQL_PASSWORD", TEST_PASSWORD) + .withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD) + .waitingFor(Wait.forListeningPort()); + container.start(); + awaitTcpLoginReady(container, TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE); + initMonitoringData(container); + + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + jdbcQueryExecutor.afterPropertiesSet(); + mysqlTemplateMetrics = loadMysqlTemplate().getMetrics(); + } + + @AfterAll + void tearDown() throws Exception { + if (jdbcQueryExecutor != null) { + jdbcQueryExecutor.destroy(); + } + if (container != null) { + container.stop(); + } + } + + @TestFactory + Stream shouldCollectOfficialMysqlTemplateThroughJdbcQueryAdapter() { + return mysqlTemplateMetrics.stream() + .map(templateMetric -> DynamicTest.dynamicTest(templateMetric.getName(), + () -> verifyTemplateMetric(templateMetric))); + } + + private void verifyTemplateMetric(Metrics templateMetric) throws Exception { + Metrics metric = materializeMetric(templateMetric); + if ("process_state".equals(metric.getName())) { + startBackgroundSleepQuery(container); + } + if ("slow_sql".equals(metric.getName())) { + generateSlowQuery(container); + } + CollectRep.MetricsData metricsData = collect(metric); + assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(), + () -> metric.getName() + " failed: " + metricsData.getMsg()); + assertEquals(metric.getFields().size(), metricsData.getFieldsCount(), + () -> metric.getName() + " fields should still be produced by the original parser"); + if ("columns".equals(metric.getJdbc().getQueryType())) { + assertEquals(1, metricsData.getValuesCount(), () -> metric.getName() + " should keep the original single-row shape"); + } + if ("basic".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0); + assertNotNull(metricsData.getValues().getFirst().getColumns(0)); + assertTrue(!Objects.equals(CommonConstants.NULL_VALUE, metricsData.getValues().getFirst().getColumns(0)), + "basic.version should be collected through the adapted query path"); + } + if ("process_state".equals(metric.getName()) + || "slow_sql".equals(metric.getName()) + || "account_expiry".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0, () -> metric.getName() + " should return at least one row"); + } + } + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L) + .tenantId(1L) + .app("mysql") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { + }); + CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch(); + MetricsCollect metricsCollect = new MetricsCollect( + metric, + new StubTimeout(timerTask), + collectDataDispatch, + "collector-test", + List.of()); + metricsCollect.run(); + assertNotNull(collectDataDispatch.metricsData, metric.getName() + " should dispatch metrics data"); + return collectDataDispatch.metricsData; + } + + private Metrics materializeMetric(Metrics templateMetric) { + Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbcProtocol = metric.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306))); + jdbcProtocol.setUsername(TEST_USERNAME); + jdbcProtocol.setPassword(TEST_PASSWORD); + jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis())); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setUrl(null); + jdbcProtocol.setSshTunnel(null); + if (jdbcProtocol.getDatabase() == null || jdbcProtocol.getDatabase().contains("^_^")) { + jdbcProtocol.setDatabase(TEST_DATABASE); + } + if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) { + metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList())); + } + return metric; + } + + private Job loadMysqlTemplate() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mysql.yml") + .toAbsolutePath() + .normalize(); + Yaml yaml = new Yaml(); + try (Reader reader = Files.newBufferedReader(template)) { + return yaml.loadAs(reader, Job.class); + } + } + + private void initMonitoringData(GenericContainer mysql) throws Exception { + execRoot(mysql, + "GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';" + + " GRANT PROCESS ON *.* TO '" + TEST_USERNAME + "'@'%';" + + " SET GLOBAL log_output='TABLE';" + + " SET GLOBAL slow_query_log='ON';" + + " SET GLOBAL long_query_time=0;" + + " FLUSH PRIVILEGES;"); + generateSlowQuery(mysql); + } + + private void generateSlowQuery(GenericContainer mysql) throws Exception { + execUser(mysql, TEST_DATABASE, "SELECT SLEEP(0.2);"); + Thread.sleep(300); + } + + private void startBackgroundSleepQuery(GenericContainer mysql) throws Exception { + String command = String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "nohup sh -lc", + "'$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + TEST_USERNAME, + "-p" + TEST_PASSWORD, + TEST_DATABASE, + "-e", + "\"SELECT SLEEP(15)\" >/tmp/process-state.log 2>&1'", + ">/dev/null 2>&1 &"); + mysql.execInContainer("sh", "-lc", command); + Thread.sleep(500); + } + + private void awaitTcpLoginReady(GenericContainer mysql, String username, String password, String database) throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis(); + while (System.currentTimeMillis() < deadline) { + try { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand(username, password, database, "SELECT 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for the MySQL entrypoint to finish bootstrapping and switch to the final TCP listener. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MySQL TCP login to become ready"); + } + + private void execRoot(GenericContainer mysql, String sql) throws Exception { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("root mysql command failed: " + result.getStderr()); + } + } + + private void execUser(GenericContainer mysql, String database, String sql) throws Exception { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand(TEST_USERNAME, TEST_PASSWORD, database, sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("user mysql command failed: " + result.getStderr()); + } + } + + private String mysqlCliCommand(String username, String password, String database, String sql) { + return String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + username, + "-p" + password, + database, + "-e", + "\"" + sql.replace("\"", "\\\"") + "\""); + } + + private static final class CapturingCollectDataDispatch implements CollectDataDispatch { + + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) { + this.metricsData = metricsData; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, List metricsDataList) { + if (metricsDataList != null && !metricsDataList.isEmpty()) { + this.metricsData = metricsDataList.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryParityIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryParityIntegrationTest.java new file mode 100644 index 00000000000..650e6e92c5c --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryParityIntegrationTest.java @@ -0,0 +1,393 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.io.IOException; +import java.io.Reader; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutorRegistry; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MysqlJdbcQueryParityIntegrationTest { + + private static final String TEST_DATABASE = "hzb"; + private static final String TEST_USERNAME = "test"; + private static final String TEST_PASSWORD = "test123"; + private static final String ROOT_PASSWORD = "root123"; + private static final String PARITY_TABLE = "collector_parity_metrics"; + + private Metrics basicTemplateMetric; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + new CollectStrategyFactory().run(); + assertDoesNotThrow(() -> Class.forName("com.mysql.cj.jdbc.Driver")); + basicTemplateMetric = loadMysqlTemplate().getMetrics().stream() + .filter(metric -> "basic".equals(metric.getName())) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Unable to locate the basic metric in app-mysql.yml")); + } + + @AfterEach + void clearRegisteredExecutors() throws Exception { + Field executorsField = JdbcQueryExecutorRegistry.class.getDeclaredField("EXECUTORS"); + executorsField.setAccessible(true); + @SuppressWarnings("unchecked") + CopyOnWriteArrayList executors = (CopyOnWriteArrayList) executorsField.get(null); + executors.clear(); + } + + @TestFactory + Stream shouldMatchJdbcResultsForRepresentativeMysqlQueryShapes() { + return Stream.of( + new DatabaseTarget("mysql-5.7.44", DockerImageName.parse("mysql:5.7.44")), + new DatabaseTarget("mysql-8.0.36", DockerImageName.parse("mysql:8.0.36"))) + .map(target -> DynamicTest.dynamicTest(target.name(), () -> verifyParityAcrossTarget(target))); + } + + private void verifyParityAcrossTarget(DatabaseTarget target) throws Exception { + try (GenericContainer container = createContainer(target)) { + container.start(); + awaitTcpLoginReady(container, TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE); + initParityData(container); + + List parityMetrics = List.of( + materializeMetric(basicTemplateMetric, container), + buildColumnsParityMetric(container), + buildOneRowParityMetric(container), + buildMultiRowParityMetric(container)); + + for (Metrics parityMetric : parityMetrics) { + CollectRep.MetricsData jdbcResult = collectWithJdbc(parityMetric); + CollectRep.MetricsData r2dbcResult = collectWithR2dbc(parityMetric); + + assertEquals(CollectRep.Code.SUCCESS, jdbcResult.getCode(), + () -> target.name() + " JDBC baseline failed for " + parityMetric.getName() + ": " + jdbcResult.getMsg()); + assertEquals(CollectRep.Code.SUCCESS, r2dbcResult.getCode(), + () -> target.name() + " R2DBC path failed for " + parityMetric.getName() + ": " + r2dbcResult.getMsg()); + assertEquals(jdbcResult.getFields(), r2dbcResult.getFields(), + () -> target.name() + " field set differs for " + parityMetric.getName()); + assertEquals(normalizeRows(jdbcResult), normalizeRows(r2dbcResult), + () -> target.name() + " row payload differs for " + parityMetric.getName()); + assertFalse(r2dbcResult.getValues().isEmpty(), + () -> target.name() + " " + parityMetric.getName() + " should return at least one row"); + } + } + } + + private CollectRep.MetricsData collectWithJdbc(Metrics metric) throws Exception { + clearRegisteredExecutors(); + return collect(JsonUtil.fromJson(JsonUtil.toJson(metric), Metrics.class)); + } + + private CollectRep.MetricsData collectWithR2dbc(Metrics metric) throws Exception { + clearRegisteredExecutors(); + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + try { + jdbcQueryExecutor.afterPropertiesSet(); + return collect(JsonUtil.fromJson(JsonUtil.toJson(metric), Metrics.class)); + } finally { + jdbcQueryExecutor.destroy(); + clearRegisteredExecutors(); + } + } + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L) + .tenantId(1L) + .app("mysql") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { + }); + CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch(); + MetricsCollect metricsCollect = new MetricsCollect( + metric, + new StubTimeout(timerTask), + collectDataDispatch, + "collector-test", + List.of()); + metricsCollect.run(); + return collectDataDispatch.metricsData; + } + + private Metrics buildColumnsParityMetric(GenericContainer container) { + return buildMetric( + "columns-parity", + List.of(field("version"), field("max_connections"), field("character_set_server")), + List.of("version", "max_connections", "character_set_server"), + "columns", + "SHOW VARIABLES WHERE Variable_name IN ('version', 'max_connections', 'character_set_server')", + container); + } + + private Metrics buildOneRowParityMetric(GenericContainer container) { + return buildMetric( + "one-row-parity", + List.of(field("answer"), field("label"), field("nullable_value")), + List.of("answer", "label", "nullable_value"), + "oneRow", + "SELECT 42 AS answer, 'adapter-parity' AS label, NULL AS nullable_value", + container); + } + + private Metrics buildMultiRowParityMetric(GenericContainer container) { + return buildMetric( + "multi-row-parity", + List.of(field("metric_name"), field("metric_value"), field("metric_note")), + List.of("metric_name", "metric_value", "metric_note"), + "multiRow", + "SELECT metric_name, metric_value, metric_note FROM " + PARITY_TABLE + " ORDER BY metric_name", + container); + } + + private Metrics buildMetric(String name, List fields, List aliasFields, + String queryType, String sql, GenericContainer container) { + JdbcProtocol jdbcProtocol = JdbcProtocol.builder() + .host(container.getHost()) + .port(String.valueOf(container.getMappedPort(3306))) + .platform("mysql") + .database(TEST_DATABASE) + .username(TEST_USERNAME) + .password(TEST_PASSWORD) + .timeout(String.valueOf(Duration.ofSeconds(8).toMillis())) + .queryType(queryType) + .reuseConnection("false") + .url(buildJdbcUrl(container)) + .sql(sql) + .build(); + return Metrics.builder() + .name(name) + .protocol("jdbc") + .priority((byte) 1) + .fields(fields) + .aliasFields(aliasFields) + .jdbc(jdbcProtocol) + .build(); + } + + private Metrics materializeMetric(Metrics templateMetric, GenericContainer container) { + Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbcProtocol = metric.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306))); + jdbcProtocol.setUsername(TEST_USERNAME); + jdbcProtocol.setPassword(TEST_PASSWORD); + jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis())); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setUrl(buildJdbcUrl(container)); + jdbcProtocol.setSshTunnel(null); + if (jdbcProtocol.getDatabase() == null || jdbcProtocol.getDatabase().contains("^_^")) { + jdbcProtocol.setDatabase(TEST_DATABASE); + } + if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) { + metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList())); + } + return metric; + } + + private Job loadMysqlTemplate() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mysql.yml") + .toAbsolutePath() + .normalize(); + Yaml yaml = new Yaml(); + try (Reader reader = Files.newBufferedReader(template)) { + return yaml.loadAs(reader, Job.class); + } + } + + private GenericContainer createContainer(DatabaseTarget target) { + return new GenericContainer<>(target.image()) + .withExposedPorts(3306) + .withEnv("MYSQL_DATABASE", TEST_DATABASE) + .withEnv("MYSQL_USER", TEST_USERNAME) + .withEnv("MYSQL_PASSWORD", TEST_PASSWORD) + .withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD) + .waitingFor(Wait.forListeningPort()); + } + + private void initParityData(GenericContainer mysql) throws Exception { + execRoot(mysql, + "GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';" + + " GRANT SELECT ON " + TEST_DATABASE + ".* TO '" + TEST_USERNAME + "'@'%';" + + " DROP TABLE IF EXISTS " + TEST_DATABASE + "." + PARITY_TABLE + ";" + + " CREATE TABLE " + TEST_DATABASE + "." + PARITY_TABLE + " (" + + " metric_name VARCHAR(32) PRIMARY KEY," + + " metric_value VARCHAR(32) NOT NULL," + + " metric_note VARCHAR(32) NULL" + + " );" + + " INSERT INTO " + TEST_DATABASE + "." + PARITY_TABLE + + " (metric_name, metric_value, metric_note) VALUES" + + " ('alpha', '1', NULL)," + + " ('beta', '2', 'steady');" + + " FLUSH PRIVILEGES;"); + } + + private void awaitTcpLoginReady(GenericContainer mysql, String username, String password, String database) throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis(); + while (System.currentTimeMillis() < deadline) { + try { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand(username, password, database, "SELECT 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for the MySQL entrypoint to finish bootstrapping and switch to the final TCP listener. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MySQL TCP login to become ready"); + } + + private void execRoot(GenericContainer mysql, String sql) throws Exception { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("root mysql command failed: " + result.getStderr()); + } + } + + private String mysqlCliCommand(String username, String password, String database, String sql) { + return String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + username, + "-p" + password, + database, + "-e", + "\"" + sql.replace("\"", "\\\"") + "\""); + } + + private String buildJdbcUrl(GenericContainer container) { + return "jdbc:mysql://%s:%d/%s?allowPublicKeyRetrieval=true&useSSL=false" + .formatted(container.getHost(), container.getMappedPort(3306), TEST_DATABASE); + } + + private List> normalizeRows(CollectRep.MetricsData metricsData) { + return metricsData.getValues().stream() + .map(valueRow -> new ArrayList<>(valueRow.getColumnsList())) + .sorted(Comparator.comparing(row -> String.join("\u0001", row))) + .collect(Collectors.toList()); + } + + private Metrics.Field field(String name) { + return Metrics.Field.builder().field(name).type((byte) 1).build(); + } + + private static final class CapturingCollectDataDispatch implements CollectDataDispatch { + + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) { + this.metricsData = metricsData; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, List metricsDataList) { + if (metricsDataList != null && !metricsDataList.isEmpty()) { + this.metricsData = metricsDataList.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } + + private record DatabaseTarget(String name, DockerImageName image) { + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutorTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutorTest.java new file mode 100644 index 00000000000..6162d5e193d --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutorTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryRowSet; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.QueryResult; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.junit.jupiter.api.Test; + +class MysqlR2dbcJdbcQueryExecutorTest { + + @Test + void shouldAutoRouteToR2dbcOnlyWhenMysqlJdbcDriverIsAbsent() { + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + MysqlJdbcDriverAvailability driverAvailability = mock(MysqlJdbcDriverAvailability.class); + when(driverAvailability.hasMysqlJdbcDriver()).thenReturn(false); + MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor( + properties, mock(MysqlQueryExecutor.class), driverAvailability); + + assertTrue(executor.supports(metrics("mysql", "columns"))); + assertTrue(executor.supports(metrics("mariadb", "columns"))); + assertTrue(executor.supports(metrics("mysql", "multiRow"))); + assertFalse(executor.supports(metrics("mysql", "runScript"))); + assertFalse(executor.supports(metrics("postgresql", "columns"))); + } + + @Test + void shouldPreferJdbcWhenMysqlJdbcDriverIsPresentInAutoMode() { + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + MysqlJdbcDriverAvailability driverAvailability = mock(MysqlJdbcDriverAvailability.class); + when(driverAvailability.hasMysqlJdbcDriver()).thenReturn(true); + MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor( + properties, mock(MysqlQueryExecutor.class), driverAvailability); + + assertFalse(executor.supports(metrics("mysql", "columns"))); + } + + @Test + void shouldHonorExplicitQueryEngineOverrides() { + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + MysqlJdbcDriverAvailability driverAvailability = mock(MysqlJdbcDriverAvailability.class); + when(driverAvailability.hasMysqlJdbcDriver()).thenReturn(true); + MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor( + properties, mock(MysqlQueryExecutor.class), driverAvailability); + + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + assertTrue(executor.supports(metrics("mysql", "columns"))); + + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.JDBC); + assertFalse(executor.supports(metrics("mysql", "columns"))); + } + + @Test + void shouldExposeQueryResultsAsJdbcStyleRowSet() throws Exception { + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + MysqlQueryExecutor mysqlQueryExecutor = mock(MysqlQueryExecutor.class); + when(mysqlQueryExecutor.execute(anyString(), any())) + .thenReturn(QueryResult.builder() + .columns(List.of("Variable_name", "Value")) + .rows(List.of( + List.of("Threads_connected", "5"), + List.of("Uptime", "10"))) + .elapsedMs(11) + .rowCount(2) + .build()); + MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor( + properties, mysqlQueryExecutor, mock(MysqlJdbcDriverAvailability.class)); + + try (JdbcQueryRowSet rowSet = executor.executeQuery(metrics("mysql", "columns"), 6000, 1000)) { + assertTrue(rowSet.next()); + assertEquals("Threads_connected", rowSet.getString(1)); + assertEquals("5", rowSet.getString(2)); + assertEquals("5", rowSet.getString("value")); + + assertTrue(rowSet.next()); + assertEquals("Uptime", rowSet.getString("variable_name")); + assertEquals("10", rowSet.getString(2)); + assertFalse(rowSet.next()); + } + } + + @Test + void shouldFailFastWhenR2dbcQueryReturnsError() { + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + MysqlQueryExecutor mysqlQueryExecutor = mock(MysqlQueryExecutor.class); + when(mysqlQueryExecutor.execute(anyString(), any())) + .thenReturn(QueryResult.builder().error("query timeout").build()); + MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor( + properties, mysqlQueryExecutor, mock(MysqlJdbcDriverAvailability.class)); + + IllegalStateException exception = assertThrows(IllegalStateException.class, + () -> executor.executeQuery(metrics("mysql", "columns"), 6000, 1000)); + assertTrue(exception.getMessage().contains("query timeout")); + } + + private Metrics metrics(String platform, String queryType) { + JdbcProtocol jdbcProtocol = JdbcProtocol.builder() + .host("127.0.0.1") + .port("3306") + .platform(platform) + .database("hzb") + .username("test") + .password("test123") + .queryType(queryType) + .sql("SHOW GLOBAL STATUS") + .timeout("6000") + .build(); + Metrics metrics = new Metrics(); + metrics.setProtocol("jdbc"); + metrics.setName("status"); + metrics.setJdbc(jdbcProtocol); + return metrics; + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/OceanbaseJdbcQueryAdapterIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/OceanbaseJdbcQueryAdapterIntegrationTest.java new file mode 100644 index 00000000000..e95cfa62052 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/OceanbaseJdbcQueryAdapterIntegrationTest.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class OceanbaseJdbcQueryAdapterIntegrationTest { + + private static final String OCEANBASE_IMAGE = "oceanbase/oceanbase-ce:latest"; + private static final String OCEANBASE_USERNAME = "root@sys"; + private static final String OCEANBASE_PASSWORD = ""; + private static final String OCEANBASE_DATABASE = "oceanbase"; + + private GenericContainer container; + private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor; + private List oceanbaseTemplateMetrics; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + new CollectStrategyFactory().run(); + container = new GenericContainer<>(DockerImageName.parse(OCEANBASE_IMAGE)) + .withExposedPorts(2881) + .withEnv("MODE", "MINI") + .withEnv("OB_MEMORY_LIMIT", "4096M") + .withEnv("OB_SYSTEM_MEMORY", "1024M") + .withEnv("OB_DATAFILE_SIZE", "2048M") + .withEnv("OB_LOG_DISK_SIZE", "2048M") + .withCommand("bash", "-lc", "/usr/sbin/sshd || true; /root/boot/start.sh || true; tail -f /dev/null") + .waitingFor(Wait.forListeningPort()) + .withStartupTimeout(Duration.ofMinutes(5)); + container.start(); + awaitSysLoginReady(); + + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + jdbcQueryExecutor.afterPropertiesSet(); + oceanbaseTemplateMetrics = loadOceanbaseTemplate().getMetrics(); + } + + @AfterAll + void tearDown() throws Exception { + if (jdbcQueryExecutor != null) { + jdbcQueryExecutor.destroy(); + } + if (container != null) { + container.stop(); + } + } + + @TestFactory + Stream shouldCollectOfficialOceanbaseTemplateThroughJdbcQueryAdapter() { + return oceanbaseTemplateMetrics.stream() + .map(templateMetric -> DynamicTest.dynamicTest(templateMetric.getName(), + () -> verifyTemplateMetric(templateMetric))); + } + + private void verifyTemplateMetric(Metrics templateMetric) throws Exception { + Metrics metric = materializeMetric(templateMetric); + if ("process_state".equals(metric.getName())) { + startBackgroundSleepQuery(); + } + CollectRep.MetricsData metricsData = collect(metric); + assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(), + () -> metric.getName() + " failed: " + metricsData.getMsg()); + assertEquals(metric.getFields().size(), metricsData.getFieldsCount(), + () -> metric.getName() + " fields should still be produced by the original parser"); + if ("basic".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0, "basic should return data"); + assertNotNull(metricsData.getValues().getFirst().getColumns(0)); + assertTrue(!Objects.equals(CommonConstants.NULL_VALUE, metricsData.getValues().getFirst().getColumns(0)), + "basic.version should be collected through the adapted query path"); + } + if ("tenant".equals(metric.getName()) || "sql".equals(metric.getName()) || "process_state".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0, () -> metric.getName() + " should return at least one row"); + } + } + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L) + .tenantId(1L) + .app("oceanbase") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { + }); + CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch(); + MetricsCollect metricsCollect = new MetricsCollect( + metric, + new StubTimeout(timerTask), + collectDataDispatch, + "collector-test", + List.of()); + metricsCollect.run(); + assertNotNull(collectDataDispatch.metricsData, metric.getName() + " should dispatch metrics data"); + return collectDataDispatch.metricsData; + } + + private Metrics materializeMetric(Metrics templateMetric) { + Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbcProtocol = metric.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(2881))); + jdbcProtocol.setUsername(OCEANBASE_USERNAME); + jdbcProtocol.setPassword(OCEANBASE_PASSWORD); + jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(12).toMillis())); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setUrl(null); + jdbcProtocol.setSshTunnel(null); + jdbcProtocol.setDatabase(OCEANBASE_DATABASE); + if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) { + metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList())); + } + return metric; + } + + private Job loadOceanbaseTemplate() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-oceanbase.yml") + .toAbsolutePath() + .normalize(); + Yaml yaml = new Yaml(); + try (Reader reader = Files.newBufferedReader(template)) { + return yaml.loadAs(reader, Job.class); + } + } + + private void awaitSysLoginReady() throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofMinutes(3).toMillis(); + while (System.currentTimeMillis() < deadline) { + try { + var result = container.execInContainer("sh", "-lc", obclientCommand("select 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for OceanBase observer bootstrap to finish and accept sys tenant logins. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for OceanBase sys tenant login to become ready"); + } + + private void startBackgroundSleepQuery() throws Exception { + String command = "nohup sh -lc '" + obclientCommand("select sleep(15);").replace("'", "'\"'\"'") + " >/tmp/oceanbase-process-state.log 2>&1' >/dev/null 2>&1 &"; + container.execInContainer("sh", "-lc", command); + Thread.sleep(500); + } + + private String obclientCommand(String sql) { + StringBuilder command = new StringBuilder("obclient -h127.0.0.1 -P2881 -u") + .append(OCEANBASE_USERNAME) + .append(" -D") + .append(OCEANBASE_DATABASE) + .append(" -A "); + if (!OCEANBASE_PASSWORD.isEmpty()) { + command.append("-p").append(OCEANBASE_PASSWORD).append(' '); + } + command.append("-e \"").append(sql.replace("\"", "\\\"")).append('"'); + return command.toString(); + } + + private static final class CapturingCollectDataDispatch implements CollectDataDispatch { + + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) { + this.metricsData = metricsData; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, List metricsDataList) { + if (metricsDataList != null && !metricsDataList.isEmpty()) { + this.metricsData = metricsDataList.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/TidbJdbcQueryAdapterIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/TidbJdbcQueryAdapterIntegrationTest.java new file mode 100644 index 00000000000..0d57526d95e --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/TidbJdbcQueryAdapterIntegrationTest.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.Reader; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TidbJdbcQueryAdapterIntegrationTest { + + private static final String TIDB_USERNAME = "root"; + private static final String TIDB_PASSWORD = ""; + private static final HttpClient HTTP_CLIENT = HttpClient.newHttpClient(); + + private GenericContainer container; + private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor; + private Metrics tidbBasicMetric; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + new CollectStrategyFactory().run(); + container = new GenericContainer<>(DockerImageName.parse("pingcap/tidb:v7.5.1")) + .withCommand("--store=unistore", "--path=") + .withExposedPorts(4000, 10080) + .waitingFor(Wait.forLogMessage(".*server is running MySQL protocol.*", 1)); + container.start(); + waitForStatusEndpoint(); + + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + jdbcQueryExecutor.afterPropertiesSet(); + tidbBasicMetric = loadTidbBasicMetric(); + } + + @AfterAll + void tearDown() throws Exception { + if (jdbcQueryExecutor != null) { + jdbcQueryExecutor.destroy(); + } + if (container != null) { + container.stop(); + } + } + + @Test + void shouldCollectTidbBasicMetricThroughMysqlCompatibleQueryAdapter() { + Metrics metric = materializeMetric(tidbBasicMetric); + CollectRep.MetricsData metricsData = collect(metric); + assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(), metricsData.getMsg()); + assertEquals(metric.getFields().size(), metricsData.getFieldsCount()); + assertTrue(metricsData.getValuesCount() > 0); + assertNotNull(metricsData.getValues().getFirst()); + assertTrue(metricsData.getValues().getFirst().getColumnsList().stream() + .anyMatch(value -> !Objects.equals(CommonConstants.NULL_VALUE, value) && !value.isEmpty()), + "TiDB basic should still return at least one concrete field value through the adapted query path"); + } + + private Metrics materializeMetric(Metrics templateMetric) { + Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbcProtocol = metric.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(4000))); + jdbcProtocol.setUsername(TIDB_USERNAME); + jdbcProtocol.setPassword(TIDB_PASSWORD); + jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis())); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setDatabase(null); + jdbcProtocol.setUrl(null); + jdbcProtocol.setSshTunnel(null); + return metric; + } + + private Metrics loadTidbBasicMetric() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-tidb.yml") + .toAbsolutePath() + .normalize(); + Yaml yaml = new Yaml(); + try (Reader reader = Files.newBufferedReader(template)) { + Job job = yaml.loadAs(reader, Job.class); + return job.getMetrics().stream() + .filter(metric -> "basic".equals(metric.getName())) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Unable to locate the basic metric in app-tidb.yml")); + } + } + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L) + .tenantId(1L) + .app("tidb") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { + }); + CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch(); + MetricsCollect metricsCollect = new MetricsCollect( + metric, + new StubTimeout(timerTask), + collectDataDispatch, + "collector-test", + List.of()); + metricsCollect.run(); + return collectDataDispatch.metricsData; + } + + private void waitForStatusEndpoint() throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis(); + String statusUrl = "http://" + container.getHost() + ":" + container.getMappedPort(10080) + "/status"; + while (System.currentTimeMillis() < deadline) { + try { + HttpRequest request = HttpRequest.newBuilder(URI.create(statusUrl)) + .GET() + .timeout(Duration.ofSeconds(3)) + .build(); + HttpResponse response = HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() == 200) { + return; + } + } catch (Exception ignored) { + // Wait for the TiDB status endpoint to become available. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for TiDB status endpoint"); + } + + private static final class CapturingCollectDataDispatch implements CollectDataDispatch { + + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) { + this.metricsData = metricsData; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, List metricsDataList) { + if (metricsDataList != null && !metricsDataList.isEmpty()) { + this.metricsData = metricsDataList.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactoryTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactoryTest.java new file mode 100644 index 00000000000..45310f52abe --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactoryTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.strategy; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.apache.hertzbeat.collector.dispatch.DispatchConstants; +import org.junit.jupiter.api.Test; + +class CollectStrategyFactoryTest { + + @Test + void shouldRegisterCommonProtocolsFromServiceLoader() throws Exception { + new CollectStrategyFactory().run(); + + assertNotNull(CollectStrategyFactory.invoke(DispatchConstants.PROTOCOL_JDBC)); + assertNotNull(CollectStrategyFactory.invoke(DispatchConstants.PROTOCOL_HTTP)); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/pom.xml b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/pom.xml new file mode 100644 index 00000000000..6c631b18978 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/pom.xml @@ -0,0 +1,77 @@ + + + + 4.0.0 + + org.apache.hertzbeat + hertzbeat-collector + 2.0-SNAPSHOT + + + hertzbeat-collector-mysql-r2dbc + ${project.artifactId} + + + ${java.version} + ${java.version} + UTF-8 + + + + + io.asyncer + r2dbc-mysql + ${r2dbc-mysql.version} + + + org.springframework.boot + spring-boot-autoconfigure + + + org.projectlombok + lombok + provided + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.testcontainers + testcontainers + ${testcontainers.version} + test + + + org.testcontainers + testcontainers-junit-jupiter + ${testcontainers.version} + test + + + io.netty + netty-tcnative-boringssl-static + 2.0.69.Final + test + + + diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlQueryExecutor.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlQueryExecutor.java new file mode 100644 index 00000000000..0c476039e89 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlQueryExecutor.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +/** + * Internal collector-side MySQL query executor. + */ +public interface MysqlQueryExecutor { + + /** + * Execute a single read-only SQL statement. + * + * @param sql SQL to execute + * @param options execution options and target connection settings + * @return normalized query result + */ + QueryResult execute(String sql, QueryOptions options); +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConfiguration.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConfiguration.java new file mode 100644 index 00000000000..fcce4781bcd --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConfiguration.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Spring beans for the collector-side MySQL R2DBC route. + */ +@Configuration(proxyBeanMethods = false) +public class MysqlR2dbcConfiguration { + + @Bean + @ConditionalOnMissingBean + public SqlGuard mysqlR2dbcSqlGuard() { + return new SqlGuard(); + } + + @Bean + @ConditionalOnMissingBean + public ResultSetMapper mysqlR2dbcResultSetMapper() { + return new ResultSetMapper(); + } + + @Bean + @ConditionalOnMissingBean + public MysqlR2dbcConnectionFactoryProvider mysqlR2dbcConnectionFactoryProvider() { + return new MysqlR2dbcConnectionFactoryProvider(); + } + + @Bean + @ConditionalOnMissingBean(MysqlQueryExecutor.class) + public MysqlQueryExecutor mysqlQueryExecutor(MysqlR2dbcConnectionFactoryProvider connectionFactoryProvider, + ResultSetMapper resultSetMapper, + SqlGuard sqlGuard) { + return new MysqlR2dbcQueryExecutor(connectionFactoryProvider, resultSetMapper, sqlGuard); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConnectionFactoryProvider.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConnectionFactoryProvider.java new file mode 100644 index 00000000000..3121538c33c --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConnectionFactoryProvider.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import io.asyncer.r2dbc.mysql.MySqlConnectionConfiguration; +import io.asyncer.r2dbc.mysql.MySqlConnectionFactory; +import io.asyncer.r2dbc.mysql.constant.SslMode; +import io.asyncer.r2dbc.mysql.constant.TlsVersions; +import org.springframework.util.StringUtils; + +/** + * Builds R2DBC MySQL connection factories for the collector. + */ +public class MysqlR2dbcConnectionFactoryProvider { + + /** + * Create a connection factory for a target MySQL instance. + * + * @param options target connection settings + * @return connection factory + */ + public MySqlConnectionFactory create(QueryOptions options) { + return create(options, SslMode.PREFERRED); + } + + public MySqlConnectionFactory create(QueryOptions options, SslMode sslMode) { + if (!StringUtils.hasText(options.getHost())) { + throw new IllegalArgumentException("R2DBC MySQL collector route requires a target host"); + } + MySqlConnectionConfiguration.Builder builder = MySqlConnectionConfiguration.builder() + .host(options.getHost()) + .port(options.getPort()) + .connectTimeout(options.getTimeout()) + .sslMode(sslMode) + .tcpKeepAlive(true) + .tcpNoDelay(true); + if (sslMode.startSsl()) { + // Pin to TLSv1.2 because JDK 25 + the current server matrix is unreliable when TLSv1.3 is negotiated. + builder.tlsVersion(TlsVersions.TLS1_2); + } + if (StringUtils.hasText(options.getUsername())) { + builder.user(options.getUsername()); + } + if (options.getPassword() != null) { + builder.password(options.getPassword()); + } + if (StringUtils.hasText(options.resolvedDatabase())) { + builder.database(options.resolvedDatabase()); + } + return MySqlConnectionFactory.from(builder.build()); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutor.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutor.java new file mode 100644 index 00000000000..ab74d430204 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutor.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import io.asyncer.r2dbc.mysql.MySqlConnectionFactory; +import io.asyncer.r2dbc.mysql.constant.SslMode; +import io.r2dbc.spi.Connection; +import io.r2dbc.spi.Statement; +import java.time.Duration; +import java.util.Locale; +import java.util.concurrent.TimeoutException; +import reactor.core.publisher.Mono; + +/** + * Collector-side MySQL query executor backed by R2DBC. + */ +public class MysqlR2dbcQueryExecutor implements MysqlQueryExecutor { + + private final MysqlR2dbcConnectionFactoryProvider connectionFactoryProvider; + private final ResultSetMapper resultSetMapper; + private final SqlGuard sqlGuard; + + public MysqlR2dbcQueryExecutor(MysqlR2dbcConnectionFactoryProvider connectionFactoryProvider, + ResultSetMapper resultSetMapper, + SqlGuard sqlGuard) { + this.connectionFactoryProvider = connectionFactoryProvider; + this.resultSetMapper = resultSetMapper; + this.sqlGuard = sqlGuard; + } + + @Override + public QueryResult execute(String sql, QueryOptions options) { + String normalizedSql = sqlGuard.normalizeAndValidate(sql); + QueryResult firstAttempt = executeOnce(normalizedSql, options, SslMode.PREFERRED); + if (!firstAttempt.hasError() || !shouldRetryWithoutSsl(firstAttempt.getError())) { + return firstAttempt; + } + QueryResult fallbackAttempt = executeOnce(normalizedSql, options, SslMode.DISABLED); + if (!fallbackAttempt.hasError()) { + return fallbackAttempt; + } + if (requiresSslCompatibleAuth(fallbackAttempt.getError())) { + return QueryResult.builder() + .error(fallbackAttempt.getError() + + ". This route currently needs a TLS-compatible runtime or a mysql_native_password monitoring user.") + .build(); + } + return fallbackAttempt; + } + + private QueryResult executeOnce(String sql, QueryOptions options, SslMode sslMode) { + MySqlConnectionFactory connectionFactory = connectionFactoryProvider.create(options, sslMode); + Duration timeout = options.getTimeout().plusSeconds(1); + Connection connection = null; + try { + connection = Mono.from(connectionFactory.create()).block(timeout); + if (connection == null) { + return QueryResult.builder().error("R2DBC MySQL collector route could not create a connection").build(); + } + QueryResult result = map(connection, sql, options).block(timeout); + if (result == null) { + return QueryResult.builder().error("R2DBC MySQL collector route returned no result").build(); + } + return result; + } catch (IllegalArgumentException exception) { + throw exception; + } catch (Exception exception) { + String message = extractErrorMessage(exception, options.getTimeout()); + return QueryResult.builder() + .error(message) + .build(); + } finally { + if (connection != null) { + safelyClose(connection); + } + } + } + + private void safelyClose(Connection connection) { + try { + Mono.from(connection.close()) + .timeout(Duration.ofSeconds(1), Mono.empty()) + .onErrorResume(_ -> Mono.empty()) + .block(Duration.ofSeconds(2)); + } catch (Exception ignored) { + // Best-effort close only. Query completion must not be turned into a failure by cleanup. + } + } + + private String extractErrorMessage(Exception exception, Duration timeout) { + if (isTimeoutException(exception)) { + return "Query timed out after " + timeout.toMillis() + "ms"; + } + String message = exception.getMessage(); + if ((message == null || message.isBlank()) && exception.getCause() != null) { + message = exception.getCause().getMessage(); + } + return message == null || message.isBlank() ? exception.getClass().getSimpleName() : message; + } + + private boolean isTimeoutException(Throwable throwable) { + Throwable current = throwable; + while (current != null) { + if (current instanceof TimeoutException) { + return true; + } + String message = current.getMessage(); + if (message != null && message.toLowerCase(Locale.ROOT).contains("timeout on blocking read")) { + return true; + } + current = current.getCause(); + } + return false; + } + + private boolean shouldRetryWithoutSsl(String error) { + if (error == null) { + return false; + } + String normalized = error.toLowerCase(Locale.ROOT); + return normalized.contains("handshake_failure") + || normalized.contains("ssl/tls handshake") + || normalized.contains("closedchannelexception") + || normalized.contains("connection unexpectedly closed"); + } + + private boolean requiresSslCompatibleAuth(String error) { + if (error == null) { + return false; + } + String normalized = error.toLowerCase(Locale.ROOT); + return normalized.contains("caching_sha2_password") + || normalized.contains("must require ssl"); + } + + private Mono map(Connection connection, String sql, QueryOptions options) { + Statement statement = connection.createStatement(sql); + // Keep the query path read-only and deterministic, and let JdbcCommonCollect own the existing parser flow. + return resultSetMapper.map(statement, options.getTimeout(), options.getMaxRows()); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryOptions.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryOptions.java new file mode 100644 index 00000000000..ffad2d25617 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryOptions.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import java.time.Duration; + +import lombok.Builder; +import lombok.Getter; +import lombok.ToString; +import org.springframework.util.StringUtils; + +/** + * Collector-only execution options for a target MySQL query. + */ +@Getter +@Builder(toBuilder = true) +@ToString(exclude = "password") +public class QueryOptions { + + private final String host; + @Builder.Default + private final int port = 3306; + private final String username; + private final String password; + private final String database; + private final String schema; + @Builder.Default + private final Duration timeout = Duration.ofSeconds(6); + @Builder.Default + private final int maxRows = 1000; + @Builder.Default + private final int fetchSize = 256; + @Builder.Default + private final boolean readOnly = true; + + public String resolvedDatabase() { + if (StringUtils.hasText(database)) { + return database; + } + if (StringUtils.hasText(schema)) { + return schema; + } + return null; + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryResult.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryResult.java new file mode 100644 index 00000000000..93c5e580a04 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryResult.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import java.util.Collections; +import java.util.List; + +import lombok.Builder; +import lombok.Getter; + +/** + * Normalized query response. + */ +@Getter +@Builder(toBuilder = true) +public class QueryResult { + + @Builder.Default + private final List columns = Collections.emptyList(); + @Builder.Default + private final List> rows = Collections.emptyList(); + @Builder.Default + private final long elapsedMs = 0L; + @Builder.Default + private final int rowCount = 0; + private final String error; + + public boolean hasError() { + return error != null && !error.isBlank(); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapper.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapper.java new file mode 100644 index 00000000000..5ad8971c527 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapper.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import io.r2dbc.spi.Row; +import io.r2dbc.spi.RowMetadata; +import io.r2dbc.spi.Statement; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +/** + * Maps R2DBC results into the collector-neutral result model. + */ +public class ResultSetMapper { + + /** + * Execute and map a single statement. + * + * @param statement statement to execute + * @param timeout timeout for the query + * @param maxRows max rows to materialize + * @return normalized query result + */ + public Mono map(Statement statement, Duration timeout, int maxRows) { + long startNanos = System.nanoTime(); + ResultAccumulator accumulator = new ResultAccumulator(); + return Flux.from(statement.execute()) + .concatMap(result -> Flux.from(result.map(accumulator::mapRow))) + .take(maxRows) + .collectList() + .map(rows -> accumulator.toQueryResult(rows, elapsedMillis(startNanos))) + .timeout(timeout); + } + + private static long elapsedMillis(long startNanos) { + return Duration.ofNanos(System.nanoTime() - startNanos).toMillis(); + } + + private static ColumnLayout extractColumns(RowMetadata metadata) { + List columns = new ArrayList<>(); + metadata.getColumnMetadatas().forEach(columnMetadata -> columns.add(columnMetadata.getName())); + return new ColumnLayout(List.copyOf(columns), columns.size()); + } + + private static List extractRow(Row row, int columnCount) { + List values = new ArrayList<>(columnCount); + for (int index = 0; index < columnCount; index++) { + Object value = row.get(index); + values.add(value == null ? null : String.valueOf(value)); + } + return values; + } + + private record ColumnLayout(List columns, int columnCount) { + } + + private static final class ResultAccumulator { + + private ColumnLayout columnLayout; + + private List mapRow(Row row, RowMetadata metadata) { + if (columnLayout == null) { + columnLayout = extractColumns(metadata); + } + return extractRow(row, columnLayout.columnCount()); + } + + private QueryResult toQueryResult(List> rows, long elapsedMs) { + return QueryResult.builder() + .columns(columnLayout == null ? List.of() : columnLayout.columns()) + .rows(rows) + .rowCount(rows.size()) + .elapsedMs(elapsedMs) + .build(); + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuard.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuard.java new file mode 100644 index 00000000000..47cb257a236 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuard.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import java.util.Locale; +import java.util.regex.Pattern; + +/** + * Minimal SQL guard for the built-in read-only MySQL collector route. + */ +public class SqlGuard { + + private static final Pattern TRAILING_SEMICOLONS = Pattern.compile(";\\s*$"); + private static final Pattern COMMENTS = Pattern.compile("(/\\*|\\*/|--|#)"); + private static final Pattern FORBIDDEN = Pattern.compile( + "\\b(insert|update|delete|replace|merge|alter|drop|truncate|create|call)\\b"); + + /** + * Normalize a single read-only SQL statement and reject obvious unsafe statements. + * + * @param sql sql text + * @return normalized sql text + */ + public String normalizeAndValidate(String sql) { + if (sql == null || sql.isBlank()) { + throw new IllegalArgumentException("R2DBC MySQL collector route requires a non-empty SQL statement"); + } + String normalized = TRAILING_SEMICOLONS.matcher(sql.trim()).replaceAll("").trim(); + if (normalized.isEmpty()) { + throw new IllegalArgumentException("R2DBC MySQL collector route requires a non-empty SQL statement"); + } + if (normalized.indexOf(';') >= 0) { + throw new IllegalArgumentException("R2DBC MySQL collector route only allows a single SQL statement"); + } + if (COMMENTS.matcher(normalized).find()) { + throw new IllegalArgumentException("R2DBC MySQL collector route does not allow SQL comments"); + } + + String lower = normalized.toLowerCase(Locale.ROOT); + if (!(lower.startsWith("select") || lower.startsWith("show"))) { + throw new IllegalArgumentException("R2DBC MySQL collector route only supports SELECT or SHOW statements"); + } + + String stripped = stripQuotedContent(lower); + if (FORBIDDEN.matcher(stripped).find()) { + throw new IllegalArgumentException("R2DBC MySQL collector route only supports read-only statements"); + } + return normalized; + } + + private String stripQuotedContent(String sql) { + StringBuilder builder = new StringBuilder(sql.length()); + char quote = 0; + for (int i = 0; i < sql.length(); i++) { + char current = sql.charAt(i); + if (quote == 0 && (current == '\'' || current == '"' || current == '`')) { + quote = current; + builder.append(' '); + continue; + } + if (quote != 0 && current == quote) { + quote = 0; + builder.append(' '); + continue; + } + builder.append(quote == 0 ? current : ' '); + } + return builder.toString(); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutorIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutorIntegrationTest.java new file mode 100644 index 00000000000..532c433c7f3 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutorIntegrationTest.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +class MysqlR2dbcQueryExecutorIntegrationTest { + + private static final String TEST_DATABASE = "hzb"; + private static final String TEST_USERNAME = "test"; + private static final String TEST_PASSWORD = "test123"; + + private final MysqlQueryExecutor executor = new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), new ResultSetMapper(), new SqlGuard()); + + @TestFactory + Stream shouldRunReadOnlyQueriesAcrossCompatibilityMatrix() { + return Stream.of( + new DatabaseTarget("mysql-5.7", DockerImageName.parse("mysql:5.7.44"), false, false), + new DatabaseTarget("mysql-8.0-default-auth", DockerImageName.parse("mysql:8.0.36"), false, false), + new DatabaseTarget("mysql-8.0-native-auth", DockerImageName.parse("mysql:8.0.36"), false, true), + new DatabaseTarget("mariadb-11.4", DockerImageName.parse("mariadb:11.4"), true, false)) + .map(target -> DynamicTest.dynamicTest(target.name(), () -> verifyReadOnlyQueries(target))); + } + + @Test + void shouldRejectIllegalSqlBeforeConnecting() { + assertThrows(IllegalArgumentException.class, () -> executor.execute("DELETE FROM sample_metrics", QueryOptions.builder() + .host("127.0.0.1") + .port(3306) + .username("test") + .password("test123") + .database("hzb") + .build())); + } + + @Test + void shouldReturnTimeoutErrorOnSlowQuery() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + DatabaseTarget target = new DatabaseTarget("mysql-8.0-native-auth", DockerImageName.parse("mysql:8.0.36"), false, true); + try (GenericContainer container = createContainer(target)) { + container.start(); + awaitTcpLoginReady(container); + QueryResult result = executor.execute("SELECT SLEEP(3)", buildOptions(container, TEST_DATABASE, Duration.ofSeconds(1))); + assertTrue(result.hasError()); + } + } + + @Test + void shouldSupportMysql8DefaultCachingSha2Users() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + DatabaseTarget target = new DatabaseTarget("mysql-8.0-default-auth", DockerImageName.parse("mysql:8.0.36"), false, false); + try (GenericContainer container = createContainer(target)) { + container.start(); + awaitTcpLoginReady(container); + initSchema(container); + QueryResult result = executor.execute("SELECT 1 AS value", buildOptions(container, TEST_DATABASE, Duration.ofSeconds(5))); + assertFalse(result.hasError(), () -> "mysql-8.0-default-auth failed: " + result.getError()); + assertEquals(List.of("value"), result.getColumns()); + assertEquals(List.of(List.of("1")), result.getRows()); + } + } + + private void verifyReadOnlyQueries(DatabaseTarget target) throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + try (GenericContainer container = createContainer(target)) { + container.start(); + awaitTcpLoginReady(container); + initSchema(container); + + QueryOptions options = buildOptions(container, TEST_DATABASE, Duration.ofSeconds(5)); + + QueryResult selectResult = executor.execute("SELECT 1 AS value", options); + assertFalse(selectResult.hasError(), + () -> target.name() + " SELECT 1 failed: " + selectResult.getError()); + assertEquals(List.of("value"), selectResult.getColumns()); + assertEquals(List.of(List.of("1")), selectResult.getRows()); + + QueryResult showResult = executor.execute("SHOW VARIABLES LIKE 'version%'", options); + assertFalse(showResult.hasError(), + () -> target.name() + " SHOW VARIABLES failed: " + showResult.getError()); + assertTrue(showResult.getRowCount() > 0); + + QueryResult businessResult = executor.execute("SELECT label FROM sample_metrics WHERE id = 1", options); + assertFalse(businessResult.hasError(), + () -> target.name() + " business SQL failed: " + businessResult.getError()); + assertEquals(List.of(List.of("alpha")), businessResult.getRows()); + } + } + + private GenericContainer createContainer(DatabaseTarget target) { + GenericContainer container = new GenericContainer<>(target.image()) + .withExposedPorts(3306) + .waitingFor(Wait.forListeningPort()); + if (target.mariaDb()) { + container.withEnv("MARIADB_DATABASE", TEST_DATABASE) + .withEnv("MARIADB_USER", TEST_USERNAME) + .withEnv("MARIADB_PASSWORD", TEST_PASSWORD) + .withEnv("MARIADB_ROOT_PASSWORD", "root123"); + return container; + } + container.withEnv("MYSQL_DATABASE", TEST_DATABASE) + .withEnv("MYSQL_USER", TEST_USERNAME) + .withEnv("MYSQL_PASSWORD", TEST_PASSWORD) + .withEnv("MYSQL_ROOT_PASSWORD", "root123"); + if (target.mysqlNativePasswordUser()) { + container.withCommand("--default-authentication-plugin=mysql_native_password"); + } + return container; + } + + private QueryOptions buildOptions(GenericContainer container, String database, Duration timeout) { + return QueryOptions.builder() + .host(normalizeLoopbackHost(container.getHost())) + .port(container.getMappedPort(3306)) + .username(TEST_USERNAME) + .password(TEST_PASSWORD) + .database(database) + .timeout(timeout) + .maxRows(1000) + .fetchSize(128) + .readOnly(true) + .build(); + } + + private void awaitTcpLoginReady(GenericContainer container) throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis(); + String command = String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + TEST_USERNAME, + "-p" + TEST_PASSWORD, + TEST_DATABASE, + "-e", + "\"SELECT 1\""); + while (System.currentTimeMillis() < deadline) { + try { + var result = container.execInContainer("sh", "-lc", command); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // The entrypoint may still be switching from the temporary bootstrap server to the final one. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MySQL TCP login to become ready"); + } + + private String normalizeLoopbackHost(String host) { + return host; + } + + private void initSchema(GenericContainer container) throws Exception { + String command = String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + TEST_USERNAME, + "-p" + TEST_PASSWORD, + TEST_DATABASE, + "-e", + "\"CREATE TABLE IF NOT EXISTS sample_metrics (id INT PRIMARY KEY, label VARCHAR(32));", + "REPLACE INTO sample_metrics (id, label) VALUES (1, 'alpha');\""); + container.execInContainer("sh", "-lc", command); + } + + private record DatabaseTarget(String name, DockerImageName image, boolean mariaDb, boolean mysqlNativePasswordUser) { + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlSqlTemplateCompatibilityTest.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlSqlTemplateCompatibilityTest.java new file mode 100644 index 00000000000..687cf0613c0 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlSqlTemplateCompatibilityTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.yaml.snakeyaml.Yaml; + +class MysqlSqlTemplateCompatibilityTest { + + private final SqlGuard sqlGuard = new SqlGuard(); + + @Test + @SuppressWarnings("unchecked") + void shouldAcceptCurrentOfficialMysqlTemplateSql() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mysql.yml") + .toAbsolutePath() + .normalize(); + assertTrue(Files.isRegularFile(template), "MySQL monitor template must exist"); + + Yaml yaml = new Yaml(); + int sqlCount = 0; + try (Reader reader = Files.newBufferedReader(template)) { + Map root = yaml.load(reader); + List> metrics = (List>) root.get("metrics"); + for (Map metric : metrics) { + Map jdbc = (Map) metric.get("jdbc"); + if (jdbc == null) { + continue; + } + Object sql = jdbc.get("sql"); + if (!(sql instanceof String sqlText)) { + continue; + } + String normalized = sqlGuard.normalizeAndValidate(sqlText); + assertFalse(normalized.isBlank(), "Normalized SQL should not be blank"); + sqlCount++; + } + } + assertTrue(sqlCount > 0, "MySQL monitor template should contain SQL statements"); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapperTest.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapperTest.java new file mode 100644 index 00000000000..2ec085b46a6 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapperTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.r2dbc.spi.ColumnMetadata; +import io.r2dbc.spi.Result; +import io.r2dbc.spi.Row; +import io.r2dbc.spi.RowMetadata; +import io.r2dbc.spi.Statement; +import java.time.Duration; +import java.util.List; +import java.util.function.BiFunction; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Flux; + +class ResultSetMapperTest { + + private final ResultSetMapper mapper = new ResultSetMapper(); + + @Test + @SuppressWarnings("unchecked") + void shouldCacheColumnLayoutAcrossRows() { + Statement statement = mock(Statement.class); + Result result = mock(Result.class); + RowMetadata metadata = mock(RowMetadata.class); + Row firstRow = mock(Row.class); + Row secondRow = mock(Row.class); + ColumnMetadata firstColumn = mock(ColumnMetadata.class); + ColumnMetadata secondColumn = mock(ColumnMetadata.class); + + when(firstColumn.getName()).thenReturn("id"); + when(secondColumn.getName()).thenReturn("label"); + doReturn(List.of(firstColumn, secondColumn)).when(metadata).getColumnMetadatas(); + when(firstRow.get(0)).thenReturn(1); + when(firstRow.get(1)).thenReturn("alpha"); + when(secondRow.get(0)).thenReturn(2); + when(secondRow.get(1)).thenReturn("beta"); + when(result.map(any(BiFunction.class))).thenAnswer(invocation -> { + BiFunction> mapping = + (BiFunction>) invocation.getArgument(0); + return Flux.just( + mapping.apply(firstRow, metadata), + mapping.apply(secondRow, metadata)); + }); + doReturn(Flux.just(result)).when(statement).execute(); + + QueryResult queryResult = mapper.map(statement, Duration.ofSeconds(1), 10).block(); + + assertNotNull(queryResult); + assertEquals(List.of("id", "label"), queryResult.getColumns()); + assertEquals(List.of( + List.of("1", "alpha"), + List.of("2", "beta")), queryResult.getRows()); + assertEquals(2, queryResult.getRowCount()); + verify(metadata, times(1)).getColumnMetadatas(); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuardTest.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuardTest.java new file mode 100644 index 00000000000..b46235c47cc --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuardTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +class SqlGuardTest { + + private final SqlGuard sqlGuard = new SqlGuard(); + + @Test + void shouldNormalizeTrailingSemicolon() { + assertEquals("SELECT 1", sqlGuard.normalizeAndValidate("SELECT 1;")); + } + + @Test + void shouldAllowShowStatement() { + assertEquals("SHOW VARIABLES LIKE 'version%'", sqlGuard.normalizeAndValidate("SHOW VARIABLES LIKE 'version%'")); + } + + @Test + void shouldRejectWriteStatement() { + assertThrows(IllegalArgumentException.class, () -> sqlGuard.normalizeAndValidate("DELETE FROM test")); + } + + @Test + void shouldRejectMultipleStatements() { + assertThrows(IllegalArgumentException.class, () -> sqlGuard.normalizeAndValidate("SELECT 1; SELECT 2")); + } + + @Test + void shouldRejectComments() { + assertThrows(IllegalArgumentException.class, () -> sqlGuard.normalizeAndValidate("SELECT 1 -- comment")); + } +} diff --git a/hertzbeat-collector/pom.xml b/hertzbeat-collector/pom.xml index bbb03e841d5..f1e1454623c 100644 --- a/hertzbeat-collector/pom.xml +++ b/hertzbeat-collector/pom.xml @@ -31,12 +31,14 @@ 25 ${java.version} ${java.version} + 2024.0.3 hertzbeat-collector-basic hertzbeat-collector-common hertzbeat-collector-collector + hertzbeat-collector-mysql-r2dbc hertzbeat-collector-mongodb hertzbeat-collector-nebulagraph hertzbeat-collector-rocketmq @@ -45,6 +47,13 @@ + + io.projectreactor + reactor-bom + ${mysql.r2dbc.reactor.bom.version} + pom + import + org.apache.hertzbeat hertzbeat-collector-common @@ -55,6 +64,11 @@ hertzbeat-collector-basic ${hertzbeat.version} + + org.apache.hertzbeat + hertzbeat-collector-mysql-r2dbc + ${hertzbeat.version} + org.apache.hertzbeat hertzbeat-collector-mongodb diff --git a/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/pom.xml new file mode 100644 index 00000000000..49b2f517428 --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/pom.xml @@ -0,0 +1,124 @@ + + + + 4.0.0 + + org.apache.hertzbeat + hertzbeat-e2e + 2.0-SNAPSHOT + + + hertzbeat-collector-mysql-r2dbc-e2e + + + ${java.version} + ${java.version} + UTF-8 + 2024.0.3 + + + + + + io.projectreactor + reactor-bom + ${mysql.r2dbc.reactor.bom.version} + pom + import + + + + + + + org.apache.hertzbeat + hertzbeat-startup + ${hertzbeat.version} + test + + + com.mysql + mysql-connector-j + + + org.springframework.boot + spring-boot-starter-webflux + + + + + org.apache.hertzbeat + hertzbeat-collector-common-e2e + ${hertzbeat.version} + test + test-jar + + + org.apache.hertzbeat + hertzbeat-collector-basic + ${hertzbeat.version} + test + + + org.apache.hertzbeat + hertzbeat-collector-common + ${hertzbeat.version} + test + + + org.apache.hertzbeat + hertzbeat-collector-collector + ${hertzbeat.version} + test + + + io.projectreactor.netty + reactor-netty-core + test + + + + org.testcontainers + testcontainers-junit-jupiter + test + + + org.testcontainers + testcontainers-mysql + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + false + + + + + + diff --git a/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/AbstractMysqlR2dbcCollectE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/AbstractMysqlR2dbcCollectE2eTest.java new file mode 100644 index 00000000000..aff3ffb50cc --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/AbstractMysqlR2dbcCollectE2eTest.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.mysql; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.hertzbeat.collector.collect.AbstractCollectE2eTest; +import org.apache.hertzbeat.collector.collect.database.JdbcCommonCollect; +import org.apache.hertzbeat.collector.collect.database.mysql.MysqlCollectorProperties; +import org.apache.hertzbeat.collector.collect.database.mysql.MysqlJdbcDriverAvailability; +import org.apache.hertzbeat.collector.collect.database.mysql.MysqlR2dbcJdbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.util.CollectUtil; +import org.apache.hertzbeat.common.entity.job.Configmap; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.job.protocol.Protocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.junit.jupiter.api.Assertions; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +/** + * Shared MySQL-compatible E2E support for the collector-side R2DBC adapter. + */ +abstract class AbstractMysqlR2dbcCollectE2eTest extends AbstractCollectE2eTest { + + protected static final String TEST_DATABASE = "hzb"; + protected static final String TEST_USERNAME = "test"; + protected static final String TEST_PASSWORD = "test123"; + protected static final String ROOT_PASSWORD = "root123"; + + protected GenericContainer container; + private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor; + + protected void setUpTarget(DatabaseTarget target) throws Exception { + super.setUp(); + collect = new JdbcCommonCollect(); + metrics = new Metrics(); + + container = createContainer(target); + container.start(); + awaitTcpLoginReady(); + initMonitoringData(); + + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + jdbcQueryExecutor.afterPropertiesSet(); + } + + protected void tearDownTarget() throws Exception { + if (jdbcQueryExecutor != null) { + jdbcQueryExecutor.destroy(); + jdbcQueryExecutor = null; + } + if (container != null) { + container.stop(); + container = null; + } + } + + protected void assertMysqlJdbcDriverAbsent() { + Assertions.assertThrows(ClassNotFoundException.class, () -> Class.forName("com.mysql.cj.jdbc.Driver")); + } + + protected void collectMysqlTemplate(Set metricFilter) throws Exception { + Job mysqlJob = appService.getAppDefine("mysql"); + List> configmapFromPreCollectData = new LinkedList<>(); + for (Metrics metricsDef : mysqlJob.getMetrics()) { + if (metricFilter != null && !metricFilter.contains(metricsDef.getName())) { + continue; + } + metricsDef = CollectUtil.replaceCryPlaceholderToMetrics(metricsDef, + configmapFromPreCollectData.isEmpty() ? new HashMap<>() : configmapFromPreCollectData.getFirst()); + String metricName = metricsDef.getName(); + if ("process_state".equals(metricName)) { + startBackgroundSleepQuery(); + } + if ("slow_sql".equals(metricName)) { + generateSlowQuery(); + } + CollectRep.MetricsData metricsData = validateMetricsCollection(metricsDef, metricName, true); + configmapFromPreCollectData = CollectUtil.getConfigmapFromPreCollectData(metricsData); + } + } + + @Override + protected CollectRep.MetricsData.Builder collectMetrics(Metrics metricsDef) { + JdbcProtocol jdbcProtocol = (JdbcProtocol) buildProtocol(metricsDef); + metrics.setJdbc(jdbcProtocol); + CollectRep.MetricsData.Builder metricsData = CollectRep.MetricsData.newBuilder(); + metricsData.setApp("mysql"); + return collectMetricsData(metrics, metricsDef, metricsData); + } + + @Override + protected Protocol buildProtocol(Metrics metricsDef) { + JdbcProtocol jdbcProtocol = metricsDef.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306))); + jdbcProtocol.setUsername(TEST_USERNAME); + jdbcProtocol.setPassword(TEST_PASSWORD); + jdbcProtocol.setDatabase(TEST_DATABASE); + jdbcProtocol.setTimeout("8000"); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setUrl(null); + jdbcProtocol.setSshTunnel(null); + return jdbcProtocol; + } + + private GenericContainer createContainer(DatabaseTarget target) { + GenericContainer mysql = new GenericContainer<>(target.image()) + .withExposedPorts(3306) + .waitingFor(Wait.forListeningPort()); + if (target.mariaDb()) { + return mysql.withEnv("MARIADB_DATABASE", TEST_DATABASE) + .withEnv("MARIADB_USER", TEST_USERNAME) + .withEnv("MARIADB_PASSWORD", TEST_PASSWORD) + .withEnv("MARIADB_ROOT_PASSWORD", ROOT_PASSWORD); + } + return mysql.withEnv("MYSQL_DATABASE", TEST_DATABASE) + .withEnv("MYSQL_USER", TEST_USERNAME) + .withEnv("MYSQL_PASSWORD", TEST_PASSWORD) + .withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD); + } + + private void initMonitoringData() throws Exception { + execRoot("GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';" + + " GRANT PROCESS ON *.* TO '" + TEST_USERNAME + "'@'%';" + + " SET GLOBAL log_output='TABLE';" + + " SET GLOBAL slow_query_log='ON';" + + " SET GLOBAL long_query_time=0;" + + " FLUSH PRIVILEGES;"); + generateSlowQuery(); + } + + private void generateSlowQuery() throws Exception { + execUser(TEST_DATABASE, "SELECT SLEEP(0.2);"); + Thread.sleep(300); + } + + private void startBackgroundSleepQuery() throws Exception { + String command = String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "nohup sh -lc", + "'$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + TEST_USERNAME, + "-p" + TEST_PASSWORD, + TEST_DATABASE, + "-e", + "\"SELECT SLEEP(15)\" >/tmp/process-state.log 2>&1'", + ">/dev/null 2>&1 &"); + container.execInContainer("sh", "-lc", command); + Thread.sleep(500); + } + + private void awaitTcpLoginReady() throws Exception { + long deadline = System.currentTimeMillis() + 30_000L; + while (System.currentTimeMillis() < deadline) { + try { + var result = container.execInContainer("sh", "-lc", + mysqlCliCommand(TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE, "SELECT 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for the MySQL entrypoint to finish bootstrapping and switch to the final TCP listener. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MySQL-compatible TCP login to become ready"); + } + + private void execRoot(String sql) throws Exception { + var result = container.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("root mysql command failed: " + result.getStderr()); + } + } + + private void execUser(String database, String sql) throws Exception { + var result = container.execInContainer("sh", "-lc", mysqlCliCommand(TEST_USERNAME, TEST_PASSWORD, database, sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("user mysql command failed: " + result.getStderr()); + } + } + + private String mysqlCliCommand(String username, String password, String database, String sql) { + return String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + username, + "-p" + password, + database, + "-e", + "\"" + sql.replace("\"", "\\\"") + "\""); + } + + protected record DatabaseTarget(String name, DockerImageName image, boolean mariaDb) { + } +} diff --git a/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectCompatibilityE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectCompatibilityE2eTest.java new file mode 100644 index 00000000000..7c34f8a6947 --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectCompatibilityE2eTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.mysql; + +import java.util.Set; +import java.util.stream.Stream; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.testcontainers.utility.DockerImageName; + +/** + * Compatibility E2E coverage for the collector-side MySQL R2DBC adapter. + */ +class MysqlR2dbcCollectCompatibilityE2eTest extends AbstractMysqlR2dbcCollectE2eTest { + + private static final Set MARIADB_REPRESENTATIVE_METRICS = + Set.of("basic", "process_state", "slow_sql"); + + @TestFactory + Stream shouldCollectMysqlTemplateAcrossCompatibilityMatrixWithoutMysqlJdbcDriver() { + return Stream.of( + new MatrixTarget( + new DatabaseTarget("mysql-5.7.44", DockerImageName.parse("mysql:5.7.44"), false), + null), + new MatrixTarget( + new DatabaseTarget("mysql-8.0.36", DockerImageName.parse("mysql:8.0.36"), false), + null), + new MatrixTarget( + new DatabaseTarget("mariadb-11.4", DockerImageName.parse("mariadb:11.4"), true), + MARIADB_REPRESENTATIVE_METRICS)) + .map(target -> DynamicTest.dynamicTest(target.databaseTarget().name(), () -> verifyTarget(target))); + } + + private void verifyTarget(MatrixTarget target) throws Exception { + setUpTarget(target.databaseTarget()); + try { + assertMysqlJdbcDriverAbsent(); + collectMysqlTemplate(target.metricFilter()); + } finally { + tearDownTarget(); + } + } + + private record MatrixTarget(DatabaseTarget databaseTarget, Set metricFilter) { + } +} diff --git a/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectE2eTest.java new file mode 100644 index 00000000000..9bd4ebafe02 --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectE2eTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.mysql; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.testcontainers.utility.DockerImageName; + +/** + * E2E test for collector-side MySQL monitoring through the R2DBC query adapter. + */ +@Slf4j +class MysqlR2dbcCollectE2eTest extends AbstractMysqlR2dbcCollectE2eTest { + + @Test + void shouldCollectMysqlTemplateWithoutMysqlJdbcDriver() throws Exception { + DatabaseTarget target = new DatabaseTarget("mysql-8.0.36", DockerImageName.parse("mysql:8.0.36"), false); + setUpTarget(target); + try { + assertMysqlJdbcDriverAbsent(); + collectMysqlTemplate(null); + } finally { + tearDownTarget(); + } + } +} diff --git a/hertzbeat-e2e/pom.xml b/hertzbeat-e2e/pom.xml index 445a8a50776..6249cf0d447 100644 --- a/hertzbeat-e2e/pom.xml +++ b/hertzbeat-e2e/pom.xml @@ -31,6 +31,7 @@ hertzbeat-collector-common-e2e hertzbeat-collector-kafka-e2e hertzbeat-collector-basic-e2e + hertzbeat-collector-mysql-r2dbc-e2e hertzbeat-log-e2e diff --git a/hertzbeat-startup/pom.xml b/hertzbeat-startup/pom.xml index b881e374d4c..90d3a3a663f 100644 --- a/hertzbeat-startup/pom.xml +++ b/hertzbeat-startup/pom.xml @@ -116,6 +116,18 @@ netty-all + + + io.projectreactor.netty + reactor-netty-core + 1.2.3 + + + io.projectreactor.netty + reactor-netty-http + 1.2.3 + + org.flywaydb @@ -150,6 +162,12 @@ spring-boot-starter-test test + + org.testcontainers + testcontainers + ${testcontainers.version} + test + diff --git a/hertzbeat-startup/src/main/resources/application.yml b/hertzbeat-startup/src/main/resources/application.yml index ecebc66681f..90c0ea4bb8e 100644 --- a/hertzbeat-startup/src/main/resources/application.yml +++ b/hertzbeat-startup/src/main/resources/application.yml @@ -336,6 +336,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ReactorNettyCompatibilityTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ReactorNettyCompatibilityTest.java new file mode 100644 index 00000000000..b7e5c02e7c2 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ReactorNettyCompatibilityTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup; + +import static org.junit.jupiter.api.Assertions.fail; + +import io.netty.channel.ChannelOption; +import java.time.Duration; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Mono; +import reactor.netty.Connection; +import reactor.netty.tcp.TcpClient; + +/** + * Guards the startup runtime against Reactor Netty / Netty mismatches that only show up when a client loop + * is created for MySQL R2DBC collection. + */ +class ReactorNettyCompatibilityTest { + + @Test + void reactorNettyClientLoopCanBeCreated() { + Connection connection = null; + try { + connection = TcpClient.create() + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 1000) + .host("127.0.0.1") + .port(9) + .connect() + .onErrorResume(throwable -> Mono.empty()) + .block(Duration.ofSeconds(3)); + } catch (NoClassDefFoundError error) { + fail("Startup runtime is missing a Reactor Netty dependency needed by MySQL R2DBC collection", error); + } finally { + if (connection != null) { + connection.disposeNow(); + } + } + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/StartupMysqlR2dbcCompatibilityTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/StartupMysqlR2dbcCompatibilityTest.java new file mode 100644 index 00000000000..b93b6c934dd --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/StartupMysqlR2dbcCompatibilityTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.time.Duration; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.QueryOptions; +import org.apache.hertzbeat.collector.mysql.r2dbc.QueryResult; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +/** + * Verifies that the startup runtime can execute a real MySQL R2DBC query. + */ +class StartupMysqlR2dbcCompatibilityTest { + + private static final String DATABASE = "hzb"; + private static final String USERNAME = "test"; + private static final String PASSWORD = "test123"; + private static final String ROOT_PASSWORD = "root123"; + + private GenericContainer container; + + @AfterEach + void tearDown() { + if (container != null) { + container.stop(); + container = null; + } + } + + @Test + void shouldExecuteMysqlQueryOnStartupRuntimeClasspath() throws Exception { + container = new GenericContainer<>(DockerImageName.parse("mysql:8.0.36")) + .withExposedPorts(3306) + .withEnv("MYSQL_DATABASE", DATABASE) + .withEnv("MYSQL_USER", USERNAME) + .withEnv("MYSQL_PASSWORD", PASSWORD) + .withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD) + .waitingFor(Wait.forListeningPort()); + container.start(); + awaitTcpLoginReady(); + + MysqlR2dbcQueryExecutor executor = new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()); + QueryOptions options = QueryOptions.builder() + .host(container.getHost()) + .port(container.getMappedPort(3306)) + .database(DATABASE) + .username(USERNAME) + .password(PASSWORD) + .timeout(Duration.ofSeconds(8)) + .build(); + + QueryResult result = executor.execute("SELECT 1 AS ok", options); + + assertFalse(result.hasError(), () -> "Unexpected query error: " + result.getError()); + assertEquals(1, result.getRowCount()); + assertEquals("1", result.getRows().getFirst().getFirst()); + } + + private void awaitTcpLoginReady() throws Exception { + long deadline = System.currentTimeMillis() + 30_000L; + while (System.currentTimeMillis() < deadline) { + try { + var result = container.execInContainer("sh", "-lc", + mysqlCliCommand("SELECT 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for the MySQL entrypoint to finish bootstrapping and switch to the final TCP listener. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MySQL TCP login to become ready"); + } + + private String mysqlCliCommand(String sql) { + return String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + USERNAME, + "-p" + PASSWORD, + DATABASE, + "-e", + "\"" + sql.replace("\"", "\\\"") + "\""); + } +} diff --git a/home/docs/download.md b/home/docs/download.md index b6d62fa5f06..e167b88e3c1 100644 --- a/home/docs/download.md +++ b/home/docs/download.md @@ -26,7 +26,7 @@ Download the latest Apache HertzBeat™ release (v1.8.0) as server binary, colle | **Docker Compose** | ~5MB | Full stack deployment | Docker environments | :::tip Native Collector Recommendation -If you do not need MySQL, OceanBase, Oracle, DB2, or other monitoring types that rely on external JDBC drivers from `ext-lib`, you can choose the native collector package for faster startup and lower memory usage. +If you do not need external JDBC drivers from `ext-lib`, you can choose the native collector package for faster startup and lower memory usage. MySQL, MariaDB, and OceanBase are included in this native-friendly path when `mysql-connector-j` is not provided. TiDB follows the same rule for its SQL query metric set. Trade-offs: native packages are platform-specific and do not support runtime `ext-lib` JDBC loading. See [Native Collector Guide](start/native-collector). ::: diff --git a/home/docs/help/mariadb.md b/home/docs/help/mariadb.md index 6bd75d08c48..1d8a155815c 100644 --- a/home/docs/help/mariadb.md +++ b/home/docs/help/mariadb.md @@ -7,11 +7,21 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo > Collect and monitor the general performance Metrics of MariaDB database. Support MariaDB5+. -### Attention, Need Add MYSQL jdbc driver jar +### Driver selection -- Download the MYSQL jdbc driver jar package, such as mysql-connector-java-8.1.0.jar. [https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0](https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0) -- Copy the jar package to the `hertzbeat/ext-lib` directory. -- Restart the HertzBeat service. +MariaDB follows the same automatic routing as MySQL: + +- If `mysql-connector-j` is present in `ext-lib`, the JVM collector or built-in server collector automatically prefers JDBC. +- If `mysql-connector-j` is absent, HertzBeat automatically uses the built-in MySQL-compatible query engine. No extra JAR is required. +- Restart HertzBeat or the standalone JVM collector after adding or removing a JAR in `ext-lib`. + +:::important Collector package selection +MariaDB monitoring supports both JVM and native deployment now. + +- Built-in server collector or JVM collector package: automatically prefers JDBC when `mysql-connector-j` exists in `ext-lib` +- Native collector package: supported when you do not rely on `ext-lib` and want the built-in query engine +- If you explicitly need runtime `ext-lib` JDBC loading, choose the JVM collector package +::: ### Configuration parameter diff --git a/home/docs/help/mysql.md b/home/docs/help/mysql.md index 7d62f7c0a46..51bcce08e17 100644 --- a/home/docs/help/mysql.md +++ b/home/docs/help/mysql.md @@ -7,18 +7,21 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo > Collect and monitor the general performance Metrics of MySQL database. Support MYSQL5+. -### Attention, Need Add MYSQL jdbc driver jar +### Driver selection -- Download the MYSQL jdbc driver jar package, such as mysql-connector-java-8.4.0.jar. [https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.4.0](https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.4.0) -- It is recommended that you use the latest available mysql-connector-java version as there are regular security fixes to JDBC drivers. -- Copy the jar package to the `hertzbeat/ext-lib` directory. -- Restart the HertzBeat service. +HertzBeat now supports two MySQL query paths: + +- If `mysql-connector-j` is present in `ext-lib`, the JVM collector or built-in server collector automatically prefers JDBC. +- If `mysql-connector-j` is absent, HertzBeat automatically uses the built-in MySQL query engine. No extra JAR is required. +- Restart HertzBeat or the standalone JVM collector after adding or removing a JAR in `ext-lib`. +- The automatic decision only checks `ext-lib`. If you want to force one path, set `hertzbeat.collector.mysql.query-engine=jdbc`, `r2dbc`, or `auto`. :::important Collector package selection -MySQL monitoring requires external JDBC driver loading from `ext-lib`. +MySQL monitoring supports both JVM and native deployment now. -- Use HertzBeat server built-in collector or the JVM collector package for MySQL monitoring -- Do not use the native collector package for MySQL monitoring +- Built-in server collector or JVM collector package: automatically prefers JDBC when `mysql-connector-j` exists in `ext-lib` +- Native collector package: supported when you do not rely on `ext-lib` and want the built-in MySQL query engine +- If you explicitly need runtime `ext-lib` JDBC loading, choose the JVM collector package ::: ### Configuration parameter diff --git a/home/docs/help/oceanbase.md b/home/docs/help/oceanbase.md index d4fe52aeb18..3bfcd3cf292 100644 --- a/home/docs/help/oceanbase.md +++ b/home/docs/help/oceanbase.md @@ -7,17 +7,20 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo > Collect and monitor the general performance Metrics of OceanBase database. Support OceanBase 4.0+. -### Attention, Need Add MYSQL jdbc driver jar +### Driver selection -- Download the MYSQL jdbc driver jar package, such as mysql-connector-java-8.1.0.jar. [https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0](https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0) -- Copy the jar package to the `hertzbeat/ext-lib` directory. -- Restart the HertzBeat service. +OceanBase now follows the same automatic routing as the MySQL-compatible query path: + +- If `mysql-connector-j` is present in `ext-lib`, the JVM collector or built-in server collector automatically prefers JDBC. +- If `mysql-connector-j` is absent, HertzBeat automatically uses the built-in MySQL-compatible query engine. No extra JAR is required. +- Restart HertzBeat or the standalone JVM collector after adding or removing a JAR in `ext-lib`. :::important Collector package selection -OceanBase monitoring depends on the external MySQL JDBC driver in `ext-lib`. +OceanBase monitoring now supports both JVM and native deployment. -- Use HertzBeat server built-in collector or the JVM collector package for OceanBase monitoring -- Do not use the native collector package for OceanBase monitoring +- Built-in server collector or JVM collector package: automatically prefers JDBC when `mysql-connector-j` exists in `ext-lib` +- Native collector package: supported when you do not rely on `ext-lib` and want the built-in MySQL-compatible query engine +- If you explicitly need runtime `ext-lib` JDBC loading, choose the JVM collector package ::: ### Configuration parameter diff --git a/home/docs/help/tidb.md b/home/docs/help/tidb.md index 73d8195aab4..78b741ea2c1 100644 --- a/home/docs/help/tidb.md +++ b/home/docs/help/tidb.md @@ -15,6 +15,22 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo **Protocol Use: HTTP and JDBC** +### Driver selection + +TiDB monitoring keeps the HTTP part unchanged, and the SQL query part now follows the same automatic routing as MySQL: + +- If `mysql-connector-j` is present in `ext-lib`, the JVM collector or built-in server collector automatically prefers JDBC for the SQL query metric set. +- If `mysql-connector-j` is absent, HertzBeat automatically uses the built-in MySQL-compatible query engine for the SQL query metric set. No extra JAR is required. +- Restart HertzBeat or the standalone JVM collector after adding or removing a JAR in `ext-lib`. + +:::important Collector package selection +The TiDB template mixes HTTP metrics and MySQL-compatible SQL queries. + +- HTTP metric sets are unaffected by JDBC driver selection +- The built-in SQL query engine can collect the default TiDB `basic` metric set without `mysql-connector-j` +- If you explicitly place `mysql-connector-j` in `ext-lib`, the JVM collector or built-in server collector will still prefer JDBC for the SQL query path +::: + ### Configuration parameter | Parameter name | Parameter help description | diff --git a/home/docs/start/docker-deploy.md b/home/docs/start/docker-deploy.md index 69c2310c023..db5bdd7a0ba 100644 --- a/home/docs/start/docker-deploy.md +++ b/home/docs/start/docker-deploy.md @@ -19,6 +19,7 @@ It is necessary to have Docker environment in your environment. If not installed ```shell $ docker run -d -p 1157:1157 -p 1158:1158 \ + -e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto \ -v $(pwd)/data:/opt/hertzbeat/data \ -v $(pwd)/logs:/opt/hertzbeat/logs \ -v $(pwd)/application.yml:/opt/hertzbeat/config/application.yml \ @@ -35,7 +36,8 @@ It is necessary to have Docker environment in your environment. If not installed - `-v $(pwd)/logs:/opt/hertzbeat/logs` : (optional) Mount the log file to the local host to facilitate viewing. - `-v $(pwd)/application.yml:/opt/hertzbeat/config/application.yml` : (optional) Mount the configuration file to the container (please ensure that the file exists locally). [Download](https://github.com/apache/hertzbeat/raw/master/script/application.yml) - `-v $(pwd)/sureness.yml:/opt/hertzbeat/config/sureness.yml` : (optional) Mount the account configuration file to the container (please ensure that the file exists locally). [Download](https://github.com/apache/hertzbeat/raw/master/script/sureness.yml) - - `-v $(pwd)/ext-lib:/opt/hertzbeat/ext-lib` : (optional) Mount external third-party JAR package [mysql-jdbc](https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip) [oracle-jdbc](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/23.4.0.24.05/ojdbc8-23.4.0.24.05.jar) [oracle-i18n](https://repo.mavenlibs.com/maven/com/oracle/database/nls/orai18n/21.5.0.0/orai18n-21.5.0.0.jar) + - `-v $(pwd)/ext-lib:/opt/hertzbeat/ext-lib` : (optional) Mount external third-party JAR packages when you need runtime JDBC extension. `mysql-jdbc` is only needed if you explicitly want the JDBC path for MySQL-compatible monitoring; [oracle-jdbc](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/23.4.0.24.05/ojdbc8-23.4.0.24.05.jar) and [oracle-i18n](https://repo.mavenlibs.com/maven/com/oracle/database/nls/orai18n/21.5.0.0/orai18n-21.5.0.0.jar) are still required for Oracle monitoring. + - `-e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto` : (optional) Override the MySQL-compatible monitoring query path used by the built-in collector. Supported values: `auto`, `jdbc`, `r2dbc`. - `--name hertzbeat` : (optional) Naming container name hertzbeat - `--restart=always` : (optional) Configure the container to restart automatically. - `apache/hertzbeat` : Use the [official application mirror](https://hub.docker.com/r/apache/hertzbeat) to start the container, if the network times out, use `quay.io/tancloud/hertzbeat` instead. @@ -71,6 +73,7 @@ By deploying multiple HertzBeat Collectors, high availability, load balancing, a -e MODE=public \ -e MANAGER_HOST=127.0.0.1 \ -e MANAGER_PORT=1158 \ + -e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto \ --name hertzbeat-collector apache/hertzbeat-collector ``` @@ -81,6 +84,7 @@ By deploying multiple HertzBeat Collectors, high availability, load balancing, a - `-e MODE=public` : set the running mode(public or private), public cluster or private - `-e MANAGER_HOST=127.0.0.1` : Important, Set the main hertzbeat server ip host, must use the server host instead of 127.0.0.1. - `-e MANAGER_PORT=1158` : (optional) Set the main hertzbeat server port, default 1158. + - `-e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto` : (optional) Override the MySQL-compatible monitoring query path. Supported values: `auto`, `jdbc`, `r2dbc`. - `-v $(pwd)/logs:/opt/hertzbeat-collector/logs` : (optional) Mount the log file to the local host to facilitate viewing. - `--name hertzbeat-collector` : Naming container name hertzbeat-collector - `apache/hertzbeat-collector` : Use the [official application mirror](https://hub.docker.com/r/apache/hertzbeat-collector) to start the container, if the network times out, use `quay.io/tancloud/hertzbeat-collector` instead. diff --git a/home/docs/start/native-collector.md b/home/docs/start/native-collector.md index e1affbbc040..36527ce068f 100644 --- a/home/docs/start/native-collector.md +++ b/home/docs/start/native-collector.md @@ -13,6 +13,8 @@ Typical native-friendly workloads include: - HTTP, HTTPS, website availability, and API checks - Port, ping, SSL certificate, and other network probes +- MySQL, MariaDB, and OceanBase when you do not rely on runtime `ext-lib` JDBC loading +- TiDB when you do not rely on runtime `ext-lib` JDBC loading for its SQL query metric set - Redis, Zookeeper, Kafka, and other non-JDBC monitoring types ## Why use it? @@ -35,10 +37,9 @@ The native collector package is not a drop-in replacement for every JVM collecto Use the JVM collector package if your monitoring depends on external JDBC drivers, especially: -- MySQL, which requires `mysql-connector-j` -- OceanBase, which also depends on the MySQL JDBC driver - Oracle, which requires `ojdbc8` and sometimes `orai18n` - DB2, which requires `jcc` +- Any MySQL, MariaDB, or OceanBase deployment where you explicitly place `mysql-connector-j` in `ext-lib` and want the JDBC path ## Package naming @@ -69,8 +70,9 @@ That means: ## Recommended decision -- Choose the native collector package when you want lower memory usage and faster startup for non-JDBC monitoring. +- Choose the native collector package when you want lower memory usage and faster startup for non-JDBC monitoring, for MySQL, MariaDB, and OceanBase without `ext-lib`, or for TiDB when its SQL query metric set can use the built-in MySQL-compatible query engine. - Choose the JVM collector package when you need `ext-lib`, external JDBC drivers, or JVM-style runtime extensibility. +- For MySQL-compatible monitoring on the JVM collector, `auto` only checks `ext-lib`. If you need to force a path, set `hertzbeat.collector.mysql.query-engine=jdbc`, `r2dbc`, or `auto`. ## How are the official multi-platform packages built? diff --git a/home/docs/start/package-deploy.md b/home/docs/start/package-deploy.md index f2e8981f7cd..c9486dbd053 100644 --- a/home/docs/start/package-deploy.md +++ b/home/docs/start/package-deploy.md @@ -65,7 +65,7 @@ Deploying multiple HertzBeat Collectors can achieve high availability, load bala ::: :::tip Native Collector Recommendation -If your monitoring workload does not depend on external JDBC drivers from `ext-lib`, prefer the native collector package for faster startup and lower memory usage. +If your monitoring workload does not depend on external JDBC drivers from `ext-lib`, prefer the native collector package for faster startup and lower memory usage. MySQL, MariaDB, and OceanBase can also use the native collector package directly when `mysql-connector-j` is not provided. TiDB follows the same rule for its SQL query metric set. Before choosing it, review the trade-offs in [Native Collector Guide](native-collector). ::: @@ -130,14 +130,13 @@ See [Native Collector Guide](native-collector) for package selection, package na If your monitoring depends on external JDBC drivers, use the JVM collector package instead of the native collector package. This currently includes: -- MySQL, which requires `mysql-connector-j` -- OceanBase, which also relies on the MySQL JDBC driver - Oracle, which requires `ojdbc8` and often `orai18n` - DB2, which requires `jcc` +- Any MySQL, MariaDB, or OceanBase deployment where you explicitly place `mysql-connector-j` in `ext-lib` and want the JDBC path Recommended deployment: -- Use the native collector package for HTTP, website, port, ping, and similar non-JDBC monitoring types +- Use the native collector package for HTTP, website, port, ping, similar non-JDBC monitoring types, and for MySQL, MariaDB, or OceanBase without `ext-lib` - Use the JVM collector package when you need `ext-lib` driver extension ::: diff --git a/home/docs/start/quickstart.md b/home/docs/start/quickstart.md index 8e4830ec023..d9d061cb626 100644 --- a/home/docs/start/quickstart.md +++ b/home/docs/start/quickstart.md @@ -59,7 +59,7 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache 3. Run command `$ ./bin/startup.sh` or `bin/startup.bat` 4. Access `http://localhost:1157` to start, default account: `admin/hertzbeat` 5. Deploy collector clusters(Optional) - - If you do not need MySQL, OceanBase, Oracle, DB2, or other `ext-lib` JDBC drivers, prefer the native collector package for faster startup and lower memory usage. See [Native Collector Guide](native-collector). + - If you do not need external JDBC drivers from `ext-lib`, prefer the native collector package for faster startup and lower memory usage. MySQL, MariaDB, and OceanBase can use the built-in query engine directly when `mysql-connector-j` is not provided. TiDB follows the same rule for its SQL query metric set. See [Native Collector Guide](native-collector). - Download the release package `apache-hertzbeat-collector-xx-bin.tar.gz` (JVM collector) or the native collector package for your target platform, such as `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz` or `apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`, to the new machine [Download Page](https://hertzbeat.apache.org/docs/download) - Configure the collector configuration yml file `hertzbeat-collector/config/application.yml`: unique `identity` name, running `mode` (public or private), hertzbeat `manager-host`, hertzbeat `manager-port` @@ -76,7 +76,7 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache ``` - Native collector trade-offs: platform-specific packages, no runtime `ext-lib` JDBC loading, and less suitable for JVM-style runtime classpath extension. See [Native Collector Guide](native-collector). - - If you need MySQL, OceanBase, Oracle, or DB2 monitoring with external JDBC drivers from `ext-lib`, use the JVM collector package. + - If `mysql-connector-j` is present in `ext-lib`, the built-in server collector or JVM collector automatically prefers JDBC for MySQL, MariaDB, and OceanBase after restart. TiDB follows the same rule for its SQL query metric set, while its HTTP metrics stay unchanged. Oracle and DB2 still require the JVM collector package because they depend on external JDBC drivers. - Run command `$ ./bin/startup.sh` or `bin/startup.bat` for the JVM collector package. Run `$ ./bin/startup.sh` for Linux or macOS native collector packages, and `bin\\startup.bat` for the Windows native collector package. - Access the HertzBeat server dashboard at `http://localhost:1157` and confirm the new collector is registered. diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md index 1b467661198..0c226c424de 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md @@ -26,7 +26,7 @@ description: Apache HertzBeat 监控系统下载 - 服务器、采集器、源 | **Docker Compose** | ~5MB | 全栈部署 | Docker 环境 | :::tip Native 采集器推荐 -如果你不需要 MySQL、OceanBase、Oracle、DB2，或其他依赖 `ext-lib` 外部 JDBC 驱动的监控类型，可以优先选择 Native 采集器安装包，通常启动更快、内存更省。 +如果你不需要 `ext-lib` 外部 JDBC 驱动，可以优先选择 Native 采集器安装包，通常启动更快、内存更省。MySQL、MariaDB、OceanBase 在没有提供 `mysql-connector-j` 时也属于这条 Native 友好路径；TiDB 的 SQL 查询指标也遵循同样规则。 它的代价是安装包按平台区分，且不支持运行时 `ext-lib` JDBC 加载。详见 [Native 采集器指南](start/native-collector)。 ::: diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mariadb.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mariadb.md index dd8b3f7d868..6b2223a6396 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mariadb.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mariadb.md @@ -7,11 +7,21 @@ keywords: [开源监控系统, 开源数据库监控, MariaDB数据库监控] > 对MariaDB数据库的通用性能指标进行采集监控。支持MariaDB5+。 -### 注意，必须添加 MYSQL jdbc 驱动 jar +### 驱动选择说明 -- 下载 MYSQL jdbc driver jar, 例如 mysql-connector-java-8.1.0.jar. [https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0](https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0) -- 将此 jar 包拷贝放入 HertzBeat 的安装目录下的 `ext-lib` 目录下. -- 重启 HertzBeat 服务。 +MariaDB 现在和 MySQL 一样支持自动分流： + +- 如果在 `ext-lib` 中放入了 `mysql-connector-j`，JVM 采集器或主程序内置采集器会自动优先走 JDBC。 +- 如果没有放入 `mysql-connector-j`，HertzBeat 会自动切换到内置的 MySQL 兼容查询引擎，不需要额外复制 JAR。 +- 每次增删 `ext-lib` 里的驱动后，都需要重启 HertzBeat 或独立 JVM 采集器。 + +:::important 采集器包选择 +MariaDB 监控现在既支持 JVM 部署，也支持 Native 部署。 + +- 主程序内置采集器或 JVM 采集器安装包：当 `ext-lib` 中存在 `mysql-connector-j` 时会自动优先走 JDBC +- Native 采集器安装包：在不依赖 `ext-lib` 时可直接使用内置查询引擎 +- 如果你明确需要运行时 `ext-lib` JDBC 加载能力，请选择 JVM 采集器安装包 +::: ### 配置参数 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md index bc7e723a558..17bf69eff4d 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md @@ -7,17 +7,21 @@ keywords: [开源监控系统, 开源数据库监控, Mysql数据库监控] > 对MYSQL数据库的通用性能指标进行采集监控。支持MYSQL5+。 -### 注意，必须添加 MYSQL jdbc 驱动 jar +### 驱动选择说明 -- 下载 MYSQL jdbc driver jar, 例如 mysql-connector-java-8.1.0.jar. [https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0](https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0) -- 将此 jar 包拷贝放入 HertzBeat 的安装目录下的 `ext-lib` 目录下. -- 重启 HertzBeat 服务。 +HertzBeat 现在支持两条 MySQL 查询链路： + +- 如果在 `ext-lib` 中放入了 `mysql-connector-j`，JVM 采集器或主程序内置采集器会自动优先走 JDBC。 +- 如果没有放入 `mysql-connector-j`，HertzBeat 会自动切换到内置 MySQL 查询引擎，不需要额外复制 JAR。 +- 每次增删 `ext-lib` 里的驱动后，都需要重启 HertzBeat 或独立 JVM 采集器。 +- 自动分流只检查 `ext-lib`。如果你想显式指定链路，可以配置 `hertzbeat.collector.mysql.query-engine=jdbc`、`r2dbc` 或 `auto`。 :::important 采集器包选择 -MySQL 监控依赖 `ext-lib` 目录下的外置 JDBC 驱动加载能力。 +MySQL 监控现在既支持 JVM 部署，也支持 Native 部署。 -- MySQL 监控请使用 HertzBeat 主程序内置采集器，或 JVM 采集器安装包 -- 不要使用 Native 采集器安装包执行 MySQL 监控 +- 主程序内置采集器或 JVM 采集器安装包：当 `ext-lib` 中存在 `mysql-connector-j` 时会自动优先走 JDBC +- Native 采集器安装包：在不依赖 `ext-lib` 时可直接使用内置 MySQL 查询引擎 +- 如果你明确需要运行时 `ext-lib` JDBC 加载能力，请选择 JVM 采集器安装包 ::: ### 配置参数 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md index 5893d9c1957..415b6445a7a 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md @@ -7,17 +7,20 @@ keywords: [开源监控系统, 开源数据库监控, OceanBase 数据库监控] > 对 OceanBase 数据库的通用性能指标进行采集监控。支持 OceanBase 4.0+。 -### 注意，必须添加 MYSQL jdbc 驱动 jar +### 驱动选择 -- 下载 MYSQL jdbc driver jar, 例如 mysql-connector-java-8.1.0.jar. [https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0](https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0) -- 将此 jar 包拷贝放入 HertzBeat 的安装目录下的 `ext-lib` 目录下. -- 重启 HertzBeat 服务。 +OceanBase 现在和 MySQL 兼容查询路径一样会自动分流： + +- 如果 `ext-lib` 中放入了 `mysql-connector-j`，JVM 采集器或主程序内置采集器会自动优先走 JDBC。 +- 如果没有放入 `mysql-connector-j`，HertzBeat 会自动切换到内置的 MySQL 兼容查询引擎，不需要额外复制 JAR。 +- 在 `ext-lib` 中新增或删除 JAR 后，请重启 HertzBeat 或独立 JVM 采集器。 :::important 采集器包选择 -OceanBase 监控同样依赖 `ext-lib` 目录下的 MySQL JDBC 驱动。 +OceanBase 监控现在同样支持 JVM 和 Native 两种部署方式。 -- OceanBase 监控请使用 HertzBeat 主程序内置采集器，或 JVM 采集器安装包 -- 不要使用 Native 采集器安装包执行 OceanBase 监控 +- 主程序内置采集器或 JVM 采集器安装包：当 `ext-lib` 中存在 `mysql-connector-j` 时会自动优先走 JDBC +- Native 采集器安装包：在不依赖 `ext-lib` 时可直接使用内置的 MySQL 兼容查询引擎 +- 如果你明确需要运行时 `ext-lib` JDBC 加载能力，仍然请选择 JVM 采集器安装包 ::: ### 配置参数 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/tidb.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/tidb.md index 69bdd6fd40b..76280d8ebc9 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/tidb.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/tidb.md @@ -7,6 +7,22 @@ keywords: [开源监控系统, 开源数据库监控, TiDB数据库监控] > 使用 HTTP 和 JDBC 协议对 TiDB 的通用性能指标进行采集监控。 +### 驱动选择 + +TiDB 监控里的 HTTP 部分保持不变，SQL 查询部分现在和 MySQL 一样会自动分流： + +- 如果 `ext-lib` 中放入了 `mysql-connector-j`，JVM 采集器或主程序内置采集器会对 SQL 查询指标自动优先走 JDBC。 +- 如果没有放入 `mysql-connector-j`，HertzBeat 会对 SQL 查询指标自动切换到内置的 MySQL 兼容查询引擎，不需要额外复制 JAR。 +- 在 `ext-lib` 中新增或删除 JAR 后，请重启 HertzBeat 或独立 JVM 采集器。 + +:::important 采集器包选择 +TiDB 默认模板同时包含 HTTP 指标和 MySQL 兼容 SQL 查询。 + +- HTTP 指标集合不受 JDBC 驱动选择影响 +- 内置 SQL 查询引擎已经可以在不放 `mysql-connector-j` 的情况下采集默认 TiDB `basic` 指标集合 +- 如果你明确把 `mysql-connector-j` 放进 `ext-lib`，JVM 采集器或主程序内置采集器仍会对 SQL 查询路径优先走 JDBC +::: + ### 配置参数 | 参数名称 | 参数帮助描述 | diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/docker-deploy.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/docker-deploy.md index 06fa83e496e..2265696f10c 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/docker-deploy.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/docker-deploy.md @@ -19,6 +19,7 @@ sidebar_label: Docker方式安装 ```shell $ docker run -d -p 1157:1157 -p 1158:1158 \ + -e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto \ -v $(pwd)/data:/opt/hertzbeat/data \ -v $(pwd)/logs:/opt/hertzbeat/logs \ -v $(pwd)/application.yml:/opt/hertzbeat/config/application.yml \ @@ -35,7 +36,8 @@ sidebar_label: Docker方式安装 - `-v $(pwd)/logs:/opt/hertzbeat/logs` : (可选) 挂载日志文件到本地主机方便查看 - `-v $(pwd)/application.yml:/opt/hertzbeat/config/application.yml` : (可选) 挂载配置文件到容器中(请确保本地已有此文件)。[下载源](https://github.com/apache/hertzbeat/raw/master/script/application.yml) - `-v $(pwd)/sureness.yml:/opt/hertzbeat/config/sureness.yml` : (可选) 挂载账户配置文件到容器中(请确保本地已有此文件)。[下载源](https://github.com/apache/hertzbeat/raw/master/script/sureness.yml) - - `-v $(pwd)/ext-lib:/opt/hertzbeat/ext-lib` : (可选) 挂载外部的第三方 JAR 包 [mysql-jdbc](https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip) [oracle-jdbc](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/23.4.0.24.05/ojdbc8-23.4.0.24.05.jar) [oracle-i18n](https://repo.mavenlibs.com/maven/com/oracle/database/nls/orai18n/21.5.0.0/orai18n-21.5.0.0.jar) + - `-v $(pwd)/ext-lib:/opt/hertzbeat/ext-lib` : (可选) 在你需要运行时 JDBC 扩展时挂载外部第三方 JAR 包。`mysql-jdbc` 只在你明确希望 MySQL 兼容监控继续走 JDBC 时才需要；Oracle 监控仍然需要 [oracle-jdbc](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/23.4.0.24.05/ojdbc8-23.4.0.24.05.jar) 和 [oracle-i18n](https://repo.mavenlibs.com/maven/com/oracle/database/nls/orai18n/21.5.0.0/orai18n-21.5.0.0.jar)。 + - `-e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto` : (可选) 覆盖主程序内置采集器的 MySQL 兼容监控查询链路。可选值：`auto`、`jdbc`、`r2dbc`。 - `--name hertzbeat` : (可选) 命名容器名称为 hertzbeat - `--restart=always` : (可选) 配置容器自动重启。 - `apache/hertzbeat` : 使用[官方应用镜像](https://hub.docker.com/r/apache/hertzbeat)来启动容器, 若网络超时可用`quay.io/tancloud/hertzbeat`代替。 @@ -69,6 +71,7 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 -e MODE=public \ -e MANAGER_HOST=127.0.0.1 \ -e MANAGER_PORT=1158 \ + -e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto \ --name hertzbeat-collector apache/hertzbeat-collector ``` @@ -79,6 +82,7 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 - `-e MODE=public` : 配置运行模式(public or private), 公共集群模式或私有云边模式。 - `-e MANAGER_HOST=127.0.0.1` : 重要, 配置连接的 HertzBeat Server 地址，127.0.0.1 需替换为 HertzBeat Server 对外 IP 地址。 - `-e MANAGER_PORT=1158` : (可选) 配置连接的 HertzBeat Server 端口，默认 1158. + - `-e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto` : (可选) 覆盖 MySQL 兼容监控查询链路。可选值：`auto`、`jdbc`、`r2dbc`。 - `-v $(pwd)/logs:/opt/hertzbeat-collector/logs` : (可选)挂载日志文件到本地主机方便查看 - `--name hertzbeat-collector` : 命名容器名称为 hertzbeat-collector - `apache/hertzbeat-collector` : 使用[官方应用镜像](https://hub.docker.com/r/apache/hertzbeat-collector)来启动容器, 若网络超时可用`quay.io/tancloud/hertzbeat-collector`代替。 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md index 6ef2037be31..9399673a846 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md @@ -13,6 +13,8 @@ description: 说明 HertzBeat Native 采集器安装包适合什么场景、优 - HTTP、HTTPS、网站可用性、API 检查 - 端口可用性、Ping、SSL 证书等网络探测 +- 不依赖运行时 `ext-lib` JDBC 加载的 MySQL、MariaDB、OceanBase +- SQL 查询指标不依赖运行时 `ext-lib` JDBC 加载的 TiDB - Redis、Zookeeper、Kafka 等非 JDBC 监控类型 ## 为什么选择它？ @@ -35,10 +37,9 @@ Native 采集器并不是所有 JVM 采集器场景的无损替代。 如果你的监控依赖外部 JDBC 驱动，请继续使用 JVM 采集器安装包，尤其包括： -- MySQL，需要 `mysql-connector-j` -- OceanBase，同样依赖 MySQL JDBC 驱动 - Oracle，需要 `ojdbc8`，部分场景还需要 `orai18n` - DB2，需要 `jcc` +- 任何明确把 `mysql-connector-j` 放进 `ext-lib` 并希望继续走 JDBC 的 MySQL、MariaDB、OceanBase 场景 ## 安装包命名规则 @@ -69,8 +70,9 @@ Native 采集器安装包和 JVM 采集器安装包使用同一套 `config/appli ## 推荐选择 -- 想要更低内存、更快启动，并且监控类型不依赖 JDBC 驱动时，优先选择 Native 采集器安装包。 +- 想要更低内存、更快启动，并且监控类型不依赖 JDBC 驱动时，优先选择 Native 采集器安装包；MySQL、MariaDB、OceanBase 在不使用 `ext-lib` 时适合直接选择 Native 采集器安装包，TiDB 的 SQL 查询指标在不使用 `ext-lib` 时也可以走内置 MySQL 兼容查询引擎。 - 需要 `ext-lib`、外置 JDBC 驱动，或者依赖 JVM 风格运行时扩展能力时，使用 JVM 采集器安装包。 +- 对 MySQL 兼容监控来说，`auto` 只检查 `ext-lib`。如果你想手动指定链路，可以配置 `hertzbeat.collector.mysql.query-engine=jdbc`、`r2dbc` 或 `auto`。 ## 官方多平台安装包是怎么构建的？ diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md index 3091205add9..c3d8120c564 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md @@ -64,7 +64,7 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 ::: :::tip Native 采集器推荐 -如果你的监控任务不依赖从 `ext-lib` 动态加载外部 JDBC 驱动，优先选择 Native 采集器安装包，通常启动更快、常驻内存更低。 +如果你的监控任务不依赖从 `ext-lib` 动态加载外部 JDBC 驱动，优先选择 Native 采集器安装包，通常启动更快、常驻内存更低。MySQL、MariaDB、OceanBase 在没有提供 `mysql-connector-j` 时，也可以直接使用 Native 采集器安装包；TiDB 的 SQL 查询指标也遵循同样规则。 在选择前，建议先阅读 [Native 采集器指南](native-collector) 了解它的限制和取舍。 ::: @@ -128,14 +128,13 @@ Native 采集器适合不依赖外部 JVM classpath 扩展的监控类型。 因此，凡是依赖外置 JDBC 驱动的监控类型，请使用 JVM 采集器，不要使用 Native 采集器。当前至少包括： -- MySQL，需要 `mysql-connector-j` -- OceanBase，同样依赖 MySQL JDBC 驱动 - Oracle，需要 `ojdbc8`，部分场景还需要 `orai18n` - DB2，需要 `jcc` +- 任何明确把 `mysql-connector-j` 放进 `ext-lib` 并希望继续走 JDBC 的 MySQL、MariaDB、OceanBase 场景 建议部署方式： -- `API`、`网站`、`端口可用性`、`Ping` 等非 JDBC 类型优先使用 Native 采集器 +- `API`、`网站`、`端口可用性`、`Ping` 等非 JDBC 类型，以及不依赖 `ext-lib` 的 MySQL、MariaDB、OceanBase，优先使用 Native 采集器 - 需要 `ext-lib` 扩展驱动时使用 JVM 采集器 ::: diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md index 0031ecebd45..d4d2b7c854c 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md @@ -63,7 +63,7 @@ HertzBeat 提供多种安装选项： 3. 部署启动 `$ ./bin/startup.sh` 或 `bin/startup.bat` 4. 浏览器访问 `http://localhost:1157` 即可开始，默认账号密码 `admin/hertzbeat` 5. 部署采集器集群(可选) - - 如果你不需要 MySQL、OceanBase、Oracle、DB2 这类依赖 `ext-lib` JDBC 驱动的监控，优先选择 Native 采集器安装包，通常启动更快、内存更省。详见 [Native 采集器指南](native-collector)。 + - 如果你不需要 `ext-lib` 外置 JDBC 驱动，优先选择 Native 采集器安装包，通常启动更快、内存更省。MySQL、MariaDB、OceanBase 在没有提供 `mysql-connector-j` 时可以直接使用内置查询引擎；TiDB 的 SQL 查询指标也遵循同样规则。详见 [Native 采集器指南](native-collector)。 - 下载您系统环境对应采集器安装包 `apache-hertzbeat-collector-xx-bin.tar.gz`（JVM 采集器）或匹配目标平台的 Native 采集器安装包，例如 `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz`、`apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`，到规划的另一台部署主机上 [Download Page](https://hertzbeat.apache.org/docs/download) - 配置采集器的配置文件 `hertzbeat-collector/config/application.yml` 里面的连接主HertzBeat服务的对外IP，端口，当前采集器名称(需保证唯一性)等参数 `identity` `mode` (public or private) `manager-host` `manager-port` @@ -80,7 +80,7 @@ HertzBeat 提供多种安装选项： ``` - Native 采集器的代价是安装包按平台区分、不支持运行时 `ext-lib` JDBC 加载，也不适合依赖 JVM 风格运行时 classpath 扩展的场景。详见 [Native 采集器指南](native-collector)。 - - 如果需要通过 `ext-lib` 加载 MySQL、OceanBase、Oracle、DB2 等外置 JDBC 驱动，请使用 JVM 采集器安装包 + - 如果在 `ext-lib` 中放入了 `mysql-connector-j`，主程序内置采集器或 JVM 采集器会在重启后自动优先走 JDBC；这一点现在适用于 MySQL、MariaDB、OceanBase，TiDB 的 SQL 查询指标也遵循同样规则，而它的 HTTP 指标不受影响。Oracle、DB2 仍然必须使用 JVM 采集器安装包，因为它们依赖外置 JDBC 驱动 - JVM 采集器安装包使用 `$ ./bin/startup.sh` 或 `bin/startup.bat` 启动。Linux 或 macOS 的 Native 采集器安装包使用 `$ ./bin/startup.sh` 启动，Windows 的 Native 采集器安装包使用 `bin\\startup.bat` 启动 - 浏览器访问主 HertzBeat 服务 `http://localhost:1157` 查看概览页面即可看到注册上来的新采集器 diff --git a/material/licenses/NOTICE b/material/licenses/NOTICE index 66e5d1db5f7..3e8e5d05a62 100644 --- a/material/licenses/NOTICE +++ b/material/licenses/NOTICE @@ -1162,6 +1162,22 @@ Copyright 2001-2024 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (https://www.apache.org/). ======================================================================== +Reactive Relational Database Connectivity + +Copyright 2017-2022 the original author or authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +======================================================================== ======================================================================== Apache ECharts diff --git a/material/licenses/backend/LICENSE b/material/licenses/backend/LICENSE index be7c0505ed8..3440fc8c31d 100644 --- a/material/licenses/backend/LICENSE +++ b/material/licenses/backend/LICENSE @@ -265,6 +265,7 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/io.jsonwebtoken/jjwt-api/0.11.2 Apache-2.0 https://mvnrepository.com/artifact/io.jsonwebtoken/jjwt-impl/0.11.2 Apache-2.0 https://mvnrepository.com/artifact/io.jsonwebtoken/jjwt-jackson/0.11.2 Apache-2.0 + https://mvnrepository.com/artifact/io.asyncer/r2dbc-mysql/1.4.1 Apache-2.0 https://mvnrepository.com/artifact/io.lettuce/lettuce-core/6.3.1.RELEASE Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-commons/1.12.3 Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-core/1.12.3 Apache-2.0 @@ -305,6 +306,8 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/io.netty/netty-transport-udt/4.1.100.Final Apache-2.0 https://mvnrepository.com/artifact/io.perfmark/perfmark-api/0.26.0 Apache-2.0 https://mvnrepository.com/artifact/io.projectreactor/reactor-core/3.6.3 Apache-2.0 + https://mvnrepository.com/artifact/io.projectreactor.netty/reactor-netty-core/1.3.3 Apache-2.0 + https://mvnrepository.com/artifact/io.r2dbc/r2dbc-spi/1.0.0.RELEASE Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient/0.16.0 Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient_tracer_common/0.16.0 Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient_tracer_otel/0.16.0 Apache-2.0 diff --git a/material/licenses/collector/LICENSE b/material/licenses/collector/LICENSE index b141b35c26a..ea272bc1567 100644 --- a/material/licenses/collector/LICENSE +++ b/material/licenses/collector/LICENSE @@ -222,6 +222,7 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/commons-lang/commons-lang/2.6 Apache-2.0 https://mvnrepository.com/artifact/commons-net/commons-net/3.10.0 Apache-2.0 https://mvnrepository.com/artifact/commons-validator/commons-validator/1.7 Apache-2.0 + https://mvnrepository.com/artifact/io.asyncer/r2dbc-mysql/1.4.1 Apache-2.0 https://mvnrepository.com/artifact/io.lettuce/lettuce-core/6.3.1.RELEASE Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-commons/1.12.3 Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-observation/1.12.3 Apache-2.0 @@ -259,6 +260,8 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/io.netty/netty-transport-sctp/4.1.100.Final Apache-2.0 https://mvnrepository.com/artifact/io.netty/netty-transport-udt/4.1.100.Final Apache-2.0 https://mvnrepository.com/artifact/io.projectreactor/reactor-core/3.6.3 Apache-2.0 + https://mvnrepository.com/artifact/io.projectreactor.netty/reactor-netty-core/1.3.3 Apache-2.0 + https://mvnrepository.com/artifact/io.r2dbc/r2dbc-spi/1.0.0.RELEASE Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient/0.16.0 Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient_tracer_common/0.16.0 Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient_tracer_otel/0.16.0 Apache-2.0 diff --git a/material/licenses/collector/NOTICE b/material/licenses/collector/NOTICE index c56b217a965..1d686114d90 100644 --- a/material/licenses/collector/NOTICE +++ b/material/licenses/collector/NOTICE @@ -746,3 +746,19 @@ Copyright 2001-2024 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (https://www.apache.org/). ======================================================================== +Reactive Relational Database Connectivity + +Copyright 2017-2022 the original author or authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +======================================================================== diff --git a/pom.xml b/pom.xml index 341a12c9066..a10511c3a1a 100644 --- a/pom.xml +++ b/pom.xml @@ -134,6 +134,7 @@ 3.7.1 4.1.117.Final 8.4.0 + 1.4.1 12.10.2.jre11 21.5.0.0 4.6.1 @@ -183,6 +184,7 @@ 2.25.0 2.25.0-alpha 1.3.1-alpha + 2.0.3 @@ -625,6 +627,28 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + 3.5.0 + + + require-java-25 + validate + + enforce + + + + + [25,) + Apache HertzBeat now requires JDK 25 or newer. Please switch JAVA_HOME before building or testing. + + + + + + org.apache.maven.plugins diff --git a/script/application.yml b/script/application.yml index ecebc66681f..90c0ea4bb8e 100644 --- a/script/application.yml +++ b/script/application.yml @@ -336,6 +336,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/script/docker-compose/README.md b/script/docker-compose/README.md index 99908b2cc4c..3a7b931bb9d 100644 --- a/script/docker-compose/README.md +++ b/script/docker-compose/README.md @@ -2,9 +2,14 @@ Suggest the [HertzBeat + GreptimeDB + Postgresql Solution](hertzbeat-postgresql-greptimedb) for the best performance and stability. +Notes: + +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If you place `mysql-connector-j` in `ext-lib`, HertzBeat prefers JDBC after restart. +- Oracle and DB2 still require external JDBC driver jars in `ext-lib`. + - Use Postgresql + GreptimeDB as Hertzbeat dependent storage -> [HertzBeat+PostgreSQL+GreptimeDB Solution](hertzbeat-postgresql-greptimedb) - Use Postgresql + VictoriaMetrics as Hertzbeat dependent storage -> [HertzBeat+PostgreSQL+VictoriaMetrics Solution](hertzbeat-postgresql-victoria-metrics) - Use Mysql + VictoriaMetrics as Hertzbeat dependent storage -> [HertzBeat+Mysql+VictoriaMetrics Solution](hertzbeat-mysql-victoria-metrics) - Use Mysql + IoTDB as Hertzbeat dependent storage -> [HertzBeat+Mysql+IoTDB Solution](hertzbeat-mysql-iotdb) - Use Mysql + Tdengine as Hertzbeat dependent storage -> [HertzBeat+Mysql+Tdengine Solution](hertzbeat-mysql-tdengine) - diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/README.md b/script/docker-compose/hertzbeat-mysql-iotdb/README.md index b5b94efcec2..b6c8fc30504 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/README.md +++ b/script/docker-compose/hertzbeat-mysql-iotdb/README.md @@ -17,10 +17,11 @@ 1. Download the hertzbeat-docker-compose installation deployment script file The script file is located in `script/docker-compose/hertzbeat-mysql-iotdb` link [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/ hertzbeat-mysql-iotdb) -2. Add MYSQL jdbc driver jar +2. Optional: add external JDBC driver jars to `ext-lib` - Download the MYSQL jdbc driver jar package, such as mysql-connector-java-8.0.25.jar. https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip - Copy the jar package to the ext-lib directory. + MySQL-compatible monitoring can use the built-in query engine directly, so `mysql-connector-j` is optional. + If you want HertzBeat to prefer JDBC after restart, place `mysql-connector-j` in `ext-lib`. + Oracle and DB2 still require external JDBC jars in `ext-lib`. 3. Enter the deployment script docker-compose directory, execute diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/README_CN.md b/script/docker-compose/hertzbeat-mysql-iotdb/README_CN.md index 9f145fe769b..e310d72ce40 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/README_CN.md +++ b/script/docker-compose/hertzbeat-mysql-iotdb/README_CN.md @@ -19,9 +19,10 @@ 1. 下载hertzbeat-docker-compose安装部署脚本文件 脚本文件位于代码仓库下`script/docker-compose/hertzbeat-mysql-iotdb` 链接 [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-mysql-iotdb) -2. 添加 MYSQL jdbc 驱动 jar - 下载 MYSQL jdbc driver jar, 例如 mysql-connector-java-8.0.25.jar. https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip - 将此 jar 包拷贝放入 ext-lib 目录下. +2. 可选：向 `ext-lib` 添加外部 JDBC 驱动 jar + MySQL 兼容监控现在可以直接使用内置查询引擎，所以 `mysql-connector-j` 不是必需项。 + 如果你希望 HertzBeat 在重启后优先走 JDBC，可以把 `mysql-connector-j` 放到 `ext-lib`。 + Oracle、DB2 这类场景仍然需要把外部 JDBC 驱动放到 `ext-lib`。 3. 进入部署脚本 docker-compose 目录, 执行 diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml index c4c9d80797e..33b0537468e 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml @@ -236,6 +236,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/docker-compose.yaml b/script/docker-compose/hertzbeat-mysql-iotdb/docker-compose.yaml index 9a1af1589e8..472cca7f2c8 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-mysql-iotdb/docker-compose.yaml @@ -68,6 +68,7 @@ services: hostname: hertzbeat restart: always environment: + HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE: auto TZ: Asia/Shanghai LANG: zh_CN.UTF-8 depends_on: diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/ext-lib/README b/script/docker-compose/hertzbeat-mysql-iotdb/ext-lib/README index 5898fde6b91..7e270434ca3 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/ext-lib/README +++ b/script/docker-compose/hertzbeat-mysql-iotdb/ext-lib/README @@ -13,9 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder like: +Please move external libs to this folder only when you need external JDBC jars, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar mysql-connector-java-8.0.30.jar +Notes: + +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If `mysql-connector-j` is present here, HertzBeat prefers JDBC after restart. +- Oracle and DB2 still require external JDBC jars in `ext-lib`. diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/README.md b/script/docker-compose/hertzbeat-mysql-tdengine/README.md index 27c1febaba5..0b7d988438b 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/README.md +++ b/script/docker-compose/hertzbeat-mysql-tdengine/README.md @@ -17,10 +17,11 @@ 1. Download the hertzbeat-docker-compose installation deployment script file The script file is located in `script/docker-compose/hertzbeat-mysql-tdengine` link [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-mysql-tdengine) -2. Add MYSQL jdbc driver jar +2. Optional: add external JDBC driver jars to `ext-lib` - Download the MYSQL jdbc driver jar package, such as mysql-connector-java-8.0.25.jar. https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip - Copy the jar package to the ext-lib directory. + MySQL-compatible monitoring can use the built-in query engine directly, so `mysql-connector-j` is optional. + If you want HertzBeat to prefer JDBC after restart, place `mysql-connector-j` in `ext-lib`. + Oracle and DB2 still require external JDBC jars in `ext-lib`. 3. Enter the deployment script docker-compose directory, execute diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/README_CN.md b/script/docker-compose/hertzbeat-mysql-tdengine/README_CN.md index 329344585d3..ad2d2758dad 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/README_CN.md +++ b/script/docker-compose/hertzbeat-mysql-tdengine/README_CN.md @@ -19,9 +19,10 @@ 1. 下载hertzbeat-docker-compose安装部署脚本文件 脚本文件位于代码仓库下`script/docker-compose/hertzbeat-mysql-tdengine` 链接 [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-mysql-tdengine) -2. 添加 MYSQL jdbc 驱动 jar - 下载 MYSQL jdbc driver jar, 例如 mysql-connector-java-8.0.25.jar. https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip - 将此 jar 包拷贝放入 ext-lib 目录下. +2. 可选：向 `ext-lib` 添加外部 JDBC 驱动 jar + MySQL 兼容监控现在可以直接使用内置查询引擎，所以 `mysql-connector-j` 不是必需项。 + 如果你希望 HertzBeat 在重启后优先走 JDBC，可以把 `mysql-connector-j` 放到 `ext-lib`。 + Oracle、DB2 这类场景仍然需要把外部 JDBC 驱动放到 `ext-lib`。 3. 进入部署脚本 docker-compose 目录, 执行 diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml index 8a6dfe1528f..8d795ed6434 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml @@ -233,6 +233,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/docker-compose.yaml b/script/docker-compose/hertzbeat-mysql-tdengine/docker-compose.yaml index 33afa63dadd..83e151bb550 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-mysql-tdengine/docker-compose.yaml @@ -67,6 +67,7 @@ services: hostname: hertzbeat restart: always environment: + HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE: auto TZ: Asia/Shanghai LANG: zh_CN.UTF-8 depends_on: diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/ext-lib/README b/script/docker-compose/hertzbeat-mysql-tdengine/ext-lib/README index 5898fde6b91..7e270434ca3 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/ext-lib/README +++ b/script/docker-compose/hertzbeat-mysql-tdengine/ext-lib/README @@ -13,9 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder like: +Please move external libs to this folder only when you need external JDBC jars, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar mysql-connector-java-8.0.30.jar +Notes: + +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If `mysql-connector-j` is present here, HertzBeat prefers JDBC after restart. +- Oracle and DB2 still require external JDBC jars in `ext-lib`. diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/README.md b/script/docker-compose/hertzbeat-mysql-victoria-metrics/README.md index 02a6829ea8f..8d5a2c52410 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/README.md +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/README.md @@ -17,10 +17,11 @@ 1. Download the hertzbeat-docker-compose installation deployment script file The script file is located in `script/docker-compose/hertzbeat-mysql-victoria-metrics` link [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-mysql-victoria-metrics) -2. Add MYSQL jdbc driver jar +2. Optional: add external JDBC driver jars to `ext-lib` - Download the MYSQL jdbc driver jar package, such as mysql-connector-java-8.0.25.jar. https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip - Copy the jar package to the ext-lib directory. + MySQL-compatible monitoring can use the built-in query engine directly, so `mysql-connector-j` is optional. + If you want HertzBeat to prefer JDBC after restart, place `mysql-connector-j` in `ext-lib`. + Oracle and DB2 still require external JDBC jars in `ext-lib`. 3. Enter the deployment script docker-compose directory, execute diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/README_CN.md b/script/docker-compose/hertzbeat-mysql-victoria-metrics/README_CN.md index 6602b34a318..30bf8f11e96 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/README_CN.md +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/README_CN.md @@ -19,9 +19,10 @@ 1. 下载hertzbeat-docker-compose安装部署脚本文件 脚本文件位于代码仓库下`script/docker-compose/hertzbeat-mysql-victoria-metrics` 链接 [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-mysql-victoria-metrics) -2. 添加 MYSQL jdbc 驱动 jar - 下载 MYSQL jdbc driver jar, 例如 mysql-connector-java-8.0.25.jar. https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip - 将此 jar 包拷贝放入 ext-lib 目录下. +2. 可选：向 `ext-lib` 添加外部 JDBC 驱动 jar + MySQL 兼容监控现在可以直接使用内置查询引擎，所以 `mysql-connector-j` 不是必需项。 + 如果你希望 HertzBeat 在重启后优先走 JDBC，可以把 `mysql-connector-j` 放到 `ext-lib`。 + Oracle、DB2 这类场景仍然需要把外部 JDBC 驱动放到 `ext-lib`。 3. 进入部署脚本 docker-compose 目录, 执行 diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml index d9118c6c560..f92abb9f989 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml @@ -236,6 +236,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/docker-compose.yaml b/script/docker-compose/hertzbeat-mysql-victoria-metrics/docker-compose.yaml index da4188bd442..acebf9889b6 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/docker-compose.yaml @@ -67,6 +67,7 @@ services: hostname: hertzbeat restart: always environment: + HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE: auto TZ: Asia/Shanghai LANG: zh_CN.UTF-8 depends_on: diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/ext-lib/README b/script/docker-compose/hertzbeat-mysql-victoria-metrics/ext-lib/README index 5898fde6b91..7e270434ca3 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/ext-lib/README +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/ext-lib/README @@ -13,9 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder like: +Please move external libs to this folder only when you need external JDBC jars, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar mysql-connector-java-8.0.30.jar +Notes: + +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If `mysql-connector-j` is present here, HertzBeat prefers JDBC after restart. +- Oracle and DB2 still require external JDBC jars in `ext-lib`. diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/README.md b/script/docker-compose/hertzbeat-postgresql-greptimedb/README.md index 5a84321adcb..8c2dd54d1e2 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/README.md +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/README.md @@ -18,7 +18,13 @@ The script file is located in `script/docker-compose/hertzbeat-postgresql-greptimedb` link [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-postgresql-greptimedb) -2. Enter the deployment script docker-compose directory, execute +2. Optional: add external JDBC driver jars to `ext-lib` + + MySQL-compatible monitoring can use the built-in query engine directly, so `mysql-connector-j` is optional. + If you want HertzBeat to prefer JDBC after restart, place `mysql-connector-j` in `ext-lib`. + Oracle and DB2 still require external JDBC jars in `ext-lib`. + +3. Enter the deployment script docker-compose directory, execute `docker compose up -d` diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/README_CN.md b/script/docker-compose/hertzbeat-postgresql-greptimedb/README_CN.md index 609ba130f64..2b375c99d2c 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/README_CN.md +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/README_CN.md @@ -20,7 +20,12 @@ 脚本文件位于代码仓库下`script/docker-compose/hertzbeat-postgresql-greptimedb` 链接 [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-postgresql-greptimedb) -2. 进入部署脚本 docker-compose 目录, 执行 +2. 可选：向 `ext-lib` 添加外部 JDBC 驱动 jar + MySQL 兼容监控现在可以直接使用内置查询引擎，所以 `mysql-connector-j` 不是必需项。 + 如果你希望 HertzBeat 在重启后优先走 JDBC，可以把 `mysql-connector-j` 放到 `ext-lib`。 + Oracle、DB2 这类场景仍然需要把外部 JDBC 驱动放到 `ext-lib`。 + +3. 进入部署脚本 docker-compose 目录, 执行 `docker compose up -d` diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml index 0a1feb682dc..6cc14900303 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml @@ -233,6 +233,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/docker-compose.yaml b/script/docker-compose/hertzbeat-postgresql-greptimedb/docker-compose.yaml index be1603c409d..2c3782607ba 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/docker-compose.yaml @@ -87,6 +87,7 @@ services: hostname: hertzbeat restart: always environment: + HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE: auto TZ: Asia/Shanghai LANG: zh_CN.UTF-8 depends_on: diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/ext-lib/README b/script/docker-compose/hertzbeat-postgresql-greptimedb/ext-lib/README index 5898fde6b91..7e270434ca3 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/ext-lib/README +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/ext-lib/README @@ -13,9 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder like: +Please move external libs to this folder only when you need external JDBC jars, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar mysql-connector-java-8.0.30.jar +Notes: + +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If `mysql-connector-j` is present here, HertzBeat prefers JDBC after restart. +- Oracle and DB2 still require external JDBC jars in `ext-lib`. diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README.md b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README.md index ff4edb7525e..d791a30dd13 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README.md +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README.md @@ -18,7 +18,13 @@ The script file is located in `script/docker-compose/hertzbeat-postgresql-victoria-metrics` link [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-postgresql-victoria-metrics) -2. Enter the deployment script docker-compose directory, execute +2. Optional: add external JDBC driver jars to `ext-lib` + + MySQL-compatible monitoring can use the built-in query engine directly, so `mysql-connector-j` is optional. + If you want HertzBeat to prefer JDBC after restart, place `mysql-connector-j` in `ext-lib`. + Oracle and DB2 still require external JDBC jars in `ext-lib`. + +3. Enter the deployment script docker-compose directory, execute `docker compose up -d` diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README_CN.md b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README_CN.md index f809587e05d..d6982c63092 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README_CN.md +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README_CN.md @@ -20,7 +20,12 @@ 脚本文件位于代码仓库下`script/docker-compose/hertzbeat-postgre-victoria-metrics` 链接 [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-postgresql-victoria-metrics) -2. 进入部署脚本 docker-compose 目录, 执行 +2. 可选：向 `ext-lib` 添加外部 JDBC 驱动 jar + MySQL 兼容监控现在可以直接使用内置查询引擎，所以 `mysql-connector-j` 不是必需项。 + 如果你希望 HertzBeat 在重启后优先走 JDBC，可以把 `mysql-connector-j` 放到 `ext-lib`。 + Oracle、DB2 这类场景仍然需要把外部 JDBC 驱动放到 `ext-lib`。 + +3. 进入部署脚本 docker-compose 目录, 执行 `docker compose up -d` diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml index ed4c59bcbb4..3108f5b9944 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml @@ -235,6 +235,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml index fa28d86ea79..fda39244719 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml @@ -70,6 +70,7 @@ services: hostname: hertzbeat restart: always environment: + HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE: auto TZ: Asia/Shanghai LANG: zh_CN.UTF-8 depends_on: diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/ext-lib/README b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/ext-lib/README index 5898fde6b91..7e270434ca3 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/ext-lib/README +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/ext-lib/README @@ -13,9 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder like: +Please move external libs to this folder only when you need external JDBC jars, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar mysql-connector-java-8.0.30.jar +Notes: + +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If `mysql-connector-j` is present here, HertzBeat prefers JDBC after restart. +- Oracle and DB2 still require external JDBC jars in `ext-lib`. diff --git a/script/ext-lib/README b/script/ext-lib/README index 0afb3b3bbec..30800cd2328 100644 --- a/script/ext-lib/README +++ b/script/ext-lib/README @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder for JVM-based server / collector packages, for example: +Please move external libs to this folder only when you need JVM runtime extension, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar @@ -23,5 +23,7 @@ jcc-11.5.9.0.jar Note: - `ext-lib` is loaded by the JVM server package and the JVM collector package. +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If `mysql-connector-j` is present here, the JVM server package or JVM collector package prefers JDBC after restart. - The native collector package does not support loading external JDBC driver jars from `ext-lib` at runtime. -- If you need MySQL, OceanBase, Oracle, or DB2 monitoring with external JDBC drivers, use the JVM collector package. +- If you need Oracle or DB2 monitoring, or you explicitly want the JDBC path for MySQL-compatible monitoring, use the JVM collector package. From f11d58fd7678d167d144f70756743de00f63a563 Mon Sep 17 00:00:00 2001 From: Logic Date: Sat, 14 Mar 2026 11:47:10 +0800 Subject: [PATCH 2/4] feat: add MySQL R2DBC query engine support and update documentation --- .../nativex/NativeCollectorDefaults.java | 20 +++++++++---------- .../src/main/resources/application.yml | 3 --- .../nativex/NativeCollectorDefaultsTest.java | 12 +++++++++-- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java index 5f97c36ecbf..1926c48cfb8 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java @@ -22,16 +22,18 @@ import org.springframework.core.NativeDetector; /** - * Applies the native-only collector defaults without forking {@code application.yml}. + * Applies collector defaults without forking {@code application.yml}. */ public final class NativeCollectorDefaults { static final String AUTOCONFIGURE_EXCLUDE_PROPERTY = "spring.autoconfigure.exclude"; - static final String NATIVE_AUTOCONFIGURE_EXCLUDES = String.join(",", + static final String JVM_AUTOCONFIGURE_EXCLUDES = String.join(",", "org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration", "org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration", "org.springframework.boot.jdbc.autoconfigure.DataSourceAutoConfiguration", - "org.springframework.boot.hibernate.autoconfigure.HibernateJpaAutoConfiguration", + "org.springframework.boot.hibernate.autoconfigure.HibernateJpaAutoConfiguration"); + static final String NATIVE_AUTOCONFIGURE_EXCLUDES = String.join(",", + JVM_AUTOCONFIGURE_EXCLUDES, "org.springframework.boot.data.jpa.autoconfigure.DataJpaRepositoriesAutoConfiguration", "org.springframework.boot.jdbc.autoconfigure.DataSourceInitializationAutoConfiguration", "org.springframework.boot.jdbc.autoconfigure.DataSourceTransactionManagerAutoConfiguration", @@ -43,16 +45,12 @@ private NativeCollectorDefaults() { } public static void applyTo(SpringApplication application) { - Map defaultProperties = defaultProperties(NativeDetector.inNativeImage()); - if (!defaultProperties.isEmpty()) { - application.setDefaultProperties(defaultProperties); - } + application.setDefaultProperties(defaultProperties(NativeDetector.inNativeImage())); } static Map defaultProperties(boolean nativeImage) { - if (!nativeImage) { - return Map.of(); - } - return Map.of(AUTOCONFIGURE_EXCLUDE_PROPERTY, NATIVE_AUTOCONFIGURE_EXCLUDES); + return Map.of( + AUTOCONFIGURE_EXCLUDE_PROPERTY, + nativeImage ? NATIVE_AUTOCONFIGURE_EXCLUDES : JVM_AUTOCONFIGURE_EXCLUDES); } } diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml index 08d98b88717..03702ef5627 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml @@ -27,9 +27,6 @@ spring: timeout-per-shutdown-phase: 10s jackson: default-property-inclusion: ALWAYS - # need to disable spring boot mongodb auto config, or default mongodb connection tried and failed... - autoconfigure: - exclude: org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration, org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration, org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration, org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration management: endpoints: web: diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java index 2ffa4beeebc..811259ecc16 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java @@ -33,7 +33,15 @@ void shouldProvideNativeAutoconfigureExcludesWhenNativeImage() { } @Test - void shouldNotProvideNativeSpecificPropertiesForJvmCollector() { - assertTrue(NativeCollectorDefaults.defaultProperties(false).isEmpty()); + void shouldProvideJvmAutoconfigureExcludesForJvmCollector() { + Map properties = NativeCollectorDefaults.defaultProperties(false); + assertEquals(NativeCollectorDefaults.JVM_AUTOCONFIGURE_EXCLUDES, + properties.get(NativeCollectorDefaults.AUTOCONFIGURE_EXCLUDE_PROPERTY)); + } + + @Test + void nativeAutoconfigureExcludesShouldExtendJvmExcludes() { + assertTrue(NativeCollectorDefaults.NATIVE_AUTOCONFIGURE_EXCLUDES + .startsWith(NativeCollectorDefaults.JVM_AUTOCONFIGURE_EXCLUDES)); } } From 0df7830f3446e7deffab4979f8e3a1a070e41db6 Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 17 Mar 2026 12:29:29 +0800 Subject: [PATCH 3/4] fix: satisfy mysql r2dbc checkstyle naming --- .../collector/mysql/r2dbc/MysqlR2dbcQueryExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutor.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutor.java index ab74d430204..d8a5e9698eb 100644 --- a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutor.java +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutor.java @@ -95,7 +95,7 @@ private void safelyClose(Connection connection) { try { Mono.from(connection.close()) .timeout(Duration.ofSeconds(1), Mono.empty()) - .onErrorResume(_ -> Mono.empty()) + .onErrorResume(error -> Mono.empty()) .block(Duration.ofSeconds(2)); } catch (Exception ignored) { // Best-effort close only. Query completion must not be turned into a failure by cleanup. From f46727dac9f920aa893312abe7078b261e3b8430 Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 17 Mar 2026 14:46:36 +0800 Subject: [PATCH 4/4] test: fix jdbc common collect platform coverage --- .../database/JdbcCommonCollectTest.java | 103 ++++++++++-------- 1 file changed, 58 insertions(+), 45 deletions(-) diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollectTest.java b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollectTest.java index 129f7e66331..88cc7adcb67 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollectTest.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollectTest.java @@ -26,10 +26,11 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.LinkedHashMap; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; /** @@ -93,37 +94,6 @@ void collect() { CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); jdbcCommonCollect.collect(builder, metrics); }); - - String[] platforms = new String[]{ - "mysql", "mariadb", - "postgresql", - "clickhouse", - "sqlserver", - "oracle", - "dm" - }; - for (String platform : platforms) { - JdbcProtocol jdbc = new JdbcProtocol(); - jdbc.setPlatform(platform); - - Metrics metrics = new Metrics(); - metrics.setJdbc(jdbc); - - CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); - jdbcCommonCollect.collect(builder, metrics); - assertNotEquals(builder.getMsg(), "Query Error: Not support database platform: " + platform); - } - // invalid platform - JdbcProtocol jdbc = new JdbcProtocol(); - jdbc.setPlatform("invalid"); - - Metrics metrics = new Metrics(); - metrics.setJdbc(jdbc); - - CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); - jdbcCommonCollect.collect(builder, metrics); - assertEquals(builder.getCode(), CollectRep.Code.FAIL); - assertEquals(builder.getMsg(), "Query Error: Not support database platform: invalid"); } @Test @@ -151,13 +121,10 @@ void testUrlPassThrough() { .database("test") .url(originalUrl) .build(); - - // Use reflection to call constructDatabaseUrl method - Method constructMethod = JdbcCommonCollect.class.getDeclaredMethod("constructDatabaseUrl", JdbcProtocol.class, String.class, String.class); - constructMethod.setAccessible(true); - String processedUrl = (String) constructMethod.invoke(jdbcCollect, jdbcProtocol, "localhost", "3306"); + + String processedUrl = constructDatabaseUrl(jdbcCollect, jdbcProtocol, "localhost", "3306"); // Verify that the processed URL is the same as the original URL - assertEquals(originalUrl, processedUrl, + assertEquals(originalUrl, processedUrl, "URL should be passed through without modification: " + originalUrl); } catch (Exception e) { System.out.println("URL rejected by security validation: " + originalUrl + ", reason: " + e.getMessage()); @@ -165,6 +132,39 @@ void testUrlPassThrough() { } } + @Test + void testConstructDatabaseUrlByPlatform() throws Exception { + Map expectedUrls = new LinkedHashMap<>(); + expectedUrls.put("mysql", "jdbc:mysql://localhost:3306/test?useUnicode=true&characterEncoding=utf-8&useSSL=false"); + expectedUrls.put("mariadb", "jdbc:mysql://localhost:3306/test?useUnicode=true&characterEncoding=utf-8&useSSL=false"); + expectedUrls.put("postgresql", "jdbc:postgresql://localhost:3306/test"); + expectedUrls.put("clickhouse", "jdbc:clickhouse://localhost:3306/test"); + expectedUrls.put("sqlserver", "jdbc:sqlserver://localhost:3306;DatabaseName=test;trustServerCertificate=true;"); + expectedUrls.put("oracle", "jdbc:oracle:thin:@localhost:3306/test"); + expectedUrls.put("dm", "jdbc:dm://localhost:3306"); + + for (Map.Entry entry : expectedUrls.entrySet()) { + JdbcProtocol jdbcProtocol = JdbcProtocol.builder() + .platform(entry.getKey()) + .database("test") + .build(); + + assertEquals(entry.getValue(), constructDatabaseUrl(jdbcCommonCollect, jdbcProtocol, "localhost", "3306")); + } + } + + @Test + void testConstructDatabaseUrlRejectsUnsupportedPlatform() { + JdbcProtocol jdbcProtocol = JdbcProtocol.builder() + .platform("invalid") + .database("test") + .build(); + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> constructDatabaseUrl(jdbcCommonCollect, jdbcProtocol, "localhost", "3306")); + assertEquals("Not support database platform: invalid", exception.getMessage()); + } + @Test void testConstructDatabaseUrlSecurityInterception() { JdbcCommonCollect jdbcCollect = new JdbcCommonCollect(); @@ -197,15 +197,28 @@ void testConstructDatabaseUrlSecurityInterception() { .build(); assertThrows(Exception.class, () -> { - try { - Method constructMethod = JdbcCommonCollect.class.getDeclaredMethod("constructDatabaseUrl", JdbcProtocol.class, String.class, String.class); - constructMethod.setAccessible(true); - constructMethod.invoke(jdbcCollect, jdbcProtocol, "localhost", "3306"); - } catch (InvocationTargetException e) { - throw e.getCause(); - } + constructDatabaseUrl(jdbcCollect, jdbcProtocol, "localhost", "3306"); }, "Malicious URL should be blocked: " + maliciousUrl); } } + private String constructDatabaseUrl(JdbcCommonCollect jdbcCollect, JdbcProtocol jdbcProtocol, + String host, String port) throws Exception { + try { + Method constructMethod = JdbcCommonCollect.class + .getDeclaredMethod("constructDatabaseUrl", JdbcProtocol.class, String.class, String.class); + constructMethod.setAccessible(true); + return (String) constructMethod.invoke(jdbcCollect, jdbcProtocol, host, port); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof Exception exception) { + throw exception; + } + if (cause instanceof Error error) { + throw error; + } + throw new RuntimeException(cause); + } + } + }