diff --git a/README.md b/README.md index 664f009ad..2e5be82ea 100644 --- a/README.md +++ b/README.md @@ -474,3 +474,6 @@ Then re-run the start script: ## Smart Contract ## If you want to use smart contracts, please go to: https://blog.resilientdb.com/2025/02/14/GettingStartedSmartContract.html + + +# New Change \ No newline at end of file diff --git a/WORKSPACE b/WORKSPACE index b5071813e..134386e2a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -177,6 +177,9 @@ http_archive( sha256 = "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9", strip_prefix = "zlib-1.2.12", urls = [ + "https://zlib.net/fossils/zlib-1.2.12.tar.gz", + "https://downloads.sourceforge.net/project/libpng/zlib/1.2.12/zlib-1.2.12.tar.gz", + "https://zlib.net/fossils/zlib-1.2.12.tar.gz", "https://storage.googleapis.com/bazel-mirror/zlib.net/zlib-1.2.12.tar.gz", ], ) @@ -188,6 +191,19 @@ http_archive( strip_prefix = "leveldb-1.23", url = "https://github.com/google/leveldb/archive/refs/tags/1.23.zip", ) +http_archive( + name = "duckdb", + urls = [ + # pick one version and stick to it + "https://github.com/duckdb/duckdb/releases/download/v1.4.0/libduckdb-src.zip", + ], + # This zip is just flat: duckdb.cpp, duckdb.hpp, duckdb.h at the root. + # So no strip_prefix is needed. + build_file = "//third_party:duckdb.BUILD", + # Optional: you can add sha256 once Bazel prints it for you. + # sha256 = "", +) + bind( name = "snappy", diff --git a/chain/storage/BUILD b/chain/storage/BUILD index b2e0dfb33..23dbd47ff 100644 --- a/chain/storage/BUILD +++ b/chain/storage/BUILD @@ -81,3 +81,49 @@ cc_test( "//platform/statistic:stats", ], ) + +cc_library( + name = "duckdb_storage", + srcs = ["duckdb.cpp"], + hdrs = ["duckdb.h"], + deps = [ + ":storage", + "//chain/storage/proto:duckdb_config_cc_proto", + "//third_party:duckdb", + "//common:glog", + ], +) + +cc_binary( + name = "duckdb_test", + srcs = ["duckdb_test.cpp"], + deps = [ + ":duckdb_storage", + "//third_party:duckdb", + "//common:glog", + "//chain/storage/proto:duckdb_config_cc_proto", + ], +) + +cc_library( + name = "duckdb_storage", + srcs = ["duckdb.cpp"], + hdrs = ["duckdb.h"], + deps = [ + ":storage", + "//chain/storage/proto:duckdb_config_cc_proto", + "//third_party:duckdb", + "//common:glog", + ], +) + +cc_binary( + name = "duckdb_test", + srcs = ["duckdb_test.cpp"], + deps = [ + ":duckdb_storage", + "//third_party:duckdb", + "//common:glog", + "//chain/storage/proto:duckdb_config_cc_proto", + ], +) \ No newline at end of file diff --git a/chain/storage/duckdb.cpp b/chain/storage/duckdb.cpp new file mode 100644 index 000000000..1b4366c82 --- /dev/null +++ b/chain/storage/duckdb.cpp @@ -0,0 +1,99 @@ +#include "chain/storage/duckdb.h" + +#include + +#include +#include +#include + +#include "duckdb.hpp" + +namespace resdb { +namespace storage { + +std::unique_ptr NewResQL(const std::string& path, + const DuckDBInfo& config) { + DuckDBInfo cfg = config; + cfg.set_path(path); + + return std::make_unique(cfg); +} + +ResQL::ResQL(const DuckDBInfo& config) : config_(config) { + std::string path = "/tmp/resql-duckdb"; + if (!config.path().empty()) { + path = config.path(); + } + CreateDB(config); +} + +ResQL::~ResQL() = default; + +void ResQL::CreateDB(const DuckDBInfo& config) { + std::string db_path = config.path(); + + duckdb::DBConfig db_config; + + if (config.has_max_memory()) { + db_config.SetOption("max_memory", duckdb::Value(config.max_memory())); + } + + LOG(INFO) << "Initializing DuckDB at " + << (db_path.empty() ? "in-memory" : db_path) + << " (autoload/autoinstall extensions enabled)"; + + if (db_path.empty()) { + db_ = std::make_unique(nullptr, &db_config); + } else { + db_ = std::make_unique(db_path, &db_config); + } + + // Ensure extension auto-install/load is enabled at the connection level too. + try { + duckdb::Connection init_conn(*db_); + init_conn.Query("SET autoload_known_extensions=1"); + init_conn.Query("SET autoinstall_known_extensions=1"); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to set DuckDB extension auto-load/install flags: " + << e.what(); + } +} + +std::string ResQL::ExecuteSQL(const std::string& sql_string){ + if (sql_string.empty()) { + return "Error: empty SQL query"; + } + + if (!db_) { + LOG(ERROR) << "DuckDB is not initialized"; + return "Error: database not initialized"; + } + + try { + LOG(INFO) << "Executing SQL: " << sql_string; + conn_ = std::make_unique (*db_); + auto result = conn_->Query(sql_string); + + if (!result) { + LOG(ERROR) << "Query returned nullptr"; + return "Error: query returned no result"; + } + + if (result->HasError()) { + LOG(ERROR) << "SQL Error: " << result->GetError(); + return "Error: " + result->GetError(); + } + + std::string response = result->ToString(); + LOG(INFO) << "SQL succeeded. Rows: " << result->RowCount() + << " Cols: " << result->ColumnCount(); + LOG(INFO) << "SQL Result:\n" << response; + return response; + } catch (const std::exception& e) { + LOG(ERROR) << "SQL execution threw: " << e.what(); + return std::string("Error: ") + e.what(); + } +} + +} // namespace storage +} // namespace resdb diff --git a/chain/storage/duckdb.h b/chain/storage/duckdb.h new file mode 100644 index 000000000..77187dc25 --- /dev/null +++ b/chain/storage/duckdb.h @@ -0,0 +1,70 @@ +#pragma once + +#include +#include +#include +#include +#include + + +#include "chain/storage/proto/duckdb_config.pb.h" + +#include "storage.h" +#include "duckdb.hpp" + +namespace resdb { +namespace storage { + +class ResQL : public Storage { + public: + explicit ResQL(const DuckDBInfo& config); + ~ResQL() override; + + // Main functionality + std::string ExecuteSQL(const std::string& sql_string) override; + void CreateDB(const DuckDBInfo& config); + + // No-op overrides + int SetValue(const std::string&, const std::string&) override { return 0; } + std::string GetValue(const std::string&) override { return ""; } + std::string GetAllValues() override { return ""; } + std::string GetRange(const std::string&, const std::string&) override { + return ""; + } + int SetValueWithVersion(const std::string&, const std::string&, int) override { + return 0; + } + std::pair GetValueWithVersion(const std::string&, + int) override { + return {"", 0}; + } + std::map> GetAllItems() override { + return {}; + } + std::map> GetKeyRange( + const std::string&, const std::string&) override { + return {}; + } + std::vector> GetHistory(const std::string&, int, + int) override { + return {}; + } + std::vector> GetTopHistory(const std::string&, + int) override { + return {}; + } + bool Flush() override { return true; } + + private: + std::unique_ptr db_; + std::unique_ptr conn_; + std::optional config_; +}; + + +// Factory function +std::unique_ptr NewResQL( + const std::string& path, + const DuckDBInfo& config = DuckDBInfo()); +} // namespace storage +} // namespace resdb diff --git a/chain/storage/duckdb_test.cpp b/chain/storage/duckdb_test.cpp new file mode 100644 index 000000000..3a571aca1 --- /dev/null +++ b/chain/storage/duckdb_test.cpp @@ -0,0 +1,53 @@ +#include "chain/storage/duckdb.h" +#include "chain/storage/proto/duckdb_config.pb.h" + +#include +#include +#include +#include + +int main() { + using namespace resdb::storage; + + // 1. Create a DuckDBInfo proto for file-backed DB + DuckDBInfo config; + config.set_path("/home/aryan/resilientdb-resql/dtest/dtest.db"); + + // 2. Construct ResQL instance + std::unique_ptr my_db; + try { + my_db = std::make_unique(config); + std::cout << "ResQL (DuckDB) created successfully!" << std::endl; + } catch (std::exception& e) { + std::cerr << "Failed to create ResQL: " << e.what() << std::endl; + return 1; + } + + { + std::string create_sql = + "CREATE TABLE IF NOT EXISTS users (" + "id INTEGER, " + "name TEXT" + ");"; + + std::string resp = my_db->ExecuteSQL(create_sql); + std::cout << "CREATE: " << resp << std::endl; + } + + { + std::string insert_sql = + "INSERT INTO users VALUES (1, 'batman');"; + + std::string resp = my_db->ExecuteSQL(insert_sql); + std::cout << "INSERT: " << resp << std::endl; + } + + { + std::string select_sql = "SELECT * FROM users;"; + + std::string resp = my_db->ExecuteSQL(select_sql); + std::cout << "SELECT: " << resp << std::endl; + } + + return 0; +} diff --git a/chain/storage/proto/BUILD b/chain/storage/proto/BUILD index 0b1c7263f..2558977b8 100644 --- a/chain/storage/proto/BUILD +++ b/chain/storage/proto/BUILD @@ -42,3 +42,14 @@ cc_proto_library( name = "leveldb_config_cc_proto", deps = [":leveldb_config_proto"], ) + +proto_library( + name = "duckdb_config_proto", + srcs = ["duckdb_config.proto"], +) + +cc_proto_library( + name = "duckdb_config_cc_proto", + deps = [":duckdb_config_proto"], +) + diff --git a/chain/storage/proto/duckdb_config.proto b/chain/storage/proto/duckdb_config.proto new file mode 100644 index 000000000..e90fa890c --- /dev/null +++ b/chain/storage/proto/duckdb_config.proto @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +syntax = "proto3"; + +package resdb.storage; + +message DuckDBInfo { + string path = 1; + optional bool use_wal = 2; + optional string max_memory = 3; + optional uint32 threads = 4; +} diff --git a/chain/storage/storage.h b/chain/storage/storage.h index 8fa95fee9..7a0bfe536 100644 --- a/chain/storage/storage.h +++ b/chain/storage/storage.h @@ -59,6 +59,9 @@ class Storage { virtual std::vector> GetTopHistory( const std::string& key, int number) = 0; + // Default no-op SQL execution for non-SQL backends. + virtual std::string ExecuteSQL(const std::string& sql_string) { return ""; } + virtual bool Flush() { return true; }; virtual uint64_t GetLastCheckpoint() { return 0; } diff --git a/executor/kv/kv_executor.cpp b/executor/kv/kv_executor.cpp index 719093a4b..9751e6453 100644 --- a/executor/kv/kv_executor.cpp +++ b/executor/kv/kv_executor.cpp @@ -71,10 +71,14 @@ std::unique_ptr KVExecutor::ExecuteRequest( } else if (kv_request.cmd() == KVRequest::GET_TOP) { GetTopHistory(kv_request.key(), kv_request.top_number(), kv_response.mutable_items()); - } else if (!kv_request.smart_contract_request().empty()) { - std::unique_ptr resp = - contract_manager_->ExecuteData(kv_request.smart_contract_request()); - if (resp != nullptr) { + } else if (kv_request.cmd() == KVRequest::SQL) { + std::string result = ExecuteSQL(kv_request.sql_query()); + kv_response.set_sql_response(result); + kv_response.set_value(result); // keep legacy field populated + } + else if(!kv_request.smart_contract_request().empty()){ + std::unique_ptr resp = contract_manager_->ExecuteData(kv_request.smart_contract_request()); + if(resp != nullptr){ kv_response.set_smart_contract_response(*resp); } } @@ -121,10 +125,14 @@ std::unique_ptr KVExecutor::ExecuteData( } else if (kv_request.cmd() == KVRequest::GET_TOP) { GetTopHistory(kv_request.key(), kv_request.top_number(), kv_response.mutable_items()); - } else if (!kv_request.smart_contract_request().empty()) { - std::unique_ptr resp = - contract_manager_->ExecuteData(kv_request.smart_contract_request()); - if (resp != nullptr) { + } else if (kv_request.cmd() == KVRequest::SQL) { + std::string result = ExecuteSQL(kv_request.sql_query()); + kv_response.set_sql_response(result); + kv_response.set_value(result); // keep legacy field populated + } + else if(!kv_request.smart_contract_request().empty()){ + std::unique_ptr resp = contract_manager_->ExecuteData(kv_request.smart_contract_request()); + if(resp != nullptr){ kv_response.set_smart_contract_response(*resp); } } @@ -202,4 +210,15 @@ void KVExecutor::GetTopHistory(const std::string& key, int top_number, } } +std::string KVExecutor::ExecuteSQL(const std::string& sql_query) { + // Basic validation: SQL commands should carry a query string. + if (sql_query.empty()) { + LOG(ERROR) << "SQL command received with empty sql_query"; + return "Error: empty SQL query"; + } + + return storage_->ExecuteSQL(sql_query); +} + + } // namespace resdb diff --git a/executor/kv/kv_executor.h b/executor/kv/kv_executor.h index 229dc8377..f0199b21f 100644 --- a/executor/kv/kv_executor.h +++ b/executor/kv/kv_executor.h @@ -56,6 +56,7 @@ class KVExecutor : public TransactionManager { void GetHistory(const std::string& key, int min_key, int max_key, Items* items); void GetTopHistory(const std::string& key, int top_number, Items* items); + std::string ExecuteSQL(const std::string& sql_query); private: std::unique_ptr contract_manager_; diff --git a/interface/kv/kv_client.cpp b/interface/kv/kv_client.cpp index edf7546be..82023dea3 100644 --- a/interface/kv/kv_client.cpp +++ b/interface/kv/kv_client.cpp @@ -133,4 +133,30 @@ std::unique_ptr KVClient::GetKeyTopHistory(const std::string& key, return std::make_unique(response.items()); } +std::unique_ptr KVClient::ExecuteSQL(const std::string& sql_query) { + if (sql_query.empty()) { + LOG(ERROR) << "SQL query is empty"; + return nullptr; + } + + KVRequest request; + request.set_cmd(KVRequest::SQL); + request.set_sql_query(sql_query); + + KVResponse response; + int ret = SendRequest(request, &response); + if (ret != 0) { + LOG(ERROR) << "send SQL request fail, ret:" << ret; + return nullptr; + } + + if (!response.sql_response().empty()) { + return std::make_unique(response.sql_response()); + } + if (!response.value().empty()) { + return std::make_unique(response.value()); + } + return std::make_unique(); +} + } // namespace resdb diff --git a/interface/kv/kv_client.h b/interface/kv/kv_client.h index a58aa424e..44daa9017 100644 --- a/interface/kv/kv_client.h +++ b/interface/kv/kv_client.h @@ -60,6 +60,9 @@ class KVClient : public TransactionConstructor { std::unique_ptr Get(const std::string& key); std::unique_ptr GetRange(const std::string& min_key, const std::string& max_key); + + // Execute an arbitrary SQL query when the KV service is backed by DuckDB. + std::unique_ptr ExecuteSQL(const std::string& sql_query); }; } // namespace resdb diff --git a/platform/proto/BUILD b/platform/proto/BUILD index 3b9166c39..8c35c8b43 100644 --- a/platform/proto/BUILD +++ b/platform/proto/BUILD @@ -37,6 +37,7 @@ proto_library( name = "replica_info_proto", srcs = ["replica_info.proto"], deps = [ + "//chain/storage/proto:duckdb_config_proto", "//chain/storage/proto:leveldb_config_proto", "//common/proto:signature_info_proto", ], @@ -53,6 +54,7 @@ python_proto_library( name = "replica_info_py_proto", protos = [ ":replica_info_proto", + "//chain/storage/proto:duckdb_config_proto", "//chain/storage/proto:leveldb_config_proto", ], deps = [ diff --git a/platform/proto/replica_info.proto b/platform/proto/replica_info.proto index eaeac73e1..8435ad0f4 100644 --- a/platform/proto/replica_info.proto +++ b/platform/proto/replica_info.proto @@ -23,6 +23,7 @@ package resdb; import "common/proto/signature_info.proto"; import "chain/storage/proto/leveldb_config.proto"; +import "chain/storage/proto/duckdb_config.proto"; message ReplicaInfo { int64 id = 1; @@ -40,6 +41,7 @@ message ResConfigData{ repeated RegionInfo region = 1; int32 self_region_id = 2; optional storage.LevelDBInfo leveldb_info = 4; + optional storage.DuckDBInfo duckdb_info = 25; optional bool enable_viewchange = 5; optional int32 view_change_timeout_ms = 10; optional bool not_need_signature = 6; // when delivering messages, it should be signed or not. diff --git a/proto/kv/kv.proto b/proto/kv/kv.proto index 750058e75..613403048 100644 --- a/proto/kv/kv.proto +++ b/proto/kv/kv.proto @@ -34,6 +34,8 @@ message KVRequest { GET_KEY_RANGE = 8; GET_HISTORY = 9; GET_TOP = 10; + // NEW: SQL command type + SQL = 11; } CMD cmd = 1; string key = 2; @@ -48,6 +50,8 @@ message KVRequest { // For top history int32 top_number = 9; bytes smart_contract_request = 10; + // NEW: raw SQL text for SQL commands (cmd == SQL) + string sql_query = 11; } message ValueInfo { @@ -70,5 +74,6 @@ message KVResponse { ValueInfo value_info = 3; Items items = 4; bytes smart_contract_response = 10; + // NEW: response payload for SQL commands + string sql_response = 11; } - diff --git a/service/kv/BUILD b/service/kv/BUILD index 0e44111f3..3a34c8ba3 100644 --- a/service/kv/BUILD +++ b/service/kv/BUILD @@ -35,7 +35,9 @@ cc_binary( "//common:comm", "//proto/kv:kv_cc_proto", "//chain/storage:memory_db", - ] + select({ + "//chain/storage:duckdb_storage", + "//chain/storage/proto:duckdb_config_cc_proto", + ] + select({ "//chain/storage/setting:enable_leveldb_setting": ["//chain/storage:leveldb"], "//conditions:default": [], }), diff --git a/service/kv/kv_service.cpp b/service/kv/kv_service.cpp index a3d2fabda..4c663bcef 100644 --- a/service/kv/kv_service.cpp +++ b/service/kv/kv_service.cpp @@ -19,7 +19,13 @@ #include +#include +#include +#include + #include "chain/storage/memory_db.h" +#include "chain/storage/duckdb.h" +#include "chain/storage/proto/duckdb_config.pb.h" #include "executor/kv/kv_executor.h" #include "platform/config/resdb_config_utils.h" #include "platform/statistic/stats.h" @@ -37,11 +43,25 @@ void SignalHandler(int sig_num) { } void ShowUsage() { - printf(" [logging_dir]\n"); + printf(" " + "[--enable_duckdb] [--duckdb_path=] " + "[--grafana_port= | ]\n"); } std::unique_ptr NewStorage(const std::string& db_path, - const ResConfigData& config_data) { + const ResConfigData& config_data, + bool enable_duckdb, + const std::string& duckdb_path) { + if (enable_duckdb) { + std::string path = duckdb_path; + if (path.empty()) { + path = db_path + "duckdb.db"; + } + LOG(INFO) << "use duckdb storage, path=" << path; + resdb::storage::DuckDBInfo duckdb_info; + duckdb_info.set_path(path); + return NewResQL(path, duckdb_info); + } #ifdef ENABLE_LEVELDB LOG(INFO) << "use leveldb storage."; return NewResLevelDB(db_path, config_data.leveldb_info()); @@ -64,8 +84,30 @@ int main(int argc, char** argv) { char* private_key_file = argv[2]; char* cert_file = argv[3]; - if (argc == 5) { - std::string grafana_port = argv[4]; + bool enable_duckdb = false; + std::string duckdb_path; + std::string grafana_port; + + for (int i = 4; i < argc; ++i) { + std::string arg = argv[i]; + if (arg == "--enable_duckdb") { + enable_duckdb = true; + } else if (arg.rfind("--duckdb_path=", 0) == 0) { + duckdb_path = arg.substr(std::string("--duckdb_path=").size()); + } else if (arg.rfind("--grafana_port=", 0) == 0) { + grafana_port = arg.substr(std::string("--grafana_port=").size()); + } else if (grafana_port.empty() && + std::all_of(arg.begin(), arg.end(), + [](unsigned char ch) { return std::isdigit(ch); })) { + // Backward-compatible: allow bare grafana port as 4th arg. + grafana_port = arg; + } else { + ShowUsage(); + exit(1); + } + } + + if (!grafana_port.empty()) { std::string grafana_address = "0.0.0.0:" + grafana_port; auto monitor_port = Stats::GetGlobalStats(5); @@ -82,6 +124,8 @@ int main(int argc, char** argv) { auto server = GenerateResDBServer( config_file, private_key_file, cert_file, - std::make_unique(NewStorage(db_path, config_data)), nullptr); + std::make_unique( + NewStorage(db_path, config_data, enable_duckdb, duckdb_path)), + nullptr); server->Run(); } diff --git a/service/tools/config/interface/service.config b/service/tools/config/interface/service.config index a437dbc7f..77b706cc1 100644 --- a/service/tools/config/interface/service.config +++ b/service/tools/config/interface/service.config @@ -25,4 +25,4 @@ ] } - +5 127.0.0.1 20005 diff --git a/service/tools/config/server.config b/service/tools/config/server.config index 756dc00f2..16ace299b 100644 --- a/service/tools/config/server.config +++ b/service/tools/config/server.config @@ -1,19 +1,3 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. 1 127.0.0.1 20001 2 127.0.0.1 20002 diff --git a/service/tools/config/server/server.config b/service/tools/config/server/server.config index 9aa3e6fd6..5b5cfa3c1 100644 --- a/service/tools/config/server/server.config +++ b/service/tools/config/server/server.config @@ -50,6 +50,4 @@ enable_viewchange:false, enable_resview:true, enable_faulty_switch:false -} - - +} \ No newline at end of file diff --git a/service/tools/kv/api_tools/kv_service_tools.cpp b/service/tools/kv/api_tools/kv_service_tools.cpp index 71953ff70..8c64cf101 100644 --- a/service/tools/kv/api_tools/kv_service_tools.cpp +++ b/service/tools/kv/api_tools/kv_service_tools.cpp @@ -40,9 +40,10 @@ void ShowUsage() { "--config: config path\n" "--cmd " "set/get/set_with_version/get_with_version/get_key_range/" - "get_key_range_with_version/get_top/get_history\n" + "get_key_range_with_version/get_top/get_history/sql\n" "--key key\n" "--value value, if cmd is a get operation\n" + "--sql SQL text, if cmd is sql\n" "--version version of the value, if cmd is vesion based\n" "--min_key the min key if cmd is get_key_range\n" "--max_key the max key if cmd is get_key_range\n" @@ -65,6 +66,7 @@ static struct option long_options[] = { {"min_key", required_argument, NULL, 'y'}, {"max_key", required_argument, NULL, 'Y'}, {"top", required_argument, NULL, 't'}, + {"sql", required_argument, NULL, 'q'}, }; void OldAPI(char** argv) { @@ -119,6 +121,7 @@ int main(int argc, char** argv) { int min_version = -1, max_version = -1; std::string min_key, max_key; std::string value; + std::string sql; std::string client_config_file; int top = 0; char c; @@ -166,6 +169,9 @@ int main(int argc, char** argv) { case 't': top = strtoull(optarg, NULL, 10); break; + case 'q': + sql = optarg; + break; case 'h': ShowUsage(); break; @@ -269,6 +275,17 @@ int main(int argc, char** argv) { printf("getrange value fail, min key = %s, max key = %s\n", min_key.c_str(), max_key.c_str()); } + } else if (cmd == "sql") { + if (sql.empty()) { + ShowUsage(); + return 0; + } + auto res = client.ExecuteSQL(sql); + if (res != nullptr) { + printf("SQL result:\n%s\n", res->c_str()); + } else { + printf("SQL query failed\n"); + } } else { ShowUsage(); } diff --git a/service/tools/kv/server_tools/start_resql_service.sh b/service/tools/kv/server_tools/start_resql_service.sh new file mode 100755 index 000000000..73c0c10bd --- /dev/null +++ b/service/tools/kv/server_tools/start_resql_service.sh @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Launcher for the KV service with optional DuckDB flags. +# Uses the same topology as start_kv_service.sh but injects DuckDB flags +# when present. + +set -euo pipefail + +killall -9 kv_service >/dev/null 2>&1 || true + +SERVER_PATH=./bazel-bin/service/kv/kv_service +SERVER_CONFIG=service/tools/config/server/server.config +WORK_PATH=$PWD +CERT_PATH=${WORK_PATH}/service/tools/data/cert/ + +# Always enable DuckDB with a default path per node. +DEFAULT_DUCKDB_PATH=/tmp +EXTRA_FLAGS="--enable_duckdb" + + +# No additional flag parsing; DuckDB is always enabled. +bazel build //service/kv:kv_service + +for port in {20001..20005}; do + DIR="${WORK_PATH}/${port}_db" + if [ ! -d "$DIR" ]; then + echo "Creating directory $DIR" + mkdir -p "$DIR" + else + echo "Directory $DIR already exists. Skipping." + fi +done + +nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node1.key.pri $CERT_PATH/cert_1.cert $EXTRA_FLAGS > server0.log 2>&1 & +nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node2.key.pri $CERT_PATH/cert_2.cert $EXTRA_FLAGS > server1.log 2>&1 & +nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node3.key.pri $CERT_PATH/cert_3.cert $EXTRA_FLAGS > server2.log 2>&1 & +nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node4.key.pri $CERT_PATH/cert_4.cert $EXTRA_FLAGS > server3.log 2>&1 & + +# Optional client node +nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node5.key.pri $CERT_PATH/cert_5.cert $EXTRA_FLAGS > client.log 2>&1 & + +echo "Started KV service with DuckDB enabled." diff --git a/test_sql.sh b/test_sql.sh new file mode 100755 index 000000000..a41d0896a --- /dev/null +++ b/test_sql.sh @@ -0,0 +1,145 @@ +#!/bin/bash +set -euo pipefail + +TOOL="bazel-bin/service/tools/kv/api_tools/kv_service_tools" +CONFIG="service/tools/config/interface/service.config" +LOGFILE="sql_test.log" + +# Colors +BLUE="\033[1;34m" +GREEN="\033[1;32m" +YELLOW="\033[1;33m" +RESET="\033[0m" + +run_sql() { + echo -e "${GREEN}>>> $1${RESET}" + echo "---- $1 ----" >> $LOGFILE + $TOOL --config $CONFIG --cmd sql --sql "$2" | tee -a $LOGFILE + echo "" >> $LOGFILE +} + +echo -e "${BLUE}===== Starting SQL Test Suite =====${RESET}" +echo "SQL Test Log - $(date)" > $LOGFILE +echo "" >> $LOGFILE + +# 1. CREATE TABLES +run_sql "Create table: users" \ +"CREATE TABLE users (id INTEGER, name VARCHAR, age INTEGER);" + +run_sql "Create table: orders" \ +"CREATE TABLE orders (order_id INTEGER, user_id INTEGER, amount DOUBLE, status VARCHAR);" + +# 2. INSERT DATA +run_sql "Insert into users" \ +"INSERT INTO users VALUES + (1,'Alice',30), + (2,'Bob',22), + (3,'Charlie',27), + (4,'Diana',35);" + +run_sql "Insert into orders" \ +"INSERT INTO orders VALUES + (101,1,250.50,'shipped'), + (102,1,120.00,'processing'), + (103,2,75.00,'cancelled'), + (104,3,500.00,'shipped'), + (105,3,20.00,'processing');" + +# 3. BASIC SELECTS +run_sql "Select all users" \ +"SELECT * FROM users;" + +run_sql "Filter: age > 25" \ +"SELECT name, age FROM users WHERE age > 25;" + +# 4. ORDER BY +run_sql "Order users by age DESC" \ +"SELECT * FROM users ORDER BY age DESC;" + +# 5. AGGREGATIONS +run_sql "Count users" \ +"SELECT COUNT(*) AS total_users FROM users;" + +run_sql "Average order amount" \ +"SELECT AVG(amount) AS avg_order FROM orders;" + +run_sql "Total spent per user" \ +"SELECT user_id, SUM(amount) AS total_spent FROM orders GROUP BY user_id;" + +# 6. JOIN TESTS +run_sql "JOIN: users + orders" \ +"SELECT u.name, o.order_id, o.amount + FROM users u + JOIN orders o ON u.id = o.user_id;" + +run_sql "JOIN + filter amount > 100" \ +"SELECT u.name, o.amount + FROM users u + JOIN orders o ON u.id = o.user_id + WHERE o.amount > 100;" + +run_sql "LEFT JOIN" \ +"SELECT u.id, u.name, o.order_id + FROM users u + LEFT JOIN orders o ON u.id = o.user_id + ORDER BY u.id;" + +# 7. UPDATE / DELETE +run_sql "Update Bob's age" \ +"UPDATE users SET age = age + 1 WHERE id = 2;" + +run_sql "Delete cancelled orders" \ +"DELETE FROM orders WHERE status = 'cancelled';" + +# 8. GROUPED JOIN AGGREGATE +run_sql "Total spent by each user (ordered)" \ +"SELECT u.name, SUM(o.amount) AS total_spent + FROM users u + JOIN orders o ON u.id = o.user_id + GROUP BY u.id, u.name + ORDER BY total_spent DESC;" + +# 9. DISTINCT / LIKE / BETWEEN / LIMIT +run_sql "Distinct order statuses" \ +"SELECT DISTINCT status FROM orders;" + +run_sql "Names starting with A" \ +"SELECT * FROM users WHERE name LIKE 'A%';" + +run_sql "Age between 25 and 35" \ +"SELECT * FROM users WHERE age BETWEEN 25 AND 35;" + +run_sql "Top orders (limit + offset)" \ +"SELECT * FROM orders ORDER BY amount DESC LIMIT 2 OFFSET 1;" + +# 10. STRING FUNCTIONS +run_sql "String functions test" \ +"SELECT UPPER(name), LENGTH(name) FROM users;" + +# 11. TRANSACTION BLOCK +run_sql "Transaction test" \ +"BEGIN TRANSACTION; + INSERT INTO users VALUES (10, 'Eva', 40); + UPDATE users SET age = 41 WHERE id = 10; + COMMIT;" + +# 12. ERROR TESTS (expected failure) +echo -e "${YELLOW}>>> Testing expected errors (these should fail)${RESET}" + +run_sql "Malformed row insert (should error)" \ +"INSERT INTO users VALUES ('bad','data',123);" || true + +run_sql "Select from invalid table (should error)" \ +"SELECT * FROM nonexistent;" || true + +echo -e "${BLUE}===== Now cleaning up =====${RESET}" + +# 13. CLEANUP +run_sql "Drop table: orders" \ +"DROP TABLE orders;" + +run_sql "Drop table: users" \ +"DROP TABLE users;" + +echo -e "${BLUE}===== SQL Test Suite Complete =====${RESET}" +echo "Full output written to $LOGFILE" diff --git a/third_party/BUILD b/third_party/BUILD index 8d2b1b8aa..6c2898028 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -57,3 +57,12 @@ cc_library( "@com_crowcpp_crow//:crow", ], ) + +cc_library( + name = "duckdb", + deps = [ + "@duckdb//:duckdb", + ], +) + + diff --git a/third_party/duckdb.BUILD b/third_party/duckdb.BUILD new file mode 100644 index 000000000..0d3ddc1f6 --- /dev/null +++ b/third_party/duckdb.BUILD @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "duckdb", + # The amalgamation zip usually just has these three at the root. + srcs = ["duckdb.cpp"], + hdrs = [ + "duckdb.hpp", + "duckdb.h", + ], + includes = ["."], + # ResilientDB already builds with modern C++; C++17 is safe for DuckDB. + copts = [ + "-std=c++17", + # Optional: silence some noisy warnings that big amalgamations usually trigger. + "-Wno-unused-parameter", + "-Wno-unused-variable", + "-Wno-sign-compare", + ], +) + + + + diff --git a/third_party/zlib.BUILD b/third_party/zlib.BUILD index c94061643..e83ec22f3 100644 --- a/third_party/zlib.BUILD +++ b/third_party/zlib.BUILD @@ -56,5 +56,6 @@ cc_library( ], copts = [ "-D_LARGEFILE64_SOURCE=1", + "-UNO_FDOPEN", ], )