From c91941d8435a07231e40e885da35225cb364eeac Mon Sep 17 00:00:00 2001 From: ubyte Date: Tue, 25 Nov 2025 16:30:33 +0000 Subject: [PATCH 01/19] SQS Topic GetQueueUrl stub (#27623) --- .github/last_commit.txt | 2 +- src/api/grpc/draft/ydb_sqs_topic_v1.proto | 29 +++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 src/api/grpc/draft/ydb_sqs_topic_v1.proto diff --git a/.github/last_commit.txt b/.github/last_commit.txt index b766f8b3117..d2acc8f541d 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -eaa66fe4cc75727a60d89a9ec9d99970e03ce0e0 +65a59dfb8903f3e9489caf08ad84d291babb8755 diff --git a/src/api/grpc/draft/ydb_sqs_topic_v1.proto b/src/api/grpc/draft/ydb_sqs_topic_v1.proto new file mode 100644 index 00000000000..8a575ec7e45 --- /dev/null +++ b/src/api/grpc/draft/ydb_sqs_topic_v1.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; +option cc_enable_arenas = true; + +package Ydb.SqsTopic.V1; + +import "src/api/protos/draft/ymq.proto"; + +option java_package = "com.yandex.ydb.sqs_topic.V1."; + +service SqsTopicService { + rpc SqsTopicGetQueueUrl(Ydb.Ymq.V1.GetQueueUrlRequest) returns (Ydb.Ymq.V1.GetQueueUrlResponse); + rpc SqsTopicCreateQueue(Ydb.Ymq.V1.CreateQueueRequest) returns (Ydb.Ymq.V1.CreateQueueResponse); + rpc SqsTopicSendMessage(Ydb.Ymq.V1.SendMessageRequest) returns (Ydb.Ymq.V1.SendMessageResponse); + rpc SqsTopicReceiveMessage(Ydb.Ymq.V1.ReceiveMessageRequest) returns (Ydb.Ymq.V1.ReceiveMessageResponse); + rpc SqsTopicGetQueueAttributes(Ydb.Ymq.V1.GetQueueAttributesRequest) returns (Ydb.Ymq.V1.GetQueueAttributesResponse); + rpc SqsTopicListQueues(Ydb.Ymq.V1.ListQueuesRequest) returns (Ydb.Ymq.V1.ListQueuesResponse); + rpc SqsTopicDeleteMessage(Ydb.Ymq.V1.DeleteMessageRequest) returns (Ydb.Ymq.V1.DeleteMessageResponse); + rpc SqsTopicPurgeQueue(Ydb.Ymq.V1.PurgeQueueRequest) returns (Ydb.Ymq.V1.PurgeQueueResponse); + rpc SqsTopicDeleteQueue(Ydb.Ymq.V1.DeleteQueueRequest) returns (Ydb.Ymq.V1.DeleteQueueResponse); + rpc SqsTopicChangeMessageVisibility(Ydb.Ymq.V1.ChangeMessageVisibilityRequest) returns (Ydb.Ymq.V1.ChangeMessageVisibilityResponse); + rpc SqsTopicSetQueueAttributes(Ydb.Ymq.V1.SetQueueAttributesRequest) returns (Ydb.Ymq.V1.SetQueueAttributesResponse); + rpc SqsTopicSendMessageBatch(Ydb.Ymq.V1.SendMessageBatchRequest) returns (Ydb.Ymq.V1.SendMessageBatchResponse); + rpc SqsTopicDeleteMessageBatch(Ydb.Ymq.V1.DeleteMessageBatchRequest) returns (Ydb.Ymq.V1.DeleteMessageBatchResponse); + rpc SqsTopicChangeMessageVisibilityBatch(Ydb.Ymq.V1.ChangeMessageVisibilityBatchRequest) returns (Ydb.Ymq.V1.ChangeMessageVisibilityBatchResponse); + rpc SqsTopicListDeadLetterSourceQueues(Ydb.Ymq.V1.ListDeadLetterSourceQueuesRequest) returns (Ydb.Ymq.V1.ListDeadLetterSourceQueuesResponse); + rpc SqsTopicListQueueTags(Ydb.Ymq.V1.ListQueueTagsRequest) returns (Ydb.Ymq.V1.ListQueueTagsResponse); + rpc SqsTopicTagQueue(Ydb.Ymq.V1.TagQueueRequest) returns (Ydb.Ymq.V1.TagQueueResponse); + rpc SqsTopicUntagQueue(Ydb.Ymq.V1.UntagQueueRequest) returns (Ydb.Ymq.V1.UntagQueueResponse); +} From e751201eff9683b10904c4657561b171e3837e53 Mon Sep 17 00:00:00 2001 From: ubyte Date: Tue, 25 Nov 2025 16:30:40 +0000 Subject: [PATCH 02/19] explicitly call TString::MutRef in the conversion to std::string (#28207) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/library/issue/yql_issue.h | 13 ++----------- src/library/issue/yql_issue.cpp | 13 +++++++++++++ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index d2acc8f541d..97577f26f61 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -65a59dfb8903f3e9489caf08ad84d291babb8755 +b7967978c1926e4abd91c052885ecea3c3cdcaea diff --git a/include/ydb-cpp-sdk/library/issue/yql_issue.h b/include/ydb-cpp-sdk/library/issue/yql_issue.h index 7ceab9be9bc..ed80ac0f32c 100644 --- a/include/ydb-cpp-sdk/library/issue/yql_issue.h +++ b/include/ydb-cpp-sdk/library/issue/yql_issue.h @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -183,11 +182,7 @@ class TIssue: public TThrRefBase { void PrintTo(IOutputStream& out, bool oneLine = false) const; - std::string ToString(bool oneLine = false) const { - TStringStream out; - PrintTo(out, oneLine); - return out.Str(); - } + std::string ToString(bool oneLine = false) const; // Unsafe method. Doesn't call SanitizeNonAscii(Message) std::string* MutableMessage() { @@ -295,11 +290,7 @@ class TIssues { const std::string& programFilename, const std::string& programText) const; - inline std::string ToString(bool oneLine = false) const { - TStringStream out; - PrintTo(out, oneLine); - return out.Str(); - } + std::string ToString(bool oneLine = false) const; std::string ToOneLineString() const { return ToString(true); diff --git a/src/library/issue/yql_issue.cpp b/src/library/issue/yql_issue.cpp index b7e73df2d11..49fe10286bd 100644 --- a/src/library/issue/yql_issue.cpp +++ b/src/library/issue/yql_issue.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -79,6 +80,12 @@ void TIssue::PrintTo(IOutputStream& out, bool oneLine) const { } } +std::string TIssue::ToString(bool oneLine) const { + TStringStream out; + PrintTo(out, oneLine); + return std::move(out.Str().MutRef()); +} + void WalkThroughIssues(const TIssue& topIssue, bool leafOnly, std::function fn, std::function afterChildrenFn) { enum class EFnType { Main, @@ -224,6 +231,12 @@ void TIssues::PrintWithProgramTo( } } +std::string TIssues::ToString(bool oneLine) const { + TStringStream out; + PrintTo(out, oneLine); + return std::move(out.Str().MutRef()); +} + TIssue ExceptionToIssue(const std::exception& e, const TPosition& pos) { std::string_view messageBuf = e.what(); auto parsedPos = TryParseTerminationMessage(messageBuf); From 1f6568899eceae7e255b5f662c07730812679a65 Mon Sep 17 00:00:00 2001 From: stanislav_shchetinin Date: Tue, 25 Nov 2025 16:30:47 +0000 Subject: [PATCH 03/19] ExportToFs / ImportFromFs API (#28072) --- .github/last_commit.txt | 2 +- src/api/grpc/ydb_export_v1.proto | 4 +++ src/api/grpc/ydb_import_v1.proto | 4 +++ src/api/protos/ydb_export.proto | 51 ++++++++++++++++++++++++++ src/api/protos/ydb_import.proto | 62 +++++++++++++++++++++++++++++++- 5 files changed, 121 insertions(+), 2 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 97577f26f61..ecd56de8ccf 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -b7967978c1926e4abd91c052885ecea3c3cdcaea +ab114b4a2362763ffcd64ba31d1ffc4d80aa6a1b diff --git a/src/api/grpc/ydb_export_v1.proto b/src/api/grpc/ydb_export_v1.proto index 8bdb8bf24c5..6e4a37f09a4 100644 --- a/src/api/grpc/ydb_export_v1.proto +++ b/src/api/grpc/ydb_export_v1.proto @@ -14,4 +14,8 @@ service ExportService { // Exports data to S3. // Method starts an asynchronous operation that can be cancelled while it is in progress. rpc ExportToS3(Export.ExportToS3Request) returns (Export.ExportToS3Response); + + // Exports data to file system. + // Method starts an asynchronous operation that can be cancelled while it is in progress. + rpc ExportToFs(ExportToFsRequest) returns (ExportToFsResponse); } diff --git a/src/api/grpc/ydb_import_v1.proto b/src/api/grpc/ydb_import_v1.proto index 48034218699..49c69424977 100644 --- a/src/api/grpc/ydb_import_v1.proto +++ b/src/api/grpc/ydb_import_v1.proto @@ -11,6 +11,10 @@ service ImportService { // Method starts an asynchronous operation that can be cancelled while it is in progress. rpc ImportFromS3(Import.ImportFromS3Request) returns (Import.ImportFromS3Response); + // Imports data from file system. + // Method starts an asynchronous operation that can be cancelled while it is in progress. + rpc ImportFromFs(Import.ImportFromFsRequest) returns (Import.ImportFromFsResponse); + // List objects from existing export stored in S3 bucket rpc ListObjectsInS3Export(Import.ListObjectsInS3ExportRequest) returns (Import.ListObjectsInS3ExportResponse); diff --git a/src/api/protos/ydb_export.proto b/src/api/protos/ydb_export.proto index f8f3a8e7f0f..91512305f4e 100644 --- a/src/api/protos/ydb_export.proto +++ b/src/api/protos/ydb_export.proto @@ -181,3 +181,54 @@ message EncryptionSettings { bytes key = 1 [(Ydb.sensitive) = true]; } } + +/// File system (FS) +message ExportToFsSettings { + message Item { + // Database path to a table to be exported + string source_path = 1 [(required) = true]; + + /* The tables are exported to one or more files in FS. + The files are saved to the destination_path directory. */ + string destination_path = 2 [(required) = true]; + } + + // Base path on FS where to write export + // Path to the mounted directory in the case of NFS + // Example: /mnt/exports + string base_path = 1 [(required) = true]; + + // List of items to export + repeated Item items = 2; + + // Optional description + string description = 3 [(length).le = 128]; + + // Number of retries for failed operations + uint32 number_of_retries = 4; + + // Codec used to compress data. Codecs are available: + // - zstd. + // - zstd-N, where N is compression level, e.g. zstd-3. + string compression = 5; +} + +message ExportToFsResult { +} + +message ExportToFsMetadata { + ExportToFsSettings settings = 1; + ExportProgress.Progress progress = 2; + repeated ExportItemProgress items_progress = 3; +} + +message ExportToFsRequest { + Ydb.Operations.OperationParams operation_params = 1; + ExportToFsSettings settings = 2 [(required) = true]; +} + +message ExportToFsResponse { + // operation.result = ExportToFsResult + // operation.metadata = ExportToFsMetadata + Ydb.Operations.Operation operation = 1; +} diff --git a/src/api/protos/ydb_import.proto b/src/api/protos/ydb_import.proto index 50e48dbba7b..d247a4715e0 100644 --- a/src/api/protos/ydb_import.proto +++ b/src/api/protos/ydb_import.proto @@ -45,7 +45,8 @@ message ImportFromS3Settings { The S3 object name begins with a prefix, followed by: * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero; * '/scheme.pb' - object with information about scheme, indexes, etc; - * '/permissions.pb' - object with information about ACL and owner. + * '/permissions.pb' - object with information about ACL and owner; + * '/metadata.json' - object with metadata about the backup. */ // The S3 object prefix can be either provided explicitly @@ -121,6 +122,65 @@ message ImportFromS3Response { Ydb.Operations.Operation operation = 1; } +/// File system (FS) +message ImportFromFsSettings { + message Item { + /* YDB tables in FS are stored in a directory structure (see ydb_export.proto). + The directory contains: + * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero; + * '/scheme.pb' - object with information about scheme, indexes, etc; + * '/permissions.pb' - object with information about ACL and owner; + * '/metadata.json' - object with metadata about the backup. + The path in FS is specified relative to base_path. + Example: "my_export/table1" + */ + string source_path = 1 [(required) = true]; + + // Database path to a table to import to. + string destination_path = 2 [(required) = true]; + } + + // Base path on FS where the export is located + // Path to the mounted directory in the case of NFS + // Example: /mnt/exports + string base_path = 1 [(required) = true]; + + repeated Item items = 2; // Empty collection means import of all export objects + + // Optional description + string description = 3 [(length).le = 128]; + + // Number of retries for failed file operations + uint32 number_of_retries = 4; + + // Prevent importing of ACL and owner. If true, objects are created with empty ACL + // and their owner will be the user who started the import. + bool no_acl = 5; + + // Skip checksum validation during import + bool skip_checksum_validation = 6; +} + +message ImportFromFsResult { +} + +message ImportFromFsMetadata { + ImportFromFsSettings settings = 1; + ImportProgress.Progress progress = 2; + repeated ImportItemProgress items_progress = 3; +} + +message ImportFromFsRequest { + Ydb.Operations.OperationParams operation_params = 1; + ImportFromFsSettings settings = 2 [(required) = true]; +} + +message ImportFromFsResponse { + // operation.result = ImportFromFsResult + // operation.metadata = ImportFromFsMetadata + Ydb.Operations.Operation operation = 1; +} + message ListObjectsInS3ExportSettings { message Item { // Database object path From c0c04e0a760d8f58b1d7664aee235097cbf97b01 Mon Sep 17 00:00:00 2001 From: Nikolay Shestakov Date: Tue, 25 Nov 2025 16:30:53 +0000 Subject: [PATCH 04/19] =?UTF-8?q?=20=D0=98=D0=B7=D0=BC=D0=B5=D0=BD=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D1=8F=20=D0=B2=20public=20API=20=D0=B4=D0=BB=D1=8F?= =?UTF-8?q?=20=D0=BF=D0=BE=D0=B4=D0=B4=D0=B5=D1=80=D0=B6=D0=BA=D0=B8=20Mes?= =?UTF-8?q?sage=20Level=20Parallelism=20(#27707)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/last_commit.txt | 2 +- .../ydb-cpp-sdk/client/topic/control_plane.h | 235 +++++++- src/api/protos/ydb_topic.proto | 81 ++- src/client/topic/impl/topic.cpp | 184 ++++++ src/client/topic/impl/topic_impl.h | 29 +- src/client/topic/ut/basic_usage_ut.cpp | 533 ++++++++++++++++++ 6 files changed, 1028 insertions(+), 36 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index ecd56de8ccf..fa9765ac068 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -ab114b4a2362763ffcd64ba31d1ffc4d80aa6a1b +f6c8d8cb6c24a823a2ef2885113d4b1fdabc7eaa diff --git a/include/ydb-cpp-sdk/client/topic/control_plane.h b/include/ydb-cpp-sdk/client/topic/control_plane.h index 2b18590d83c..f065fb78cde 100644 --- a/include/ydb-cpp-sdk/client/topic/control_plane.h +++ b/include/ydb-cpp-sdk/client/topic/control_plane.h @@ -36,6 +36,18 @@ enum class EAutoPartitioningStrategy: uint32_t { Paused = 4, }; +enum class EConsumerType { + Unspecified = 0, + Streaming = 1, + Shared = 2, +}; + +enum class EDeadLetterAction { + Unspecified = 0, + Delete = 1, + Move = 2, +}; + // 0 - unspecified // 1 - disabeld // 2 - database level metrics @@ -43,28 +55,61 @@ enum class EAutoPartitioningStrategy: uint32_t { // 4 - detailed metrics using EMetricsLevel = uint32_t; +class TDeadLetterPolicyCondition { +public: + TDeadLetterPolicyCondition(const Ydb::Topic::DeadLetterPolicyCondition&); + + ui32 GetMaxProcessingAttempts() const; + +private: + ui32 MaxProcessingAttempts_ = 0; +}; + +class TDeadLetterPolicy { +public: + TDeadLetterPolicy(const Ydb::Topic::DeadLetterPolicy&); + + bool GetEnabled() const; + const TDeadLetterPolicyCondition& GetCondition() const; + EDeadLetterAction GetAction() const; + const std::string& GetDeadLetterQueue() const; + +private: + bool Enabled_; + TDeadLetterPolicyCondition Condition_; + EDeadLetterAction Action_; + std::string DeadLetterQueue_; + +}; + class TConsumer { public: TConsumer(const Ydb::Topic::Consumer&); const std::string& GetConsumerName() const; + EConsumerType GetConsumerType() const; bool GetImportant() const; TDuration GetAvailabilityPeriod() const; const TInstant& GetReadFrom() const; const std::vector& GetSupportedCodecs() const; const std::map& GetAttributes() const; + bool GetKeepMessagesOrder() const; + TDuration GetDefaultProcessingTimeout() const; + const TDeadLetterPolicy& GetDeadLetterPolicy() const; private: std::string ConsumerName_; + EConsumerType ConsumerType_; bool Important_; TDuration AvailabilityPeriod_; TInstant ReadFrom_; std::map Attributes_; std::vector SupportedCodecs_; - + bool KeepMessagesOrder_; + TDuration DefaultProcessingTimeout_; + TDeadLetterPolicy DeadLetterPolicy_; }; - class TTopicStats { public: TTopicStats(const Ydb::Topic::DescribeTopicResult::TopicStats& topicStats); @@ -450,10 +495,149 @@ struct TAlterTopicSettings; using TAlterConsumerAttributesBuilder = TAlterAttributesBuilderImpl; using TAlterTopicAttributesBuilder = TAlterAttributesBuilderImpl; +struct TDeadLetterPolicyConditionSettings { + using TSelf = TDeadLetterPolicyConditionSettings; + + TDeadLetterPolicyConditionSettings(const Ydb::Topic::DeadLetterPolicyCondition&); + TDeadLetterPolicyConditionSettings() = default; + TDeadLetterPolicyConditionSettings(const TSelf&) = default; + TDeadLetterPolicyConditionSettings(TSelf&&) = default; + TSelf& operator=(const TSelf&) = default; + + FLUENT_SETTING_OPTIONAL(ui32, MaxProcessingAttempts); +}; + +template +struct TDeadLetterPolicyConditionBuilder { + TDeadLetterPolicyConditionBuilder(TParent& parent) : Parent_(parent) {}; + + TDeadLetterPolicyConditionBuilder& MaxProcessingAttempts(ui32 maxProcessingAttempts) { + Parent_.Parent_.DeadLetterPolicy_.Condition_.MaxProcessingAttempts_ = maxProcessingAttempts; + return *this; + } + + TParent& EndCondition() { return Parent_; } + +private: + TParent& Parent_; +}; + +struct TDeadLetterPolicySettings { + using TSelf = TDeadLetterPolicySettings; + + TDeadLetterPolicySettings() = default; + TDeadLetterPolicySettings(const Ydb::Topic::DeadLetterPolicy&); + + FLUENT_SETTING_OPTIONAL(bool, Enabled); + FLUENT_SETTING(TDeadLetterPolicyConditionSettings, Condition); + FLUENT_SETTING_DEFAULT(EDeadLetterAction, Action, EDeadLetterAction::Unspecified); + FLUENT_SETTING_OPTIONAL(std::string, DeadLetterQueue); +}; + +template +struct TDeadLetterPolicyBuilder { + using TSelf = TDeadLetterPolicyBuilder; + friend struct TDeadLetterPolicyConditionBuilder; + + TDeadLetterPolicyBuilder(TParent& parent) : Parent_(parent) {}; + + TSelf& Enable() { + return Enabled(true); + } + + TSelf& Disable() { + return Enabled(false); + } + + TSelf& Enabled(bool enabled) { + Parent_.DeadLetterPolicy_.Enabled_ = enabled; + return *this; + } + + TDeadLetterPolicyConditionBuilder BeginCondition() { + return TDeadLetterPolicyConditionBuilder(*this); + } + + TSelf& DeleteAction() { + Parent_.DeadLetterPolicy_.Action_ = EDeadLetterAction::Delete; + return *this; + } + + TSelf& MoveAction(const std::string& deadLetterQueue) { + Parent_.DeadLetterPolicy_.Action_ = EDeadLetterAction::Move; + Parent_.DeadLetterPolicy_.DeadLetterQueue_ = deadLetterQueue; + return *this; + } + + TParent& EndDeadLetterPolicy() { return Parent_; } + +private: + TParent& Parent_; +}; + +struct TAlterDeadLetterPolicySettings { + using TSelf = TAlterDeadLetterPolicySettings; + + FLUENT_SETTING_OPTIONAL(bool, Enabled); + FLUENT_SETTING_DEFAULT(bool, DeadLetterPolicyChanged, false); + FLUENT_SETTING_OPTIONAL(EDeadLetterAction, Action); + FLUENT_SETTING(TDeadLetterPolicyConditionSettings, Condition); + FLUENT_SETTING_OPTIONAL(std::string, DeadLetterQueue); +}; + +template +struct TAlterDeadLetterPolicyBuilder { + using TSelf = TAlterDeadLetterPolicyBuilder; + friend struct TDeadLetterPolicyConditionBuilder; + + TAlterDeadLetterPolicyBuilder(TParent& parent) : Parent_(parent) {}; + + TSelf& Enable() { + return Enabled(true); + } + + TSelf& Disable() { + return Enabled(false); + } + + TSelf& Enabled(bool enabled) { + Parent_.DeadLetterPolicy_.Enabled_ = enabled; + return *this; + } + + TDeadLetterPolicyConditionBuilder BeginCondition() { + return TDeadLetterPolicyConditionBuilder(*this); + } + + TSelf& SetDeleteAction() { + Parent_.DeadLetterPolicy_.DeadLetterPolicyChanged_ = true; + Parent_.DeadLetterPolicy_.Action_ = EDeadLetterAction::Delete; + return *this; + } + + TSelf& AlterMoveAction(const std::string& deadLetterQueue) { + Parent_.DeadLetterPolicy_.DeadLetterPolicyChanged_ = false; + Parent_.DeadLetterPolicy_.Action_ = EDeadLetterAction::Move; + Parent_.DeadLetterPolicy_.DeadLetterQueue_ = deadLetterQueue; + return *this; + } + + TSelf& SetMoveAction(const std::string& deadLetterQueue) { + Parent_.DeadLetterPolicy_.DeadLetterPolicyChanged_ = true; + Parent_.DeadLetterPolicy_.Action_ = EDeadLetterAction::Move; + Parent_.DeadLetterPolicy_.DeadLetterQueue_ = deadLetterQueue; + return *this; + } + + TParent& EndAlterDeadLetterPolicy() { return Parent_; } + +private: + TParent& Parent_; +}; + template struct TConsumerSettings { using TSelf = TConsumerSettings; - using TAttributes = std::map; TConsumerSettings(TSettings& parent) : Parent_(parent) {} @@ -463,12 +647,17 @@ struct TConsumerSettings { void SerializeTo(Ydb::Topic::Consumer& proto) const; FLUENT_SETTING(std::string, ConsumerName); + FLUENT_SETTING_DEFAULT(EConsumerType, ConsumerType, EConsumerType::Streaming); FLUENT_SETTING_DEFAULT(bool, Important, false); FLUENT_SETTING_DEFAULT(TDuration, AvailabilityPeriod, TDuration::Zero()); FLUENT_SETTING_DEFAULT(TInstant, ReadFrom, TInstant::Zero()); FLUENT_SETTING_VECTOR(ECodec, SupportedCodecs); + FLUENT_SETTING_OPTIONAL(bool, KeepMessagesOrder); + FLUENT_SETTING_OPTIONAL(TDuration, DefaultProcessingTimeout); + FLUENT_SETTING(TDeadLetterPolicySettings, DeadLetterPolicy) + FLUENT_SETTING(TAttributes, Attributes); TConsumerSettings& AddAttribute(const std::string& key, const std::string& value) { @@ -506,6 +695,15 @@ struct TConsumerSettings { return *this; } + TConsumerSettings& StreamingConsumerType() { + ConsumerType_ = EConsumerType::Streaming; + return *this; + } + + TDeadLetterPolicyBuilder BeginDeadLetterPolicy() { + return TDeadLetterPolicyBuilder(*this); + } + TSettings& EndAddConsumer() { return Parent_; }; private: @@ -520,6 +718,8 @@ struct TAlterConsumerSettings { TAlterConsumerSettings(TAlterTopicSettings& parent): Parent_(parent) {} TAlterConsumerSettings(TAlterTopicSettings& parent, const std::string& name) : ConsumerName_(name), Parent_(parent) {} + void SerializeTo(Ydb::Topic::AlterConsumer& proto) const; + FLUENT_SETTING(std::string, ConsumerName); FLUENT_SETTING_OPTIONAL(bool, SetImportant); FLUENT_SETTING_OPTIONAL(TDuration, SetAvailabilityPeriod); @@ -529,6 +729,9 @@ struct TAlterConsumerSettings { FLUENT_SETTING(TAlterAttributes, AlterAttributes); + FLUENT_SETTING_OPTIONAL(TDuration, DefaultProcessingTimeout); + FLUENT_SETTING(TAlterDeadLetterPolicySettings, DeadLetterPolicy); + TAlterConsumerAttributesBuilder BeginAlterAttributes() { return TAlterConsumerAttributesBuilder(*this); } @@ -548,6 +751,10 @@ struct TAlterConsumerSettings { return *this; } + TAlterDeadLetterPolicyBuilder BeginAlterDeadLetterPolicy() { + return TAlterDeadLetterPolicyBuilder(*this); + } + TAlterTopicSettings& EndAlterConsumer() { return Parent_; }; private: @@ -593,13 +800,31 @@ struct TCreateTopicSettings : public TOperationRequestSettings& BeginAddConsumer() { + TConsumerSettings& BeginAddSharedConsumer() { + return BeginAddConsumer(EConsumerType::Shared); + } + + TConsumerSettings& BeginAddStreamingConsumer() { + return BeginAddConsumer(EConsumerType::Streaming); + } + + TConsumerSettings& BeginAddConsumer(EConsumerType consumerType = EConsumerType::Streaming) { Consumers_.push_back({*this}); + Consumers_.back().ConsumerType(consumerType); return Consumers_.back(); } - TConsumerSettings& BeginAddConsumer(const std::string& name) { + TConsumerSettings& BeginAddSharedConsumer(const std::string& name) { + return BeginAddConsumer(name, EConsumerType::Shared); + } + + TConsumerSettings& BeginAddStreamingConsumer(const std::string& name) { + return BeginAddConsumer(name, EConsumerType::Streaming); + } + + TConsumerSettings& BeginAddConsumer(const std::string& name, EConsumerType consumerType = EConsumerType::Streaming) { Consumers_.push_back({*this, name}); + Consumers_.back().ConsumerType(consumerType); return Consumers_.back(); } diff --git a/src/api/protos/ydb_topic.proto b/src/api/protos/ydb_topic.proto index fced1b62fac..40f2c31ea4f 100644 --- a/src/api/protos/ydb_topic.proto +++ b/src/api/protos/ydb_topic.proto @@ -123,8 +123,6 @@ message StreamWriteMessage { // Explicitly request for last sequential number // It may be expensive, if producer wrote to many partitions before. bool get_last_seq_no = 6; - - } // Response to the handshake. @@ -788,6 +786,75 @@ message MultipleWindowsStat { int64 per_day = 3; } + +message DeadLetterPolicyCondition { + uint32 max_processing_attempts = 1; +} + +message AlterDeadLetterPolicyCondition { + optional uint32 set_max_processing_attempts = 1; +} + +message DeleteDeadLetterPolicyAction { +} + +message MoveDeadLetterPolicyAction { + string dead_letter_queue = 1; +} + +message AlterMoveDeadLetterPolicyAction { + optional string set_dead_letter_queue = 1; +} + +message DeadLetterPolicy { + bool enabled = 1; + + DeadLetterPolicyCondition condition = 2; + + oneof action { + DeleteDeadLetterPolicyAction delete_action = 4; + MoveDeadLetterPolicyAction move_action = 5; + } +} + +message AlterDeadLetterPolicy { + optional bool set_enabled = 1; + + optional AlterDeadLetterPolicyCondition alter_condition = 2; + + oneof action { + AlterMoveDeadLetterPolicyAction alter_move_action = 3; + DeleteDeadLetterPolicyAction set_delete_action = 4; + MoveDeadLetterPolicyAction set_move_action = 5; + } +} + +message StreamingConsumerType { +} + +message AlterStreamingConsumerType { +} + +message SharedConsumerType { + // If this is a shared consumer, then the value true means that the order of messages within the same message group will be preserved. + // For streaming consumers, the order of messages is always preserved. + optional bool keep_messages_order = 10; + // The duration of blocking messages for its processing. If the message is not committed during this time, + // it will be returned to the queue and sent for re-processing. + optional google.protobuf.Duration default_processing_timeout = 11; + // The dedad letter policy for this consumer. + optional DeadLetterPolicy dead_letter_policy = 12; +} + +message AlterSharedConsumerType { + // The duration of blocking messages for its processing. If the message is not committed during this time, + // it will be returned to the queue and sent for re-processing. + google.protobuf.Duration set_default_processing_timeout = 10; + + // Change dead letter policy. + AlterDeadLetterPolicy alter_dead_letter_policy = 11; +} + // Consumer description. message Consumer { // Must have valid not empty name as a key. @@ -826,6 +893,11 @@ message Consumer { // Message for this consumer will not expire due to retention for at least `availability_period` if they aren't commited. optional google.protobuf.Duration availability_period = 8; + + oneof consumer_type { + StreamingConsumerType streaming_consumer_type = 9; + SharedConsumerType shared_consumer_type = 10; + } } // Consumer alter description. @@ -854,6 +926,11 @@ message AlterConsumer { google.protobuf.Duration set_availability_period = 7; google.protobuf.Empty reset_availability_period = 8; } + + oneof alter_consumer_type { + AlterStreamingConsumerType alter_streaming_consumer_type = 9; + AlterSharedConsumerType alter_shared_consumer_type = 10; + } } enum AutoPartitioningStrategy { diff --git a/src/client/topic/impl/topic.cpp b/src/client/topic/impl/topic.cpp index 891e89509d6..2f765750c6b 100644 --- a/src/client/topic/impl/topic.cpp +++ b/src/client/topic/impl/topic.cpp @@ -95,6 +95,7 @@ TConsumer::TConsumer(const Ydb::Topic::Consumer& consumer) , Important_(consumer.important()) , AvailabilityPeriod_(ConvertPositiveDuration(consumer.availability_period())) , ReadFrom_(TInstant::Seconds(consumer.read_from().seconds())) + , DeadLetterPolicy_(consumer.shared_consumer_type().dead_letter_policy()) { for (const auto& codec : consumer.supported_codecs().codecs()) { SupportedCodecs_.push_back((ECodec)codec); @@ -102,12 +103,24 @@ TConsumer::TConsumer(const Ydb::Topic::Consumer& consumer) for (const auto& pair : consumer.attributes()) { Attributes_[pair.first] = pair.second; } + + if (consumer.has_shared_consumer_type()) { + ConsumerType_ = EConsumerType::Shared; + KeepMessagesOrder_ = consumer.shared_consumer_type().keep_messages_order(); + DefaultProcessingTimeout_ = TDuration::Seconds(consumer.shared_consumer_type().default_processing_timeout().seconds()); + } else { + ConsumerType_ = EConsumerType::Streaming; + } } const std::string& TConsumer::GetConsumerName() const { return ConsumerName_; } +EConsumerType TConsumer::GetConsumerType() const { + return ConsumerType_; +} + bool TConsumer::GetImportant() const { return Important_; } @@ -128,6 +141,17 @@ const std::map& TConsumer::GetAttributes() const { return Attributes_; } +bool TConsumer::GetKeepMessagesOrder() const { + return KeepMessagesOrder_; +} +TDuration TConsumer::GetDefaultProcessingTimeout() const { + return DefaultProcessingTimeout_; +} + +const TDeadLetterPolicy& TConsumer::GetDeadLetterPolicy() const { + return DeadLetterPolicy_; +} + const TPartitioningSettings& TTopicDescription::GetPartitioningSettings() const { return PartitioningSettings_; } @@ -633,6 +657,59 @@ std::vector> DeserializeConsumers(TSettings& parent } +TDeadLetterPolicyCondition::TDeadLetterPolicyCondition(const Ydb::Topic::DeadLetterPolicyCondition& proto) + : MaxProcessingAttempts_(proto.max_processing_attempts()) +{ +} + +ui32 TDeadLetterPolicyCondition::GetMaxProcessingAttempts() const { + return MaxProcessingAttempts_; +} + +TDeadLetterPolicy::TDeadLetterPolicy(const Ydb::Topic::DeadLetterPolicy& proto) + : Enabled_(proto.enabled()) + , Condition_(proto.condition()) +{ + if (proto.has_delete_action()) { + Action_ = EDeadLetterAction::Delete; + } else if (proto.has_move_action()) { + Action_ = EDeadLetterAction::Move; + DeadLetterQueue_ = proto.move_action().dead_letter_queue(); + } else { + Action_ = EDeadLetterAction::Unspecified; + } +} + +bool TDeadLetterPolicy::GetEnabled() const { + return Enabled_; +} + +const TDeadLetterPolicyCondition& TDeadLetterPolicy::GetCondition() const { + return Condition_; +} + +EDeadLetterAction TDeadLetterPolicy::GetAction() const { + return Action_; +} + +const std::string& TDeadLetterPolicy::GetDeadLetterQueue() const { + return DeadLetterQueue_; +} + +TDeadLetterPolicyConditionSettings::TDeadLetterPolicyConditionSettings(const Ydb::Topic::DeadLetterPolicyCondition& proto) + : MaxProcessingAttempts_(proto.max_processing_attempts()) +{ +} + +TDeadLetterPolicySettings::TDeadLetterPolicySettings(const Ydb::Topic::DeadLetterPolicy& proto) + : Enabled_(proto.enabled()) + , Condition_(proto.condition()) +{ + if (proto.has_move_action()) { + DeadLetterQueue_ = proto.move_action().dead_letter_queue(); + } +} + template TConsumerSettings::TConsumerSettings(TSettings& parent, const Ydb::Topic::Consumer& proto) : ConsumerName_(proto.name()) @@ -640,9 +717,17 @@ TConsumerSettings::TConsumerSettings(TSettings& parent, const Ydb::To , AvailabilityPeriod_(ConvertPositiveDuration(proto.availability_period())) , ReadFrom_(TInstant::Seconds(proto.read_from().seconds())) , SupportedCodecs_(DeserializeCodecs(proto.supported_codecs())) + , KeepMessagesOrder_(proto.shared_consumer_type().keep_messages_order()) + , DefaultProcessingTimeout_(TDuration::Seconds(proto.shared_consumer_type().default_processing_timeout().seconds())) + , DeadLetterPolicy_(proto.shared_consumer_type().dead_letter_policy()) , Attributes_(DeserializeAttributes(proto.attributes())) , Parent_(parent) { + if (proto.has_shared_consumer_type()) { + ConsumerType_ = EConsumerType::Streaming; + } else { + ConsumerType_ = EConsumerType::Streaming; + } } template @@ -658,6 +743,105 @@ void TConsumerSettings::SerializeTo(Ydb::Topic::Consumer& proto) cons proto.mutable_read_from()->set_seconds(ReadFrom_.Seconds()); *proto.mutable_supported_codecs() = SerializeCodecs(SupportedCodecs_); *proto.mutable_attributes() = SerializeAttributes(Attributes_); + + switch (ConsumerType_) { + case EConsumerType::Shared: { + auto* shared = proto.mutable_shared_consumer_type(); + if (KeepMessagesOrder_) { + shared->set_keep_messages_order(KeepMessagesOrder_.value()); + } + if (DefaultProcessingTimeout_) { + shared->mutable_default_processing_timeout()->set_seconds(DefaultProcessingTimeout_.value().Seconds()); + } + if (DeadLetterPolicy_.Enabled_) { + shared->mutable_dead_letter_policy()->set_enabled(DeadLetterPolicy_.Enabled_.value()); + } + if (DeadLetterPolicy_.Condition_.MaxProcessingAttempts_) { + shared->mutable_dead_letter_policy()->mutable_condition()->set_max_processing_attempts( + DeadLetterPolicy_.Condition_.MaxProcessingAttempts_.value()); + } + + switch (DeadLetterPolicy_.Action_) { + case EDeadLetterAction::Move: + shared->mutable_dead_letter_policy()->mutable_move_action()->set_dead_letter_queue( + DeadLetterPolicy_.DeadLetterQueue_.value()); + break; + case EDeadLetterAction::Delete: + shared->mutable_dead_letter_policy()->mutable_delete_action(); + break; + case EDeadLetterAction::Unspecified: + break; + } + + break; + } + case EConsumerType::Streaming: + case EConsumerType::Unspecified: + proto.mutable_streaming_consumer_type(); + break; + } +} + +void TAlterConsumerSettings::SerializeTo(Ydb::Topic::AlterConsumer& proto) const { + proto.set_name(TStringType{ConsumerName_}); + if (SetImportant_) { + proto.set_set_important(*SetImportant_); + } + if (SetAvailabilityPeriod_) { + if (SetAvailabilityPeriod_ != TDuration::Zero()) { + proto.mutable_set_availability_period()->set_seconds(SetAvailabilityPeriod_->Seconds()); + proto.mutable_set_availability_period()->set_nanos((SetAvailabilityPeriod_->MicroSeconds() % 1'000'000) * 1'000); + } else { + proto.mutable_reset_availability_period(); + } + } + if (SetReadFrom_) { + proto.mutable_set_read_from()->set_seconds(SetReadFrom_->Seconds()); + } + if (SetSupportedCodecs_) { + for (const auto& codec : *SetSupportedCodecs_) { + proto.mutable_set_supported_codecs()->add_codecs((static_cast(codec))); + } + } + + for (auto& pair : AlterAttributes_) { + (*proto.mutable_alter_attributes())[pair.first] = pair.second; + } + + if (DefaultProcessingTimeout_) { + proto.mutable_alter_shared_consumer_type()->mutable_set_default_processing_timeout() + ->set_seconds(DefaultProcessingTimeout_.value().Seconds()); + } + + if (DeadLetterPolicy_.Enabled_) { + proto.mutable_alter_shared_consumer_type()->mutable_alter_dead_letter_policy() + ->set_set_enabled(DeadLetterPolicy_.Enabled_.value()); + } + if (DeadLetterPolicy_.Condition_.MaxProcessingAttempts_) { + proto.mutable_alter_shared_consumer_type()->mutable_alter_dead_letter_policy()->mutable_alter_condition() + ->set_set_max_processing_attempts(DeadLetterPolicy_.Condition_.MaxProcessingAttempts_.value()); + } + + if (DeadLetterPolicy_.Action_) { + switch (DeadLetterPolicy_.Action_.value()) { + case EDeadLetterAction::Move: + if (DeadLetterPolicy_.DeadLetterQueue_) { + if (DeadLetterPolicy_.DeadLetterPolicyChanged_) { + proto.mutable_alter_shared_consumer_type()->mutable_alter_dead_letter_policy()->mutable_set_move_action() + ->set_dead_letter_queue(DeadLetterPolicy_.DeadLetterQueue_.value()); + } else { + proto.mutable_alter_shared_consumer_type()->mutable_alter_dead_letter_policy()->mutable_alter_move_action() + ->set_set_dead_letter_queue(DeadLetterPolicy_.DeadLetterQueue_.value()); + } + } + break; + case EDeadLetterAction::Delete: + proto.mutable_alter_shared_consumer_type()->mutable_alter_dead_letter_policy()->mutable_set_delete_action(); + break; + case EDeadLetterAction::Unspecified: + break; + } + } } template struct TConsumerSettings; diff --git a/src/client/topic/impl/topic_impl.h b/src/client/topic/impl/topic_impl.h index 5d19a74ea52..1983d4734d6 100644 --- a/src/client/topic/impl/topic_impl.h +++ b/src/client/topic/impl/topic_impl.h @@ -42,33 +42,6 @@ class TTopicClient::TImpl : public TClientImplCommon { { } - static void ConvertAlterConsumerToProto(const TAlterConsumerSettings& settings, Ydb::Topic::AlterConsumer& consumerProto) { - consumerProto.set_name(TStringType{settings.ConsumerName_}); - if (settings.SetImportant_) - consumerProto.set_set_important(*settings.SetImportant_); - if (settings.SetAvailabilityPeriod_) { - if (settings.SetAvailabilityPeriod_ != TDuration::Zero()) { - consumerProto.mutable_set_availability_period()->set_seconds(settings.SetAvailabilityPeriod_->Seconds()); - consumerProto.mutable_set_availability_period()->set_nanos((settings.SetAvailabilityPeriod_->MicroSeconds() % 1'000'000) * 1'000); - } else { - consumerProto.mutable_reset_availability_period(); - } - } - if (settings.SetReadFrom_) - consumerProto.mutable_set_read_from()->set_seconds(settings.SetReadFrom_->Seconds()); - - if (settings.SetSupportedCodecs_) { - for (const auto& codec : *settings.SetSupportedCodecs_) { - consumerProto.mutable_set_supported_codecs()->add_codecs((static_cast(codec))); - } - } - - for (auto& pair : settings.AlterAttributes_) { - (*consumerProto.mutable_alter_attributes())[pair.first] = pair.second; - } - } - - static Ydb::Topic::CreateTopicRequest MakePropsCreateRequest(const std::string& path, const TCreateTopicSettings& settings) { Ydb::Topic::CreateTopicRequest request = MakeOperationRequest(settings); request.set_path(TStringType{path}); @@ -147,7 +120,7 @@ class TTopicClient::TImpl : public TClientImplCommon { for (const auto& consumer : settings.AlterConsumers_) { Ydb::Topic::AlterConsumer& consumerProto = *request.add_alter_consumers(); - ConvertAlterConsumerToProto(consumer, consumerProto); + consumer.SerializeTo(consumerProto); } if (auto level = std::get_if(&settings.MetricsLevel_)) { diff --git a/src/client/topic/ut/basic_usage_ut.cpp b/src/client/topic/ut/basic_usage_ut.cpp index 38fe6a2d51e..59fdce4993b 100644 --- a/src/client/topic/ut/basic_usage_ut.cpp +++ b/src/client/topic/ut/basic_usage_ut.cpp @@ -107,6 +107,539 @@ Y_UNIT_TEST_SUITE(BasicUsage) { setup.CreateTopic(name, TEST_CONSUMER, 1); } + Y_UNIT_TEST(CreateTopicWithStreamingConsumer) { + TTopicSdkTestSetup setup{TEST_CASE_NAME, TTopicSdkTestSetup::MakeServerSettings(), false}; + + TTopicClient client(setup.MakeDriver()); + + TCreateTopicSettings topics; + topics.BeginAddStreamingConsumer("consumer_name"); + + auto status = client.CreateTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToOneLineString()); + + auto describe = client.DescribeTopic("topic_name").GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(describe.IsSuccess(), describe.GetIssues().ToOneLineString()); + + auto& d = describe.GetTopicDescription(); + UNIT_ASSERT_VALUES_EQUAL(d.GetConsumers().size(), 1); + auto& c = d.GetConsumers()[0]; + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerName(), "consumer_name"); + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerType(), EConsumerType::Streaming); + } + + Y_UNIT_TEST(CreateTopicWithSharedConsumer_MoveDeadLetterPolicy) { + TTopicSdkTestSetup setup{TEST_CASE_NAME, TTopicSdkTestSetup::MakeServerSettings(), false}; + + TTopicClient client(setup.MakeDriver()); + + TCreateTopicSettings topics; + topics.BeginAddConsumer() + .ConsumerName("shared_consumer_name") + .ConsumerType(EConsumerType::Shared) + .DefaultProcessingTimeout(TDuration::Seconds(7)) + .KeepMessagesOrder(true) + .BeginDeadLetterPolicy() + .Enable() + .BeginCondition() + .MaxProcessingAttempts(11) + .EndCondition() + .MoveAction("deadLetterQueue-topic") + .EndDeadLetterPolicy() + .EndAddConsumer(); + + auto status = client.CreateTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToOneLineString()); + + auto describe = client.DescribeTopic("topic_name").GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(describe.IsSuccess(), describe.GetIssues().ToOneLineString()); + + auto& d = describe.GetTopicDescription(); + UNIT_ASSERT_VALUES_EQUAL(d.GetConsumers().size(), 1); + auto& c = d.GetConsumers()[0]; + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerName(), "shared_consumer_name"); + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerType(), EConsumerType::Shared); + UNIT_ASSERT_VALUES_EQUAL(c.GetKeepMessagesOrder(), true); + UNIT_ASSERT_VALUES_EQUAL(c.GetDefaultProcessingTimeout(), TDuration::Seconds(7)); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetEnabled(), true); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetCondition().GetMaxProcessingAttempts(), 11); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetAction(), EDeadLetterAction::Move); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetDeadLetterQueue(), "deadLetterQueue-topic"); + } + + Y_UNIT_TEST(CreateTopicWithSharedConsumer_DeleteDeadLetterPolicy) { + TTopicSdkTestSetup setup{TEST_CASE_NAME, TTopicSdkTestSetup::MakeServerSettings(), false}; + + TTopicClient client(setup.MakeDriver()); + + TCreateTopicSettings topics; + topics.BeginAddSharedConsumer() + .ConsumerName("shared_consumer_name") + .DefaultProcessingTimeout(TDuration::Seconds(7)) + .KeepMessagesOrder(true) + .BeginDeadLetterPolicy() + .Enabled(true) + .DeleteAction() + .BeginCondition() + .MaxProcessingAttempts(11) + .EndCondition() + .EndDeadLetterPolicy() + .EndAddConsumer(); + Cerr << ">>>>> " << topics.Consumers_[0].DeadLetterPolicy_.Action_ << Endl; + + auto status = client.CreateTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToOneLineString()); + + auto describe = client.DescribeTopic("topic_name").GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(describe.IsSuccess(), describe.GetIssues().ToOneLineString()); + + auto& d = describe.GetTopicDescription(); + UNIT_ASSERT_VALUES_EQUAL(d.GetConsumers().size(), 1); + auto& c = d.GetConsumers()[0]; + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerName(), "shared_consumer_name"); + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerType(), EConsumerType::Shared); + UNIT_ASSERT_VALUES_EQUAL(c.GetKeepMessagesOrder(), true); + UNIT_ASSERT_VALUES_EQUAL(c.GetDefaultProcessingTimeout(), TDuration::Seconds(7)); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetEnabled(), true); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetCondition().GetMaxProcessingAttempts(), 11); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetAction(), EDeadLetterAction::Delete); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetDeadLetterQueue(), ""); + } + + Y_UNIT_TEST(CreateTopicWithSharedConsumer_DisabledDeadLetterPolicy) { + TTopicSdkTestSetup setup{TEST_CASE_NAME, TTopicSdkTestSetup::MakeServerSettings(), false}; + + TTopicClient client(setup.MakeDriver()); + + TCreateTopicSettings topics; + topics.BeginAddConsumer() + .ConsumerName("shared_consumer_name") + .ConsumerType(EConsumerType::Shared) + .DefaultProcessingTimeout(TDuration::Seconds(7)) + .KeepMessagesOrder(true) + .EndAddConsumer(); + + auto status = client.CreateTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToOneLineString()); + + auto describe = client.DescribeTopic("topic_name").GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(describe.IsSuccess(), describe.GetIssues().ToOneLineString()); + + auto& d = describe.GetTopicDescription(); + UNIT_ASSERT_VALUES_EQUAL(d.GetConsumers().size(), 1); + auto& c = d.GetConsumers()[0]; + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerName(), "shared_consumer_name"); + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerType(), EConsumerType::Shared); + UNIT_ASSERT_VALUES_EQUAL(c.GetKeepMessagesOrder(), true); + UNIT_ASSERT_VALUES_EQUAL(c.GetDefaultProcessingTimeout(), TDuration::Seconds(7)); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetEnabled(), false); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetAction(), EDeadLetterAction::Unspecified); + } + + Y_UNIT_TEST(CreateTopicWithSharedConsumer_KeepMessagesOrder_False) { + TTopicSdkTestSetup setup{TEST_CASE_NAME, TTopicSdkTestSetup::MakeServerSettings(), false}; + + TTopicClient client(setup.MakeDriver()); + + TCreateTopicSettings topics; + topics.BeginAddSharedConsumer("shared_consumer_name") + .KeepMessagesOrder(false) + .EndAddConsumer(); + + auto status = client.CreateTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToOneLineString()); + + auto describe = client.DescribeTopic("topic_name").GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(describe.IsSuccess(), describe.GetIssues().ToOneLineString()); + + auto& d = describe.GetTopicDescription(); + UNIT_ASSERT_VALUES_EQUAL(d.GetConsumers().size(), 1); + auto& c = d.GetConsumers()[0]; + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerName(), "shared_consumer_name"); + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerType(), EConsumerType::Shared); + UNIT_ASSERT_VALUES_EQUAL(c.GetKeepMessagesOrder(), false); + } + + Y_UNIT_TEST(CreateTopicWithSharedConsumer_KeepMessagesOrder_True) { + TTopicSdkTestSetup setup{TEST_CASE_NAME, TTopicSdkTestSetup::MakeServerSettings(), false}; + + TTopicClient client(setup.MakeDriver()); + + TCreateTopicSettings topics; + topics.BeginAddSharedConsumer("shared_consumer_name") + .KeepMessagesOrder(true) + .EndAddConsumer(); + + auto status = client.CreateTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToOneLineString()); + + auto describe = client.DescribeTopic("topic_name").GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(describe.IsSuccess(), describe.GetIssues().ToOneLineString()); + + auto& d = describe.GetTopicDescription(); + UNIT_ASSERT_VALUES_EQUAL(d.GetConsumers().size(), 1); + auto& c = d.GetConsumers()[0]; + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerName(), "shared_consumer_name"); + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerType(), EConsumerType::Shared); + UNIT_ASSERT_VALUES_EQUAL(c.GetKeepMessagesOrder(), true); + } + + Y_UNIT_TEST(AlterTopicWithSharedConsumer_MoveDeadLetterPolicy) { + TTopicSdkTestSetup setup{TEST_CASE_NAME, TTopicSdkTestSetup::MakeServerSettings(), false}; + + TTopicClient client(setup.MakeDriver()); + + { + TCreateTopicSettings topics; + topics.BeginAddConsumer() + .ConsumerName("shared_consumer_name") + .ConsumerType(EConsumerType::Shared) + .DefaultProcessingTimeout(TDuration::Seconds(7)) + .KeepMessagesOrder(true) + .BeginDeadLetterPolicy() + .Enabled(true) + .MoveAction("deadLetterQueue-topic") + .BeginCondition() + .MaxProcessingAttempts(11) + .EndCondition() + .EndDeadLetterPolicy() + .EndAddConsumer(); + + auto status = client.CreateTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToOneLineString()); + } + + { + TAlterTopicSettings topics; + topics.BeginAlterConsumer() + .ConsumerName("shared_consumer_name") + .DefaultProcessingTimeout(TDuration::Seconds(13)) + .BeginAlterDeadLetterPolicy() + .Enabled(true) + .AlterMoveAction("deadLetterQueue-topic-new") + .BeginCondition() + .MaxProcessingAttempts(17) + .EndCondition() + .EndAlterDeadLetterPolicy() + .EndAlterConsumer(); + auto status = client.AlterTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToOneLineString()); + } + + auto describe = client.DescribeTopic("topic_name").GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(describe.IsSuccess(), describe.GetIssues().ToOneLineString()); + + auto& d = describe.GetTopicDescription(); + UNIT_ASSERT_VALUES_EQUAL(d.GetConsumers().size(), 1); + auto& c = d.GetConsumers()[0]; + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerName(), "shared_consumer_name"); + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerType(), EConsumerType::Shared); + UNIT_ASSERT_VALUES_EQUAL(c.GetKeepMessagesOrder(), true); + UNIT_ASSERT_VALUES_EQUAL(c.GetDefaultProcessingTimeout(), TDuration::Seconds(13)); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetEnabled(), true); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetCondition().GetMaxProcessingAttempts(), 17); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetAction(), EDeadLetterAction::Move); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetDeadLetterQueue(), "deadLetterQueue-topic-new"); + } + + Y_UNIT_TEST(AlterTopicWithSharedConsumer_DisableDeadLetterPolicy) { + TTopicSdkTestSetup setup{TEST_CASE_NAME, TTopicSdkTestSetup::MakeServerSettings(), false}; + + TTopicClient client(setup.MakeDriver()); + + { + TCreateTopicSettings topics; + topics.BeginAddConsumer() + .ConsumerName("shared_consumer_name") + .ConsumerType(EConsumerType::Shared) + .DefaultProcessingTimeout(TDuration::Seconds(7)) + .KeepMessagesOrder(true) + .BeginDeadLetterPolicy() + .Enabled(true) + .MoveAction("deadLetterQueue-topic") + .BeginCondition() + .MaxProcessingAttempts(11) + .EndCondition() + .EndDeadLetterPolicy() + .EndAddConsumer(); + + auto status = client.CreateTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToOneLineString()); + + auto describe = client.DescribeTopic("topic_name").GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(describe.IsSuccess(), describe.GetIssues().ToOneLineString()); + auto& c = describe.GetTopicDescription().GetConsumers()[0]; + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetEnabled(), true); + } + + { + TAlterTopicSettings topics; + topics.BeginAlterConsumer() + .ConsumerName("shared_consumer_name") + .BeginAlterDeadLetterPolicy() + .Enabled(false) + .EndAlterDeadLetterPolicy() + .EndAlterConsumer(); + auto status = client.AlterTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToOneLineString()); + } + + auto describe = client.DescribeTopic("topic_name").GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(describe.IsSuccess(), describe.GetIssues().ToOneLineString()); + + auto& d = describe.GetTopicDescription(); + UNIT_ASSERT_VALUES_EQUAL(d.GetConsumers().size(), 1); + auto& c = d.GetConsumers()[0]; + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerName(), "shared_consumer_name"); + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerType(), EConsumerType::Shared); + UNIT_ASSERT_VALUES_EQUAL(c.GetKeepMessagesOrder(), true); + UNIT_ASSERT_VALUES_EQUAL(c.GetDefaultProcessingTimeout(), TDuration::Seconds(7)); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetEnabled(), false); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetCondition().GetMaxProcessingAttempts(), 11); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetAction(), EDeadLetterAction::Move); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetDeadLetterQueue(), "deadLetterQueue-topic"); + } + + Y_UNIT_TEST(AlterTopicWithSharedConsumer_SetDeleteDeadLetterPolicy) { + TTopicSdkTestSetup setup{TEST_CASE_NAME, TTopicSdkTestSetup::MakeServerSettings(), false}; + + TTopicClient client(setup.MakeDriver()); + + { + TCreateTopicSettings topics; + topics.BeginAddConsumer() + .ConsumerName("shared_consumer_name") + .ConsumerType(EConsumerType::Shared) + .DefaultProcessingTimeout(TDuration::Seconds(7)) + .KeepMessagesOrder(true) + .BeginDeadLetterPolicy() + .Enabled(true) + .MoveAction("deadLetterQueue-topic") + .BeginCondition() + .MaxProcessingAttempts(11) + .EndCondition() + .EndDeadLetterPolicy() + .EndAddConsumer(); + + auto status = client.CreateTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToOneLineString()); + } + + { + TAlterTopicSettings topics; + topics.BeginAlterConsumer() + .ConsumerName("shared_consumer_name") + .BeginAlterDeadLetterPolicy() + .SetDeleteAction() + .EndAlterDeadLetterPolicy() + .EndAlterConsumer(); + auto status = client.AlterTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToOneLineString()); + } + + auto describe = client.DescribeTopic("topic_name").GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(describe.IsSuccess(), describe.GetIssues().ToOneLineString()); + + auto& d = describe.GetTopicDescription(); + UNIT_ASSERT_VALUES_EQUAL(d.GetConsumers().size(), 1); + auto& c = d.GetConsumers()[0]; + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerName(), "shared_consumer_name"); + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerType(), EConsumerType::Shared); + UNIT_ASSERT_VALUES_EQUAL(c.GetKeepMessagesOrder(), true); + UNIT_ASSERT_VALUES_EQUAL(c.GetDefaultProcessingTimeout(), TDuration::Seconds(7)); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetEnabled(), true); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetCondition().GetMaxProcessingAttempts(), 11); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetAction(), EDeadLetterAction::Delete); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetDeadLetterQueue(), ""); + } + + Y_UNIT_TEST(AlterTopicWithSharedConsumer_SetMoveDeadLetterPolicy) { + TTopicSdkTestSetup setup{TEST_CASE_NAME, TTopicSdkTestSetup::MakeServerSettings(), false}; + + TTopicClient client(setup.MakeDriver()); + + { + TCreateTopicSettings topics; + topics.BeginAddConsumer() + .ConsumerName("shared_consumer_name") + .ConsumerType(EConsumerType::Shared) + .DefaultProcessingTimeout(TDuration::Seconds(7)) + .KeepMessagesOrder(true) + .BeginDeadLetterPolicy() + .Enabled(true) + .DeleteAction() + .BeginCondition() + .MaxProcessingAttempts(11) + .EndCondition() + .EndDeadLetterPolicy() + .EndAddConsumer(); + + auto status = client.CreateTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToOneLineString()); + } + + { + TAlterTopicSettings topics; + topics.BeginAlterConsumer() + .ConsumerName("shared_consumer_name") + .BeginAlterDeadLetterPolicy() + .SetMoveAction("dlq-topic") + .EndAlterDeadLetterPolicy() + .EndAlterConsumer(); + auto status = client.AlterTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToOneLineString()); + } + + auto describe = client.DescribeTopic("topic_name").GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(describe.IsSuccess(), describe.GetIssues().ToOneLineString()); + + auto& d = describe.GetTopicDescription(); + UNIT_ASSERT_VALUES_EQUAL(d.GetConsumers().size(), 1); + auto& c = d.GetConsumers()[0]; + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerName(), "shared_consumer_name"); + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerType(), EConsumerType::Shared); + UNIT_ASSERT_VALUES_EQUAL(c.GetKeepMessagesOrder(), true); + UNIT_ASSERT_VALUES_EQUAL(c.GetDefaultProcessingTimeout(), TDuration::Seconds(7)); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetEnabled(), true); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetCondition().GetMaxProcessingAttempts(), 11); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetAction(), EDeadLetterAction::Move); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetDeadLetterQueue(), "dlq-topic"); + } + + Y_UNIT_TEST(AlterTopicWithSharedConsumer_AlterMoveDeadLetterPolicy) { + TTopicSdkTestSetup setup{TEST_CASE_NAME, TTopicSdkTestSetup::MakeServerSettings(), false}; + + TTopicClient client(setup.MakeDriver()); + + { + TCreateTopicSettings topics; + topics.BeginAddConsumer() + .ConsumerName("shared_consumer_name") + .ConsumerType(EConsumerType::Shared) + .DefaultProcessingTimeout(TDuration::Seconds(7)) + .KeepMessagesOrder(true) + .BeginDeadLetterPolicy() + .Enabled(true) + .MoveAction("dlq-topic") + .BeginCondition() + .MaxProcessingAttempts(11) + .EndCondition() + .EndDeadLetterPolicy() + .EndAddConsumer(); + + auto status = client.CreateTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToOneLineString()); + } + + { + TAlterTopicSettings topics; + topics.BeginAlterConsumer() + .ConsumerName("shared_consumer_name") + .BeginAlterDeadLetterPolicy() + .AlterMoveAction("dlq-topic-new") + .EndAlterDeadLetterPolicy() + .EndAlterConsumer(); + auto status = client.AlterTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToOneLineString()); + } + + auto describe = client.DescribeTopic("topic_name").GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(describe.IsSuccess(), describe.GetIssues().ToOneLineString()); + + auto& d = describe.GetTopicDescription(); + UNIT_ASSERT_VALUES_EQUAL(d.GetConsumers().size(), 1); + auto& c = d.GetConsumers()[0]; + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerName(), "shared_consumer_name"); + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerType(), EConsumerType::Shared); + UNIT_ASSERT_VALUES_EQUAL(c.GetKeepMessagesOrder(), true); + UNIT_ASSERT_VALUES_EQUAL(c.GetDefaultProcessingTimeout(), TDuration::Seconds(7)); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetEnabled(), true); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetCondition().GetMaxProcessingAttempts(), 11); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetAction(), EDeadLetterAction::Move); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetDeadLetterQueue(), "dlq-topic-new"); + } + + Y_UNIT_TEST(AlterTopicWithSharedConsumer_DeleteDeadLetterPolicy_AlterMoveDeadLetterPolicy) { + TTopicSdkTestSetup setup{TEST_CASE_NAME, TTopicSdkTestSetup::MakeServerSettings(), false}; + + TTopicClient client(setup.MakeDriver()); + + { + TCreateTopicSettings topics; + topics.BeginAddConsumer() + .ConsumerName("shared_consumer_name") + .ConsumerType(EConsumerType::Shared) + .DefaultProcessingTimeout(TDuration::Seconds(7)) + .KeepMessagesOrder(true) + .BeginDeadLetterPolicy() + .Enabled(true) + .DeleteAction() + .BeginCondition() + .MaxProcessingAttempts(11) + .EndCondition() + .EndDeadLetterPolicy() + .EndAddConsumer(); + + auto status = client.CreateTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToOneLineString()); + } + + { + TAlterTopicSettings topics; + topics.BeginAlterConsumer() + .ConsumerName("shared_consumer_name") + .BeginAlterDeadLetterPolicy() + .AlterMoveAction("dlq-topic-new") + .EndAlterDeadLetterPolicy() + .EndAlterConsumer(); + auto status = client.AlterTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(!status.IsSuccess(), status.GetIssues().ToOneLineString()); + } + + auto describe = client.DescribeTopic("topic_name").GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(describe.IsSuccess(), describe.GetIssues().ToOneLineString()); + + auto& d = describe.GetTopicDescription(); + UNIT_ASSERT_VALUES_EQUAL(d.GetConsumers().size(), 1); + auto& c = d.GetConsumers()[0]; + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerName(), "shared_consumer_name"); + UNIT_ASSERT_VALUES_EQUAL(c.GetConsumerType(), EConsumerType::Shared); + UNIT_ASSERT_VALUES_EQUAL(c.GetKeepMessagesOrder(), true); + UNIT_ASSERT_VALUES_EQUAL(c.GetDefaultProcessingTimeout(), TDuration::Seconds(7)); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetEnabled(), true); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetCondition().GetMaxProcessingAttempts(), 11); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetAction(), EDeadLetterAction::Delete); + UNIT_ASSERT_VALUES_EQUAL(c.GetDeadLetterPolicy().GetDeadLetterQueue(), ""); + } + + Y_UNIT_TEST(AlterDeadLetterPolicy_StreamingConsumer) { + TTopicSdkTestSetup setup{TEST_CASE_NAME, TTopicSdkTestSetup::MakeServerSettings(), false}; + + TTopicClient client(setup.MakeDriver()); + + { + TCreateTopicSettings topics; + topics.BeginAddStreamingConsumer("shared_consumer_name"); + + auto status = client.CreateTopic("topic_name", topics).GetValueSync(); + UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToOneLineString()); + } + + { + TAlterTopicSettings topics; + topics.BeginAlterConsumer() + .ConsumerName("shared_consumer_name") + .BeginAlterDeadLetterPolicy() + .AlterMoveAction("dlq-topic-new") + .EndAlterDeadLetterPolicy() + .EndAlterConsumer(); + auto status = client.AlterTopic("topic_name", topics).GetValueSync(); + auto issue = status.GetIssues().ToOneLineString(); + UNIT_ASSERT_C(!status.IsSuccess(), issue); + UNIT_ASSERT_C(issue.contains("Cannot alter consumer type"), issue); + } + } + Y_UNIT_TEST(ReadWithoutConsumerWithRestarts) { if (EnableDirectRead) { // TODO(qyryq) Enable the test when LOGBROKER-9364 is done. From e15e8e6e00315479a8b78c09669056ca8126b84a Mon Sep 17 00:00:00 2001 From: stanislav_shchetinin Date: Tue, 25 Nov 2025 16:31:00 +0000 Subject: [PATCH 05/19] Add handlers for ExportToFs / ImportFromFs (#28126) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/client/export/export.h | 35 +++++++++++ include/ydb-cpp-sdk/client/import/import.h | 43 +++++++++++++ src/api/protos/ydb_export.proto | 1 + src/api/protos/ydb_import.proto | 1 + src/client/export/export.cpp | 65 +++++++++++++++++++ src/client/import/import.cpp | 72 ++++++++++++++++++++++ src/client/operation/operation.cpp | 12 ++++ 8 files changed, 230 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index fa9765ac068..ffa73bf5a0c 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -f6c8d8cb6c24a823a2ef2885113d4b1fdabc7eaa +ed7c647d380bc46db55840ec067d65fa851d03f2 diff --git a/include/ydb-cpp-sdk/client/export/export.h b/include/ydb-cpp-sdk/client/export/export.h index bbda1459883..2a65b7dd4ca 100644 --- a/include/ydb-cpp-sdk/client/export/export.h +++ b/include/ydb-cpp-sdk/client/export/export.h @@ -126,6 +126,40 @@ class TExportToS3Response : public TOperation { TMetadata Metadata_; }; +/// FS +struct TExportToFsSettings : public TOperationRequestSettings { + using TSelf = TExportToFsSettings; + + struct TItem { + std::string Src; + std::string Dst; + }; + + FLUENT_SETTING(std::string, BasePath); + FLUENT_SETTING_VECTOR(TItem, Item); + FLUENT_SETTING_OPTIONAL(std::string, Description); + FLUENT_SETTING_OPTIONAL(uint32_t, NumberOfRetries); + FLUENT_SETTING_OPTIONAL(std::string, Compression); +}; + +class TExportToFsResponse : public TOperation { +public: + struct TMetadata { + TExportToFsSettings Settings; + EExportProgress Progress; + std::vector ItemsProgress; + }; + +public: + using TOperation::TOperation; + TExportToFsResponse(TStatus&& status, Ydb::Operations::Operation&& operation); + + const TMetadata& Metadata() const; + +private: + TMetadata Metadata_; +}; + class TExportClient { class TImpl; @@ -134,6 +168,7 @@ class TExportClient { NThreading::TFuture ExportToYt(const TExportToYtSettings& settings); NThreading::TFuture ExportToS3(const TExportToS3Settings& settings); + NThreading::TFuture ExportToFs(const TExportToFsSettings& settings); private: std::shared_ptr Impl_; diff --git a/include/ydb-cpp-sdk/client/import/import.h b/include/ydb-cpp-sdk/client/import/import.h index 07c6a48e81f..8d6d6b45486 100644 --- a/include/ydb-cpp-sdk/client/import/import.h +++ b/include/ydb-cpp-sdk/client/import/import.h @@ -139,6 +139,48 @@ class TListObjectsInS3ExportResult : public TStatus { using TAsyncListObjectsInS3ExportResult = NThreading::TFuture; +/// FS +struct TImportFromFsSettings : public TOperationRequestSettings { + using TSelf = TImportFromFsSettings; + + struct TItem { + // Source path. + // Path to the exported table/directory in FS (relative to base_path) + std::string Src; + + // Destination path. + // database path where to import data + std::string Dst; + }; + + FLUENT_SETTING(std::string, BasePath); + FLUENT_SETTING_VECTOR(TItem, Item); + FLUENT_SETTING_OPTIONAL(std::string, Description); + FLUENT_SETTING_OPTIONAL(uint32_t, NumberOfRetries); + FLUENT_SETTING_OPTIONAL(bool, NoACL); + FLUENT_SETTING_OPTIONAL(bool, SkipChecksumValidation); +}; + +class TImportFromFsResponse : public TOperation { +public: + struct TMetadata { + TImportFromFsSettings Settings; + EImportProgress Progress; + std::vector ItemsProgress; + }; + +public: + using TOperation::TOperation; + TImportFromFsResponse(TStatus&& status, Ydb::Operations::Operation&& operation); + + const TMetadata& Metadata() const; + +private: + TMetadata Metadata_; +}; + +using TAsyncImportFromFsResponse = NThreading::TFuture; + /// Data struct TImportYdbDumpDataSettings : public TOperationRequestSettings { using TSelf = TImportYdbDumpDataSettings; @@ -162,6 +204,7 @@ class TImportClient { TImportClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); TAsyncImportFromS3Response ImportFromS3(const TImportFromS3Settings& settings); + TAsyncImportFromFsResponse ImportFromFs(const TImportFromFsSettings& settings); TAsyncListObjectsInS3ExportResult ListObjectsInS3Export(const TListObjectsInS3ExportSettings& settings, std::int64_t pageSize = 0, const std::string& pageToken = {}); diff --git a/src/api/protos/ydb_export.proto b/src/api/protos/ydb_export.proto index 91512305f4e..6deda814dcc 100644 --- a/src/api/protos/ydb_export.proto +++ b/src/api/protos/ydb_export.proto @@ -195,6 +195,7 @@ message ExportToFsSettings { // Base path on FS where to write export // Path to the mounted directory in the case of NFS + // Must be an absolute path // Example: /mnt/exports string base_path = 1 [(required) = true]; diff --git a/src/api/protos/ydb_import.proto b/src/api/protos/ydb_import.proto index d247a4715e0..96afd5f141a 100644 --- a/src/api/protos/ydb_import.proto +++ b/src/api/protos/ydb_import.proto @@ -142,6 +142,7 @@ message ImportFromFsSettings { // Base path on FS where the export is located // Path to the mounted directory in the case of NFS + // Must be an absolute path // Example: /mnt/exports string base_path = 1 [(required) = true]; diff --git a/src/client/export/export.cpp b/src/client/export/export.cpp index 45496052608..cd713852d36 100644 --- a/src/client/export/export.cpp +++ b/src/client/export/export.cpp @@ -109,6 +109,36 @@ const TExportToS3Response::TMetadata& TExportToS3Response::Metadata() const { return Metadata_; } +/// FS +TExportToFsResponse::TExportToFsResponse(TStatus&& status, Ydb::Operations::Operation&& operation) + : TOperation(std::move(status), std::move(operation)) +{ + ExportToFsMetadata metadata; + GetProto().metadata().UnpackTo(&metadata); + + // settings + Metadata_.Settings.BasePath(metadata.settings().base_path()); + + for (const auto& item : metadata.settings().items()) { + Metadata_.Settings.AppendItem({item.source_path(), item.destination_path()}); + } + + Metadata_.Settings.Description(metadata.settings().description()); + Metadata_.Settings.NumberOfRetries(metadata.settings().number_of_retries()); + + if (!metadata.settings().compression().empty()) { + Metadata_.Settings.Compression(metadata.settings().compression()); + } + + // progress + Metadata_.Progress = TProtoAccessor::FromProto(metadata.progress()); + Metadata_.ItemsProgress = ItemsProgressFromProto(metadata.items_progress()); +} + +const TExportToFsResponse::TMetadata& TExportToFsResponse::Metadata() const { + return Metadata_; +} + //////////////////////////////////////////////////////////////////////////////// class TExportClient::TImpl : public TClientImplCommon { @@ -136,6 +166,15 @@ class TExportClient::TImpl : public TClientImplCommon { TRpcRequestSettings::Make(settings)); } + TFuture ExportToFs(ExportToFsRequest&& request, + const TExportToFsSettings& settings) + { + return RunOperation( + std::move(request), + &V1::ExportService::Stub::AsyncExportToFs, + TRpcRequestSettings::Make(settings)); + } + }; //////////////////////////////////////////////////////////////////////////////// @@ -221,5 +260,31 @@ TFuture TExportClient::ExportToS3(const TExportToS3Settings return Impl_->ExportToS3(std::move(request), settings); } +TFuture TExportClient::ExportToFs(const TExportToFsSettings& settings) { + auto request = MakeOperationRequest(settings); + + request.mutable_settings()->set_base_path(TStringType{settings.BasePath_}); + + for (const auto& item : settings.Item_) { + auto& protoItem = *request.mutable_settings()->mutable_items()->Add(); + protoItem.set_source_path(TStringType{item.Src}); + protoItem.set_destination_path(TStringType{item.Dst}); + } + + if (settings.Description_) { + request.mutable_settings()->set_description(TStringType{settings.Description_.value()}); + } + + if (settings.NumberOfRetries_) { + request.mutable_settings()->set_number_of_retries(settings.NumberOfRetries_.value()); + } + + if (settings.Compression_) { + request.mutable_settings()->set_compression(TStringType{settings.Compression_.value()}); + } + + return Impl_->ExportToFs(std::move(request), settings); +} + } // namespace NExport } // namespace NYdb diff --git a/src/client/import/import.cpp b/src/client/import/import.cpp index a53c780ae54..42b0aa25c95 100644 --- a/src/client/import/import.cpp +++ b/src/client/import/import.cpp @@ -87,6 +87,40 @@ const TImportFromS3Response::TMetadata& TImportFromS3Response::Metadata() const return Metadata_; } +/// FS +TImportFromFsResponse::TImportFromFsResponse(TStatus&& status, Ydb::Operations::Operation&& operation) + : TOperation(std::move(status), std::move(operation)) +{ + ImportFromFsMetadata metadata; + GetProto().metadata().UnpackTo(&metadata); + + // settings + Metadata_.Settings.BasePath(metadata.settings().base_path()); + + for (const auto& item : metadata.settings().items()) { + Metadata_.Settings.AppendItem({item.source_path(), item.destination_path()}); + } + + Metadata_.Settings.Description(metadata.settings().description()); + Metadata_.Settings.NumberOfRetries(metadata.settings().number_of_retries()); + + if (metadata.settings().no_acl()) { + Metadata_.Settings.NoACL(metadata.settings().no_acl()); + } + + if (metadata.settings().skip_checksum_validation()) { + Metadata_.Settings.SkipChecksumValidation(metadata.settings().skip_checksum_validation()); + } + + // progress + Metadata_.Progress = TProtoAccessor::FromProto(metadata.progress()); + Metadata_.ItemsProgress = ItemsProgressFromProto(metadata.items_progress()); +} + +const TImportFromFsResponse::TMetadata& TImportFromFsResponse::Metadata() const { + return Metadata_; +} + TListObjectsInS3ExportResult::TListObjectsInS3ExportResult(TStatus&& status, const Ydb::Import::ListObjectsInS3ExportResult& proto) : TStatus(std::move(status)) , Proto_(std::make_unique(proto)) @@ -165,6 +199,13 @@ class TImportClient::TImpl : public TClientImplCommon { TRpcRequestSettings::Make(settings)); } + TAsyncImportFromFsResponse ImportFromFs(ImportFromFsRequest&& request, const TImportFromFsSettings& settings) { + return RunOperation( + std::move(request), + &V1::ImportService::Stub::AsyncImportFromFs, + TRpcRequestSettings::Make(settings)); + } + TAsyncListObjectsInS3ExportResult ListObjectsInS3Export(ListObjectsInS3ExportRequest&& request, const TListObjectsInS3ExportSettings& settings) { auto promise = NThreading::NewPromise(); @@ -277,6 +318,37 @@ TAsyncImportFromS3Response TImportClient::ImportFromS3(const TImportFromS3Settin return Impl_->ImportFromS3(std::move(request), settings); } +TAsyncImportFromFsResponse TImportClient::ImportFromFs(const TImportFromFsSettings& settings) { + auto request = MakeOperationRequest(settings); + Ydb::Import::ImportFromFsSettings& settingsProto = *request.mutable_settings(); + + settingsProto.set_base_path(TStringType{settings.BasePath_}); + + for (const auto& item : settings.Item_) { + auto& protoItem = *settingsProto.mutable_items()->Add(); + protoItem.set_source_path(item.Src); + protoItem.set_destination_path(item.Dst); + } + + if (settings.Description_) { + settingsProto.set_description(TStringType{settings.Description_.value()}); + } + + if (settings.NumberOfRetries_) { + settingsProto.set_number_of_retries(settings.NumberOfRetries_.value()); + } + + if (settings.NoACL_) { + settingsProto.set_no_acl(settings.NoACL_.value()); + } + + if (settings.SkipChecksumValidation_) { + settingsProto.set_skip_checksum_validation(settings.SkipChecksumValidation_.value()); + } + + return Impl_->ImportFromFs(std::move(request), settings); +} + TAsyncListObjectsInS3ExportResult TImportClient::ListObjectsInS3Export(const TListObjectsInS3ExportSettings& settings, std::int64_t pageSize, const std::string& pageToken) { auto request = MakeOperationRequest(settings); Ydb::Import::ListObjectsInS3ExportSettings& settingsProto = *request.mutable_settings(); diff --git a/src/client/operation/operation.cpp b/src/client/operation/operation.cpp index 7c4a80b8e24..96dd7d7b2a2 100644 --- a/src/client/operation/operation.cpp +++ b/src/client/operation/operation.cpp @@ -61,6 +61,18 @@ NThreading::TFuture> TOperationC return List("import/s3", pageSize, pageToken); } +template NThreading::TFuture TOperationClient::Get(const TOperation::TOperationId& id); +template <> +NThreading::TFuture> TOperationClient::List(std::uint64_t pageSize, const std::string& pageToken) { + return List("export/fs", pageSize, pageToken); +} + +template NThreading::TFuture TOperationClient::Get(const TOperation::TOperationId& id); +template <> +NThreading::TFuture> TOperationClient::List(std::uint64_t pageSize, const std::string& pageToken) { + return List("import/fs", pageSize, pageToken); +} + template NThreading::TFuture TOperationClient::Get(const TOperation::TOperationId& id); template <> NThreading::TFuture> TOperationClient::List(std::uint64_t pageSize, const std::string& pageToken) { From 6105a824bff062811b675999f314acd44470d4a8 Mon Sep 17 00:00:00 2001 From: Evgenik2 Date: Tue, 25 Nov 2025 16:31:06 +0000 Subject: [PATCH 06/19] Collect cluster state change api (#28622) --- .github/last_commit.txt | 2 +- src/api/protos/ydb_monitoring.proto | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index ffa73bf5a0c..800ca097cce 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -ed7c647d380bc46db55840ec067d65fa851d03f2 +5142fbd25fe9858314f429c8d0150d826801090b diff --git a/src/api/protos/ydb_monitoring.proto b/src/api/protos/ydb_monitoring.proto index 58e6e2c2164..11c3643a6c9 100644 --- a/src/api/protos/ydb_monitoring.proto +++ b/src/api/protos/ydb_monitoring.proto @@ -246,6 +246,12 @@ message ClusterStateResponse { Ydb.Operations.Operation operation = 1; } +message ClusterStateResultBlock { + string name = 1; + string content = 2; +} + message ClusterStateResult { - string result = 1; + string result = 1; // deprecated + repeated ClusterStateResultBlock blocks = 2; } From 6bf40ab4c0ef7750ed758323886764bcf2c0ee70 Mon Sep 17 00:00:00 2001 From: ubyte Date: Tue, 25 Nov 2025 16:31:12 +0000 Subject: [PATCH 07/19] use NanoSecondsOfSecond method to fill duration field (#26786) --- .github/last_commit.txt | 2 +- src/client/topic/impl/topic.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 800ca097cce..829bff491b2 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -5142fbd25fe9858314f429c8d0150d826801090b +68f5b7781abf828e57c853c69dd591477036ffd2 diff --git a/src/client/topic/impl/topic.cpp b/src/client/topic/impl/topic.cpp index 2f765750c6b..610a4b921a6 100644 --- a/src/client/topic/impl/topic.cpp +++ b/src/client/topic/impl/topic.cpp @@ -736,7 +736,7 @@ void TConsumerSettings::SerializeTo(Ydb::Topic::Consumer& proto) cons proto.set_important(Important_); if (AvailabilityPeriod_ != TDuration::Zero()) { proto.mutable_availability_period()->set_seconds(AvailabilityPeriod_.Seconds()); - proto.mutable_availability_period()->set_nanos((AvailabilityPeriod_.MicroSeconds() % 1'000'000) * 1'000); + proto.mutable_availability_period()->set_nanos(AvailabilityPeriod_.NanoSecondsOfSecond()); } else { proto.clear_availability_period(); } @@ -790,7 +790,7 @@ void TAlterConsumerSettings::SerializeTo(Ydb::Topic::AlterConsumer& proto) const if (SetAvailabilityPeriod_) { if (SetAvailabilityPeriod_ != TDuration::Zero()) { proto.mutable_set_availability_period()->set_seconds(SetAvailabilityPeriod_->Seconds()); - proto.mutable_set_availability_period()->set_nanos((SetAvailabilityPeriod_->MicroSeconds() % 1'000'000) * 1'000); + proto.mutable_set_availability_period()->set_nanos(SetAvailabilityPeriod_->NanoSecondsOfSecond()); } else { proto.mutable_reset_availability_period(); } From 1169675fd93aa557a7baf0e6fe135254880049b3 Mon Sep 17 00:00:00 2001 From: ubyte Date: Tue, 25 Nov 2025 16:31:19 +0000 Subject: [PATCH 08/19] fix java_package name in ydb/public/api/protos/draft (#28750) --- .github/last_commit.txt | 2 +- src/api/grpc/draft/ydb_sqs_topic_v1.proto | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 829bff491b2..1a817bb1458 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -68f5b7781abf828e57c853c69dd591477036ffd2 +e973211bb36be49ac7f9c15f15ec8232a970b75a diff --git a/src/api/grpc/draft/ydb_sqs_topic_v1.proto b/src/api/grpc/draft/ydb_sqs_topic_v1.proto index 8a575ec7e45..0a71e63786f 100644 --- a/src/api/grpc/draft/ydb_sqs_topic_v1.proto +++ b/src/api/grpc/draft/ydb_sqs_topic_v1.proto @@ -5,7 +5,7 @@ package Ydb.SqsTopic.V1; import "src/api/protos/draft/ymq.proto"; -option java_package = "com.yandex.ydb.sqs_topic.V1."; +option java_package = "com.yandex.ydb.sqs_topic.v1"; service SqsTopicService { rpc SqsTopicGetQueueUrl(Ydb.Ymq.V1.GetQueueUrlRequest) returns (Ydb.Ymq.V1.GetQueueUrlResponse); From f256410cadd5bd2fe4d589e719a10251a25a73ea Mon Sep 17 00:00:00 2001 From: qyryq Date: Tue, 25 Nov 2025 16:31:25 +0000 Subject: [PATCH 09/19] Topic SDK: minor changes (#28282) --- .github/last_commit.txt | 2 +- src/client/topic/impl/direct_reader.cpp | 23 ++----------- src/client/topic/impl/direct_reader.h | 1 - src/client/topic/impl/read_session_impl.ipp | 37 ++++++++++----------- 4 files changed, 20 insertions(+), 43 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 1a817bb1458..01ddc6a0015 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -e973211bb36be49ac7f9c15f15ec8232a970b75a +a32ae5c23f531f6adf0e04790f35fe229f927e17 diff --git a/src/client/topic/impl/direct_reader.cpp b/src/client/topic/impl/direct_reader.cpp index c58d7de977a..3c85ec02025 100644 --- a/src/client/topic/impl/direct_reader.cpp +++ b/src/client/topic/impl/direct_reader.cpp @@ -212,7 +212,7 @@ void TDirectReadSessionManager::StartPartitionSession(TDirectReadPartitionSessio Locations.emplace(partitionSession.PartitionSessionId, partitionSession.Location); } -// Delete a partition session from a node (TDirectReadSession), and if there are no more +// Delete partition session from the node (TDirectReadSession), and if there are no more // partition sessions on the node, drop connection to it. void TDirectReadSessionManager::DeletePartitionSession(TPartitionSessionId partitionSessionId, TNodeSessionsMap::iterator it) { LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "DeletePartitionSession " << partitionSessionId); @@ -228,7 +228,7 @@ void TDirectReadSessionManager::DeletePartitionSession(TPartitionSessionId parti NodeSessions.erase(it); } } else { - LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "DeletePartitionSession " << partitionSessionId << " not found in NodeSessions"); + LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "DeletePartitionSession " << partitionSessionId << " could not LockShared"); } if (directReadSessionContextPtr) { directReadSessionContextPtr->Cancel(); @@ -278,25 +278,6 @@ void TDirectReadSessionManager::UpdatePartitionSession(TPartitionSessionId parti }); } -TDirectReadSessionContextPtr TDirectReadSessionManager::ErasePartitionSession(TPartitionSessionId partitionSessionId) { - LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "ErasePartitionSession " << partitionSessionId); - - auto locIt = Locations.find(partitionSessionId); - Y_ABORT_UNLESS(locIt != Locations.end()); - auto nodeId = locIt->second.GetNodeId(); - - auto sessionIt = NodeSessions.find(nodeId); - Y_ABORT_UNLESS(sessionIt != NodeSessions.end()); - TDirectReadSessionContextPtr directReadSessionContextPtr = sessionIt->second; - - // Still need to Cancel the TCallbackContext. - LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "ErasePartitionSession " << partitionSessionId << " erase"); - NodeSessions.erase(sessionIt); - Locations.erase(partitionSessionId); - - return directReadSessionContextPtr; -} - void TDirectReadSessionManager::StopPartitionSession(TPartitionSessionId partitionSessionId) { auto locIt = Locations.find(partitionSessionId); if (locIt == Locations.end()) { diff --git a/src/client/topic/impl/direct_reader.h b/src/client/topic/impl/direct_reader.h index 2a25df56b4c..fbcb4147a9e 100644 --- a/src/client/topic/impl/direct_reader.h +++ b/src/client/topic/impl/direct_reader.h @@ -232,7 +232,6 @@ class TDirectReadSessionManager { void StartPartitionSession(TDirectReadPartitionSession&&); void UpdatePartitionSession(TPartitionSessionId, TPartitionId, TPartitionLocation); - TDirectReadSessionContextPtr ErasePartitionSession(TPartitionSessionId); void StopPartitionSession(TPartitionSessionId); // Update LastDirectReadId in the partition session object. diff --git a/src/client/topic/impl/read_session_impl.ipp b/src/client/topic/impl/read_session_impl.ipp index f688c9c8d81..aec1a0e2385 100644 --- a/src/client/topic/impl/read_session_impl.ipp +++ b/src/client/topic/impl/read_session_impl.ipp @@ -1372,7 +1372,7 @@ inline void TSingleClusterReadSessionImpl::StopPartitionSessionImpl( // partitionStream->ConfirmDestroy(); TClientMessage req; auto& released = *req.mutable_stop_partition_session_response(); - released.set_partition_session_id(partitionStream->GetAssignId()); + released.set_partition_session_id(partitionSessionId); WriteToProcessorImpl(std::move(req)); PartitionStreams.erase(partitionSessionId); pushRes = EventsQueue->PushEvent( @@ -1417,7 +1417,7 @@ inline void TSingleClusterReadSessionImpl::OnDirectReadDone( auto& partitionStream = partitionStreamIt->second; partitionStream->SetNextDirectReadId(response.direct_read_id() + 1); - auto stopPartitionSession = [&](){ + auto stopIfGotLastResponse = [&](){ // After we get a StopPartitionSessionRequest(graceful=true), LastDirectReadId is defined. // In this case we're waiting for the DirectReadResponse(direct_read_id=LastDirectReadId) and then close the subsession. @@ -1429,20 +1429,19 @@ inline void TSingleClusterReadSessionImpl::OnDirectReadDone( if (!response.has_partition_data() || response.partition_data().batches_size() == 0) { // Sometimes the server might send an empty DirectReadResponse with a non-zero bytes_size, that we should take into account. - stopPartitionSession(); + stopIfGotLastResponse(); ReadSizeBudget += response.bytes_size(); ReadSizeServerDelta -= response.bytes_size(); WaitingReadResponse = false; ContinueReadingDataImpl(); - return; + } else { + Ydb::Topic::StreamReadMessage::ReadResponse r; + r.set_bytes_size(response.bytes_size()); + auto* data = r.add_partition_data(); + data->Swap(response.mutable_partition_data()); + OnReadDoneImpl(std::move(r), deferred); + stopIfGotLastResponse(); } - - Ydb::Topic::StreamReadMessage::ReadResponse r; - r.set_bytes_size(response.bytes_size()); - auto* data = r.add_partition_data(); - data->Swap(response.mutable_partition_data()); - OnReadDoneImpl(std::move(r), deferred); - stopPartitionSession(); } } @@ -1586,18 +1585,18 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( ) { Y_ABORT_UNLESS(Lock.IsLocked()); - LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "StopPartitionSessionRequest " << msg.DebugString()); - auto partitionSessionId = msg.partition_session_id(); - auto partitionStreamIt = PartitionStreams.find(partitionSessionId); + + LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "StopPartitionSessionRequest " << msg.DebugString()); + if (partitionStreamIt == PartitionStreams.end()) { LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Server wants us to stop partition session id=" << partitionSessionId << ", but it's not found"); return; } - TIntrusivePtr> partitionStream = partitionStreamIt->second; + auto partitionStream = partitionStreamIt->second; if (IsDirectRead() && msg.graceful()) { // Keep reading DirectReadResponses until we get the one with direct_read_id == last_direct_read_id. @@ -1610,13 +1609,11 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( // 1. We have received the last DirectReadResponse. // 2. We have received the StopPartitionSessionRequest(graceful=true) after we received a corresponding DirectReadResponse. // This is the second case. - StopPartitionSessionImpl(partitionStreamIt->second, true, deferred); + StopPartitionSessionImpl(partitionStream, true, deferred); } - - return; + } else { + StopPartitionSessionImpl(partitionStream, msg.graceful(), deferred); } - - StopPartitionSessionImpl(partitionStreamIt->second, msg.graceful(), deferred); } template <> From cbd929be2786a54c5cba80f618cd7ad73386dcd2 Mon Sep 17 00:00:00 2001 From: Evgenik2 Date: Tue, 25 Nov 2025 16:31:32 +0000 Subject: [PATCH 10/19] Fix time stamp in collect cluster metrics (#29018) --- .github/last_commit.txt | 2 +- src/api/protos/ydb_monitoring.proto | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 01ddc6a0015..7ad179ed414 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -a32ae5c23f531f6adf0e04790f35fe229f927e17 +41750541146a8ebcd7a375d268b34b9288904a5f diff --git a/src/api/protos/ydb_monitoring.proto b/src/api/protos/ydb_monitoring.proto index 11c3643a6c9..03bb95362da 100644 --- a/src/api/protos/ydb_monitoring.proto +++ b/src/api/protos/ydb_monitoring.proto @@ -5,6 +5,7 @@ package Ydb.Monitoring; option java_package = "com.yandex.ydb.monitoring"; option java_outer_classname = "MonitoringProtos"; +import "google/protobuf/timestamp.proto"; import "src/api/protos/ydb_operation.proto"; message StatusFlag { @@ -249,6 +250,7 @@ message ClusterStateResponse { message ClusterStateResultBlock { string name = 1; string content = 2; + google.protobuf.Timestamp timestamp = 3; } message ClusterStateResult { From e08f56605cc5af78d1ba9c8e4bebbc6a36181e06 Mon Sep 17 00:00:00 2001 From: mregrock Date: Tue, 25 Nov 2025 16:31:38 +0000 Subject: [PATCH 11/19] Add TestShard public api (#28717) --- .github/last_commit.txt | 2 +- src/api/grpc/draft/ydb_test_shard_v1.proto | 15 ++++ src/api/protos/draft/ydb_test_shard.proto | 99 ++++++++++++++++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 src/api/grpc/draft/ydb_test_shard_v1.proto create mode 100644 src/api/protos/draft/ydb_test_shard.proto diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 7ad179ed414..2f0e779e507 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -41750541146a8ebcd7a375d268b34b9288904a5f +419aa07dce13f16e82f9e1e91a5d41b97ebdfc96 diff --git a/src/api/grpc/draft/ydb_test_shard_v1.proto b/src/api/grpc/draft/ydb_test_shard_v1.proto new file mode 100644 index 00000000000..02d61d01af4 --- /dev/null +++ b/src/api/grpc/draft/ydb_test_shard_v1.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package Ydb.TestShard.V1; + +import "src/api/protos/draft/ydb_test_shard.proto"; + +service TestShardService { + // Creates TestShard tablet(s) + rpc CreateTestShard(Ydb.TestShard.CreateTestShardRequest) + returns (Ydb.TestShard.CreateTestShardResponse); + + // Deletes TestShard tablet(s) + rpc DeleteTestShard(Ydb.TestShard.DeleteTestShardRequest) + returns (Ydb.TestShard.DeleteTestShardResponse); +} diff --git a/src/api/protos/draft/ydb_test_shard.proto b/src/api/protos/draft/ydb_test_shard.proto new file mode 100644 index 00000000000..1ef8a662c42 --- /dev/null +++ b/src/api/protos/draft/ydb_test_shard.proto @@ -0,0 +1,99 @@ +syntax = "proto3"; + +package Ydb.TestShard; + +option java_package = "com.yandex.ydb.test_shard"; + +import "src/api/protos/ydb_operation.proto"; + +message CreateTestShardRequest { + Ydb.Operations.OperationParams operation_params = 1; + + // Database path + string database = 2; + + // Base index for tablet ID generation + // Same owner_idx = same tablets (idempotent duplicate detection) + uint64 owner_idx = 3; + + // Storage pool names for tablet channels (optional) + // If not provided, uses first 3 storage pools from the database domain + repeated string channels = 4; + + // Number of tablets to create + uint32 count = 5; + + // TestShard initialization config in YAML format + // + // Structure: + // workload - Load generation patterns (sizes, write rate, restarts, request types) + // limits - Resource constraints (data volume, concurrency) + // timing - Temporal behavior (startup delay, period resets, synchronization) + // validation - External validation server (optional) + // tracing - Request tracing for debugging (optional) + // + // ```yaml + // workload: + // sizes: # Value size distribution (weighted random) + // - {weight: 10, min: 100, max: 1000, inline: false} + // - {weight: 1, min: 10000, max: 100000, inline: false} + // write: # Write rate (Poisson), list format + // - frequency: 100.0 # Writes per second (weight optional, default 1) + // max_interval_ms: 100 + // restart: # Automatic restarts (Poisson), list format + // - frequency: 0.01 # Restarts per second (weight optional, default 1) + // max_interval_ms: 120000 + // patch_fraction_ppm: 100000 # PATCH vs PUT ratio (parts per million) + // + // limits: + // data: + // min: 1000000 # Resume writing below this (bytes) + // max: 10000000 # Stop writing above this (bytes) + // concurrency: + // writes: 50 # Max concurrent write requests + // reads: 10 # Max concurrent read requests + // + // timing: + // delay_start: 5 # Seconds before load starts + // reset_on_full: true # Reset write timer on max concurrency + // stall_counter: 1000 # Barrier sync every N requests + // + // validation: # Optional external validation + // server: "host:9999" # Validation server address + // after_bytes: 1000000 # Start validation after N bytes + // + // tracing: # Optional request tracing + // put_fraction_ppm: 1000 # Trace fraction (parts per million) + // verbosity: 15 # Trace detail level (0-15) + // ``` + string config = 6; +} + +message CreateTestShardResponse { + Ydb.Operations.Operation operation = 1; +} + +message CreateTestShardResult { + // Created tablet IDs + repeated uint64 tablet_ids = 1; +} + +message DeleteTestShardRequest { + Ydb.Operations.OperationParams operation_params = 1; + + // Database path + string database = 2; + + // Owner index identifying tablets to delete + uint64 owner_idx = 3; + + // Number of consecutive tablets to delete starting from owner_idx + // If not specified, deletes all tablets with this owner_idx + uint32 count = 4; +} + +message DeleteTestShardResponse { + Ydb.Operations.Operation operation = 1; +} + +message DeleteTestShardResult {} From e38dc64e4a0d46a1816dc0cad09c33a2a97fb781 Mon Sep 17 00:00:00 2001 From: Bulat Date: Tue, 25 Nov 2025 16:31:45 +0000 Subject: [PATCH 12/19] [C++ SDK] Added deadline propagation to session pool (#28158) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/client/query/client.h | 4 +- src/client/common_client/impl/client.h | 4 +- .../internal/grpc_connections/actions.cpp | 25 +++++-- .../impl/internal/grpc_connections/actions.h | 27 ++++++-- .../grpc_connections/grpc_connections.cpp | 50 +++++++------- .../grpc_connections/grpc_connections.h | 5 +- src/client/impl/session/session_pool.cpp | 69 ++++++++++++------- src/client/impl/session/session_pool.h | 31 +++++---- .../impl/write_session_impl.cpp | 2 +- src/client/query/client.cpp | 26 +++++-- src/client/query/impl/exec_query.cpp | 10 +-- src/client/table/impl/client_session.h | 8 ++- src/client/table/impl/table_client.cpp | 50 +++++++++----- src/client/table/impl/table_client.h | 5 +- src/client/table/table.cpp | 10 ++- src/client/topic/impl/write_session_impl.cpp | 2 +- src/library/grpc/client/grpc_client_low.cpp | 17 +++-- src/library/grpc/client/grpc_client_low.h | 15 ++-- src/library/time/time.cpp | 34 ++++++--- src/library/time/time.h | 6 ++ 21 files changed, 262 insertions(+), 140 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 2f0e779e507..c1e79ac7873 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -419aa07dce13f16e82f9e1e91a5d41b97ebdfc96 +9bbd29fb8a7d7a06303ce368a18641108124a8e1 diff --git a/include/ydb-cpp-sdk/client/query/client.h b/include/ydb-cpp-sdk/client/query/client.h index 88cd4962a53..6a441dbcac5 100644 --- a/include/ydb-cpp-sdk/client/query/client.h +++ b/include/ydb-cpp-sdk/client/query/client.h @@ -27,7 +27,9 @@ namespace NYdb::inline V3 { namespace NYdb::inline V3::NQuery { struct TCreateSessionSettings : public TSimpleRequestSettings { - TCreateSessionSettings(); + TCreateSessionSettings() { + ClientTimeout(TDuration::Seconds(5)); + } }; using TAsyncCreateSessionResult = NThreading::TFuture; diff --git a/src/client/common_client/impl/client.h b/src/client/common_client/impl/client.h index 1ce6be40afa..0e3c1e451b8 100644 --- a/src/client/common_client/impl/client.h +++ b/src/client/common_client/impl/client.h @@ -53,7 +53,7 @@ class TClientImplCommon return DbDriverState_->DiscoveryCompleted(); } - void ScheduleTask(const std::function& fn, TDeadline::Duration timeout) override { + void ScheduleTask(const std::function& fn, TDeadline::Duration delay) override { std::weak_ptr weak = this->shared_from_this(); auto cbGuard = [weak, fn]() { auto strongClient = weak.lock(); @@ -61,7 +61,7 @@ class TClientImplCommon fn(); } }; - Connections_->ScheduleOneTimeTask(std::move(cbGuard), timeout); + Connections_->ScheduleDelayedTask(std::move(cbGuard), delay); } protected: diff --git a/src/client/impl/internal/grpc_connections/actions.cpp b/src/client/impl/internal/grpc_connections/actions.cpp index e7f8d44ad0e..acb55aa469c 100644 --- a/src/client/impl/internal/grpc_connections/actions.cpp +++ b/src/client/impl/internal/grpc_connections/actions.cpp @@ -10,11 +10,8 @@ namespace NYdb::inline V3 { constexpr TDeadline::Duration MAX_DEFERRED_CALL_DELAY = 10s; // The max delay between GetOperation calls for one operation -TSimpleCbResult::TSimpleCbResult( - TSimpleCb&& cb, - TGRpcConnectionsImpl* connections, - std::shared_ptr context) - : TGenericCbHolder(std::move(cb), connections, std::move(context)) +TSimpleCbResult::TSimpleCbResult(TSimpleCb&& cb) + : UserResponseCb_(std::move(cb)) { } void TSimpleCbResult::Process(void*) { @@ -111,4 +108,22 @@ void TPeriodicAction::OnError() { UserResponseCb_(std::move(issues), EStatus::CLIENT_INTERNAL_ERROR); } +TDelayedAction::TDelayedAction( + TDelayedCb&& userCb, + TGRpcConnectionsImpl* connection, + std::shared_ptr context, + TDeadline deadline) + : TAlarmActionBase(std::move(userCb), connection, std::move(context)) +{ + Deadline_ = deadline; +} + +void TDelayedAction::OnAlarm() { + UserResponseCb_(true); +} + +void TDelayedAction::OnError() { + UserResponseCb_(false); +} + } // namespace NYdb diff --git a/src/client/impl/internal/grpc_connections/actions.h b/src/client/impl/internal/grpc_connections/actions.h index 1c5b46be8df..4c64a32d7ba 100644 --- a/src/client/impl/internal/grpc_connections/actions.h +++ b/src/client/impl/internal/grpc_connections/actions.h @@ -26,6 +26,7 @@ struct TPlainStatus; template using TResponseCb = std::function; using TDeferredOperationCb = std::function; +using TDelayedCb = std::function; template class TGenericCbHolder { @@ -175,16 +176,14 @@ class TResult std::multimap Metadata_; }; -class TSimpleCbResult - : public TGenericCbHolder - , public IObjectInQueue +class TSimpleCbResult : public IObjectInQueue { public: - TSimpleCbResult( - TSimpleCb&& cb, - TGRpcConnectionsImpl* connections, - std::shared_ptr context); + TSimpleCbResult(TSimpleCb&& cb); void Process(void*) override; + +private: + TSimpleCb UserResponseCb_; }; //////////////////////////////////////////////////////////////////////////////// @@ -233,4 +232,18 @@ class TPeriodicAction TDeadline::Duration Period_; }; +class TDelayedAction + : public TAlarmActionBase +{ +public: + TDelayedAction( + TDelayedCb&& userCb, + TGRpcConnectionsImpl* connection, + std::shared_ptr context, + TDeadline deadline); + + void OnAlarm() override; + void OnError() override; +}; + } // namespace NYdb diff --git a/src/client/impl/internal/grpc_connections/grpc_connections.cpp b/src/client/impl/internal/grpc_connections/grpc_connections.cpp index 8c6cde808e8..496e183788e 100644 --- a/src/client/impl/internal/grpc_connections/grpc_connections.cpp +++ b/src/client/impl/internal/grpc_connections/grpc_connections.cpp @@ -219,34 +219,38 @@ void TGRpcConnectionsImpl::AddPeriodicTask(TPeriodicCb&& cb, TDeadline::Duration } } -void TGRpcConnectionsImpl::ScheduleOneTimeTask(TSimpleCb&& fn, TDeadline::Duration timeout) { - auto cbLow = [this, fn = std::move(fn)](NYdb::NIssue::TIssues&&, EStatus status) mutable { - if (status != EStatus::SUCCESS) { - return false; - } - - std::shared_ptr context; - - if (!TryCreateContext(context)) { - // Shutting down, fn must handle it - fn(); - } else { - // Enqueue to user pool - auto resp = new TSimpleCbResult( - std::move(fn), - this, - std::move(context)); - EnqueueResponse(resp); +void TGRpcConnectionsImpl::ScheduleDelayedTask(TSimpleCb&& fn, TDeadline deadline) { + auto cbLow = [this, fn = std::move(fn)](bool ok) mutable { + if (!ok) { + return; } - return false; + // Enqueue to user pool + auto resp = new TSimpleCbResult(std::move(fn)); + EnqueueResponse(resp); }; - if (timeout > TDeadline::Duration::zero()) { - AddPeriodicTask(std::move(cbLow), timeout); - } else { - cbLow(NYdb::NIssue::TIssues(), EStatus::SUCCESS); + std::shared_ptr context; + if (!TryCreateContext(context)) { + cbLow(false); + return; + } + + if (deadline <= TDeadline::Now()) { + cbLow(true); + return; } + + auto action = MakeIntrusive( + std::move(cbLow), + this, + std::move(context), + deadline); + action->Start(); +} + +void TGRpcConnectionsImpl::ScheduleDelayedTask(TSimpleCb&& fn, TDeadline::Duration delay) { + ScheduleDelayedTask(std::move(fn), TDeadline::AfterDuration(delay)); } NThreading::TFuture TGRpcConnectionsImpl::ScheduleFuture( diff --git a/src/client/impl/internal/grpc_connections/grpc_connections.h b/src/client/impl/internal/grpc_connections/grpc_connections.h index e5b44a6f862..d219d70d19e 100644 --- a/src/client/impl/internal/grpc_connections/grpc_connections.h +++ b/src/client/impl/internal/grpc_connections/grpc_connections.h @@ -46,7 +46,10 @@ class TGRpcConnectionsImpl ~TGRpcConnectionsImpl(); void AddPeriodicTask(TPeriodicCb&& cb, TDeadline::Duration period) override; - void ScheduleOneTimeTask(TSimpleCb&& fn, TDeadline::Duration timeout); + + void ScheduleDelayedTask(TSimpleCb&& fn, TDeadline deadline); + void ScheduleDelayedTask(TSimpleCb&& fn, TDeadline::Duration delay); + NThreading::TFuture ScheduleFuture( TDuration timeout, IQueueClientContextPtr token = nullptr diff --git a/src/client/impl/session/session_pool.cpp b/src/client/impl/session/session_pool.cpp index e91861a4a97..0984685f8da 100644 --- a/src/client/impl/session/session_pool.cpp +++ b/src/client/impl/session/session_pool.cpp @@ -14,7 +14,7 @@ namespace NSessionPool { using namespace NThreading; -constexpr ui64 KEEP_ALIVE_RANDOM_FRACTION = 4; +constexpr std::uint64_t KEEP_ALIVE_RANDOM_FRACTION = 4; static const TStatus CLIENT_RESOURCE_EXHAUSTED_ACTIVE_SESSION_LIMIT = TStatus( TPlainStatus( EStatus::CLIENT_RESOURCE_EXHAUSTED, @@ -33,10 +33,10 @@ TStatus GetStatus(const TStatus& status) { TDuration RandomizeThreshold(TDuration duration) { TDuration::TValue value = duration.GetValue(); if (KEEP_ALIVE_RANDOM_FRACTION) { - const i64 randomLimit = value / KEEP_ALIVE_RANDOM_FRACTION; + const std::int64_t randomLimit = value / KEEP_ALIVE_RANDOM_FRACTION; if (randomLimit < 2) return duration; - value += static_cast(RandomNumber(randomLimit)); + value += static_cast(RandomNumber(randomLimit)); } return TDuration::FromValue(value); } @@ -53,18 +53,17 @@ bool IsSessionCloseRequested(const TStatus& status) { return false; } -TSessionPool::TWaitersQueue::TWaitersQueue(ui32 maxQueueSize, TDuration maxWaitSessionTimeout) +TSessionPool::TWaitersQueue::TWaitersQueue(std::uint32_t maxQueueSize) : MaxQueueSize_(maxQueueSize) - , MaxWaitSessionTimeout_(maxWaitSessionTimeout) { } -bool TSessionPool::TWaitersQueue::TryPush(std::unique_ptr& p) { +IGetSessionCtx* TSessionPool::TWaitersQueue::TryPush(std::unique_ptr& p) { if (Waiters_.size() < MaxQueueSize_) { - Waiters_.insert(std::make_pair(TInstant::Now(), std::move(p))); - return true; + auto it = Waiters_.insert(std::make_pair(p->GetDeadline(), std::move(p))); + return it->second.get(); } - return false; + return nullptr; } std::unique_ptr TSessionPool::TWaitersQueue::TryGet() { @@ -77,11 +76,12 @@ std::unique_ptr TSessionPool::TWaitersQueue::TryGet() { return result; } -void TSessionPool::TWaitersQueue::GetOld(TInstant now, std::vector>& oldWaiters) { +void TSessionPool::TWaitersQueue::GetOld(TDeadline deadline, std::vector>& oldWaiters) { auto it = Waiters_.begin(); while (it != Waiters_.end()) { - if (now < it->first + MaxWaitSessionTimeout_) + if (deadline < it->first) { break; + } oldWaiters.emplace_back(std::move(it->second)); @@ -89,12 +89,12 @@ void TSessionPool::TWaitersQueue::GetOld(TInstant now, std::vector ctx) if (MaxActiveSessions_ == 0 || ActiveSessions_ < MaxActiveSessions_) { IncrementActiveCounterUnsafe(); - } else if (WaitersQueue_.TryPush(ctx)) { + } else if (auto* ctxPtr = WaitersQueue_.TryPush(ctx)) { sessionSource = TSessionSource::Waiter; + ctxPtr->ScheduleOnDeadlineWaiterCleanup(); } else { sessionSource = TSessionSource::Error; } @@ -150,7 +151,7 @@ void TSessionPool::GetSession(std::unique_ptr ctx) } if (sessionSource == TSessionSource::Waiter) { - // Nothing to do here + // ctxPtr->ScheduleOnDeadlineWaiterCleanup() is called after TryPush } else if (sessionSource == TSessionSource::Error) { FakeSessionsCounter_.Inc(); ctx->ReplyError(CLIENT_RESOURCE_EXHAUSTED_ACTIVE_SESSION_LIMIT); @@ -194,6 +195,22 @@ bool TSessionPool::CheckAndFeedWaiterNewSession(bool active) { return true; } +void TSessionPool::ClearOldWaiters() { + std::lock_guard guard(Mtx_); + + std::vector> oldWaiters; + WaitersQueue_.GetOld(TDeadline::Now(), oldWaiters); + + for (auto& waiter : oldWaiters) { + FakeSessionsCounter_.Inc(); + waiter->ReplyError(CLIENT_RESOURCE_EXHAUSTED_ACTIVE_SESSION_LIMIT); + } + + if (!oldWaiters.empty()) { + UpdateStats(); + } +} + bool TSessionPool::ReturnSession(TKqpSessionCommon* impl, bool active) { // Do not call ReplySessionToUser under the session pool lock std::unique_ptr getSessionCtx; @@ -267,23 +284,25 @@ TPeriodicCb TSessionPool::CreatePeriodicTask(std::weak_ptr weakC // moreover it is unsafe to touch this ptr! return false; } else { - auto keepAliveBatchSize = PERIODIC_ACTION_BATCH_SIZE; + auto sessionCountToProcess = PERIODIC_ACTION_BATCH_SIZE; std::vector> sessionsToTouch; - sessionsToTouch.reserve(keepAliveBatchSize); + sessionsToTouch.reserve(sessionCountToProcess); std::vector> sessionsToDelete; - sessionsToDelete.reserve(keepAliveBatchSize); + sessionsToDelete.reserve(sessionCountToProcess); std::vector> waitersToReplyError; - waitersToReplyError.reserve(keepAliveBatchSize); - const auto now = TInstant::Now(); + waitersToReplyError.reserve(sessionCountToProcess); + const auto now = TDeadline::Now(); + const auto nowUtil = TInstant::Now(); { std::lock_guard guard(Mtx_); { auto& sessions = Sessions_; auto it = sessions.begin(); - while (it != sessions.end() && keepAliveBatchSize--) { - if (now < it->second->GetTimeToTouchFast()) + while (it != sessions.end() && sessionCountToProcess--) { + if (nowUtil < it->second->GetTimeToTouchFast()) { break; + } if (deletePredicate(it->second.get(), sessions.size())) { it->second->UpdateServerCloseHandler(nullptr); @@ -329,16 +348,16 @@ TPeriodicCb TSessionPool::CreatePeriodicTask(std::weak_ptr weakC return periodicCb; } -i64 TSessionPool::GetActiveSessions() const { +std::int64_t TSessionPool::GetActiveSessions() const { std::lock_guard guard(Mtx_); return ActiveSessions_; } -i64 TSessionPool::GetActiveSessionsLimit() const { +std::int64_t TSessionPool::GetActiveSessionsLimit() const { return MaxActiveSessions_; } -i64 TSessionPool::GetCurrentPoolSize() const { +std::int64_t TSessionPool::GetCurrentPoolSize() const { std::lock_guard guard(Mtx_); return Sessions_.size(); } diff --git a/src/client/impl/session/session_pool.h b/src/client/impl/session/session_pool.h index 93f5fc409b3..2499515fd9d 100644 --- a/src/client/impl/session/session_pool.h +++ b/src/client/impl/session/session_pool.h @@ -17,11 +17,13 @@ class IGetSessionCtx : private TNonCopyable { virtual void ReplySessionToUser(TKqpSessionCommon* session) = 0; virtual void ReplyError(TStatus status) = 0; virtual void ReplyNewSession() = 0; + virtual void ScheduleOnDeadlineWaiterCleanup() = 0; + virtual TDeadline GetDeadline() const = 0; }; //How often run session pool keep alive check constexpr TDeadline::Duration PERIODIC_ACTION_INTERVAL = std::chrono::seconds(5); -constexpr TDuration MAX_WAIT_SESSION_TIMEOUT = TDuration::Seconds(5); // Max time to wait session +constexpr TDeadline::Duration MAX_WAIT_SESSION_TIMEOUT = std::chrono::seconds(5); // Max time to wait session constexpr std::uint64_t PERIODIC_ACTION_BATCH_SIZE = 10; // Max number of tasks to perform during one interval TStatus GetStatus(const TOperation& operation); @@ -86,24 +88,23 @@ class TSessionPool : public IServerCloseHandler { private: class TWaitersQueue { public: - TWaitersQueue(ui32 maxQueueSize, TDuration maxWaitSessionTimeout=MAX_WAIT_SESSION_TIMEOUT); + TWaitersQueue(std::uint32_t maxQueueSize); // returns true and gets ownership if queue size less than limit // otherwise returns false and doesn't not touch ctx - bool TryPush(std::unique_ptr& p); + IGetSessionCtx* TryPush(std::unique_ptr& p); std::unique_ptr TryGet(); - void GetOld(TInstant now, std::vector>& oldWaiters); - ui32 Size() const; + void GetOld(TDeadline deadline, std::vector>& oldWaiters); + std::uint32_t Size() const; private: - const ui32 MaxQueueSize_; - const TDuration MaxWaitSessionTimeout_; - std::multimap> Waiters_; + const std::uint32_t MaxQueueSize_; + std::multimap> Waiters_; }; public: using TKeepAliveCmd = std::function; using TDeletePredicate = std::function; - TSessionPool(ui32 maxActiveSessions); + TSessionPool(std::uint32_t maxActiveSessions); // Extracts session from pool or creates new one ising given ctx void GetSession(std::unique_ptr ctx); @@ -115,10 +116,12 @@ class TSessionPool : public IServerCloseHandler { // too feed it bool CheckAndFeedWaiterNewSession(bool active); + void ClearOldWaiters(); + TPeriodicCb CreatePeriodicTask(std::weak_ptr weakClient, TKeepAliveCmd&& cmd, TDeletePredicate&& predicate); - i64 GetActiveSessions() const; - i64 GetActiveSessionsLimit() const; - i64 GetCurrentPoolSize() const; + std::int64_t GetActiveSessions() const; + std::int64_t GetActiveSessionsLimit() const; + std::int64_t GetCurrentPoolSize() const; void DecrementActiveCounter(); void IncrementActiveCounterUnsafe(); @@ -137,8 +140,8 @@ class TSessionPool : public IServerCloseHandler { std::multimap> Sessions_; TWaitersQueue WaitersQueue_; - i64 ActiveSessions_; - const ui32 MaxActiveSessions_; + std::int64_t ActiveSessions_; + const std::uint32_t MaxActiveSessions_; NSdkStats::TSessionCounter ActiveSessionsCounter_; NSdkStats::TSessionCounter InPoolSessionsCounter_; NSdkStats::TSessionCounter SessionWaiterCounter_; diff --git a/src/client/persqueue_public/impl/write_session_impl.cpp b/src/client/persqueue_public/impl/write_session_impl.cpp index 9e0ff2c7c66..e090309c60b 100644 --- a/src/client/persqueue_public/impl/write_session_impl.cpp +++ b/src/client/persqueue_public/impl/write_session_impl.cpp @@ -167,7 +167,7 @@ void TWriteSessionImpl::DoCdsRequest(TDuration delay) { INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings)); // TODO: make client timeout setting }; - Connections->ScheduleOneTimeTask(std::move(cdsRequestCall), TDeadline::SafeDurationCast(delay)); + Connections->ScheduleDelayedTask(std::move(cdsRequestCall), TDeadline::SafeDurationCast(delay)); return; } } diff --git a/src/client/query/client.cpp b/src/client/query/client.cpp index 66c4d998e7e..ed5122c327f 100644 --- a/src/client/query/client.cpp +++ b/src/client/query/client.cpp @@ -33,10 +33,6 @@ NYdb::NRetry::TRetryOperationSettings GetRetrySettings(TDuration timeout, bool i .MaxTimeout(timeout); } -TCreateSessionSettings::TCreateSessionSettings() { - ClientTimeout_ = TDuration::Seconds(5); -}; - static void SetTxSettings(const TTxSettings& txSettings, Ydb::Query::TransactionSettings* proto) { switch (txSettings.GetMode()) { @@ -190,6 +186,7 @@ class TQueryClient::TImpl: public TClientImplCommon, public TAsyncCommitTransactionResult CommitTransaction(const std::string& txId, const NYdb::NQuery::TCommitTxSettings& settings, const TSession& session) { using namespace Ydb::Query; + auto rpcSettings = TRpcRequestSettings::Make(settings, session.SessionImpl_->GetEndpointKey()); auto request = MakeRequest(); request.set_session_id(TStringType{session.GetId()}); request.set_tx_id(TStringType{txId}); @@ -220,7 +217,7 @@ class TQueryClient::TImpl: public TClientImplCommon, public responseCb, &V1::QueryService::Stub::AsyncCommitTransaction, DbDriverState_, - TRpcRequestSettings::Make(settings, session.SessionImpl_->GetEndpointKey())); + rpcSettings); return promise.GetFuture(); } @@ -229,6 +226,7 @@ class TQueryClient::TImpl: public TClientImplCommon, public const TBeginTxSettings& settings, const TSession& session) { using namespace Ydb::Query; + auto rpcSettings = TRpcRequestSettings::Make(settings, session.SessionImpl_->GetEndpointKey()); auto request = MakeRequest(); request.set_session_id(TStringType{session.GetId()}); SetTxSettings(txSettings, request.mutable_tx_settings()); @@ -261,7 +259,7 @@ class TQueryClient::TImpl: public TClientImplCommon, public responseCb, &V1::QueryService::Stub::AsyncBeginTransaction, DbDriverState_, - TRpcRequestSettings::Make(settings, session.SessionImpl_->GetEndpointKey())); + rpcSettings); return promise.GetFuture(); } @@ -456,13 +454,27 @@ class TQueryClient::TImpl: public TClientImplCommon, public }); } + void ScheduleOnDeadlineWaiterCleanup() override { + Client->Connections_->ScheduleDelayedTask( + [client = Client]() { + client->SessionPool_.ClearOldWaiters(); + }, + GetDeadline() + ); + } + + TDeadline GetDeadline() const override { + return std::min(RpcSettings.Deadline, TDeadline::AfterDuration(NSessionPool::MAX_WAIT_SESSION_TIMEOUT)); + } + private: void ScheduleReply(TCreateSessionResult val) { Promise.SetValue(std::move(val)); } + NThreading::TPromise Promise; std::shared_ptr Client; - TRpcRequestSettings RpcSettings; + const TRpcRequestSettings RpcSettings; }; auto ctx = std::make_unique(shared_from_this(), rpcSettings); diff --git a/src/client/query/impl/exec_query.cpp b/src/client/query/impl/exec_query.cpp index d107083aeb4..f6707fce1cf 100644 --- a/src/client/query/impl/exec_query.cpp +++ b/src/client/query/impl/exec_query.cpp @@ -285,6 +285,11 @@ class TExecQueryInternal { const std::string& query, const TTxControl& txControl, const ::google::protobuf::Map* params, const TExecuteQuerySettings& settings, const std::optional& session) { + auto rpcSettings = TRpcRequestSettings::Make(settings); + if (session.has_value()) { + rpcSettings.PreferredEndpoint = TEndpointKey(GetNodeIdFromSession(session->GetId())); + } + auto request = MakeRequest(); request.set_exec_mode(::Ydb::Query::ExecMode(settings.ExecMode_)); request.set_stats_mode(::Ydb::Query::StatsMode(settings.StatsMode_)); @@ -349,11 +354,6 @@ class TExecQueryInternal { auto promise = NewPromise>(); - auto rpcSettings = TRpcRequestSettings::Make(settings); - if (session.has_value()) { - rpcSettings.PreferredEndpoint = TEndpointKey(GetNodeIdFromSession(session->GetId())); - } - connections->StartReadStream< Ydb::Query::V1::QueryService, Ydb::Query::ExecuteQueryRequest, diff --git a/src/client/table/impl/client_session.h b/src/client/table/impl/client_session.h index 968a06ff6e8..1bb897a4407 100644 --- a/src/client/table/impl/client_session.h +++ b/src/client/table/impl/client_session.h @@ -1,9 +1,14 @@ #pragma once +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + #include +#include + #include #include -#include #include @@ -54,6 +59,7 @@ class TSession::TImpl : public TKqpSessionCommon { NThreading::TPromise& promise, std::shared_ptr client, const TCreateSessionSettings& settings, + const TRpcRequestSettings& rpcSettings, ui32 counter, bool needUpdateActiveSessionCounter); private: diff --git a/src/client/table/impl/table_client.cpp b/src/client/table/impl/table_client.cpp index 771db4bd070..f07e93ebe70 100644 --- a/src/client/table/impl/table_client.cpp +++ b/src/client/table/impl/table_client.cpp @@ -78,8 +78,8 @@ NThreading::TFuture TTableClient::TImpl::Stop() { return Drain(); } -void TTableClient::TImpl::ScheduleTaskUnsafe(std::function&& fn, TDeadline::Duration timeout) { - Connections_->ScheduleOneTimeTask(std::move(fn), timeout); +void TTableClient::TImpl::ScheduleTaskUnsafe(std::function&& fn, TDeadline::Duration delay) { + Connections_->ScheduleDelayedTask(std::move(fn), delay); } void TTableClient::TImpl::StartPeriodicSessionPoolTask() { @@ -286,14 +286,18 @@ void TTableClient::TImpl::StartPeriodicHostScanTask() { } TAsyncCreateSessionResult TTableClient::TImpl::GetSession(const TCreateSessionSettings& settings) { - using namespace NSessionPool; + auto rpcSettings = TRpcRequestSettings::Make(settings); + rpcSettings.Header.push_back({NYdb::YDB_CLIENT_CAPABILITIES, NYdb::YDB_CLIENT_CAPABILITY_SESSION_BALANCER}); - class TTableClientGetSessionCtx : public IGetSessionCtx { + class TTableClientGetSessionCtx : public NSessionPool::IGetSessionCtx { public: - TTableClientGetSessionCtx(std::shared_ptr client, TDuration clientTimeout) + TTableClientGetSessionCtx(std::shared_ptr client, + const TCreateSessionSettings& createSessionSettings, + const TRpcRequestSettings& rpcSettings) : Promise(NewPromise()) , Client(client) - , ClientTimeout(clientTimeout) + , CreateSessionSettings(createSessionSettings) + , RpcSettings(rpcSettings) {} TAsyncCreateSessionResult GetFuture() { @@ -321,10 +325,21 @@ TAsyncCreateSessionResult TTableClient::TImpl::GetSession(const TCreateSessionSe } void ReplyNewSession() override { - TCreateSessionSettings settings; - settings.ClientTimeout(ClientTimeout); - const auto& sessionResult = Client->CreateSession(settings, false); - sessionResult.Subscribe(TSession::TImpl::GetSessionInspector(Promise, Client, settings, 0, true)); + const auto& sessionResult = Client->CreateSession(CreateSessionSettings, RpcSettings, false); + sessionResult.Subscribe(TSession::TImpl::GetSessionInspector(Promise, Client, CreateSessionSettings, RpcSettings, 0, true)); + } + + void ScheduleOnDeadlineWaiterCleanup() override { + Client->Connections_->ScheduleDelayedTask( + [client = Client]() { + client->SessionPool_.ClearOldWaiters(); + }, + GetDeadline() + ); + } + + TDeadline GetDeadline() const override { + return std::min(RpcSettings.Deadline, TDeadline::AfterDuration(NSessionPool::MAX_WAIT_SESSION_TIMEOUT)); } private: @@ -334,12 +349,14 @@ TAsyncCreateSessionResult TTableClient::TImpl::GetSession(const TCreateSessionSe promise.SetValue(std::move(val)); }, TDeadline::Duration::zero()); } + NThreading::TPromise Promise; std::shared_ptr Client; - const TDuration ClientTimeout; + const TCreateSessionSettings CreateSessionSettings; + const TRpcRequestSettings RpcSettings; }; - auto ctx = std::make_unique(shared_from_this(), settings.ClientTimeout_); + auto ctx = std::make_unique(shared_from_this(), settings, rpcSettings); auto future = ctx->GetFuture(); SessionPool_.GetSession(std::move(ctx)); return future; @@ -357,15 +374,14 @@ i64 TTableClient::TImpl::GetCurrentPoolSize() const { return SessionPool_.GetCurrentPoolSize(); } -TAsyncCreateSessionResult TTableClient::TImpl::CreateSession(const TCreateSessionSettings& settings, bool standalone, - std::string preferredLocation) +TAsyncCreateSessionResult TTableClient::TImpl::CreateSession(const TCreateSessionSettings& settings, + const TRpcRequestSettings& rpcSettings, + bool standalone) { auto request = MakeOperationRequest(settings); auto createSessionPromise = NewPromise(); auto self = shared_from_this(); - auto rpcSettings = TRpcRequestSettings::Make(settings, TEndpointKey(preferredLocation, 0)); - rpcSettings.Header.push_back({NYdb::YDB_CLIENT_CAPABILITIES, NYdb::YDB_CLIENT_CAPABILITY_SESSION_BALANCER}); auto createSessionExtractor = [createSessionPromise, self, standalone] (google::protobuf::Any* any, TPlainStatus status) mutable { @@ -395,8 +411,6 @@ TAsyncCreateSessionResult TTableClient::TImpl::CreateSession(const TCreateSessio INITIAL_DEFERRED_CALL_DELAY, rpcSettings); - std::weak_ptr state = DbDriverState_; - return createSessionPromise.GetFuture(); } diff --git a/src/client/table/impl/table_client.h b/src/client/table/impl/table_client.h index e014fabc0a8..68a20de3d9c 100644 --- a/src/client/table/impl/table_client.h +++ b/src/client/table/impl/table_client.h @@ -55,8 +55,9 @@ class TTableClient::TImpl: public TClientImplCommon, public i64 GetActiveSessionCount() const; i64 GetActiveSessionsLimit() const; i64 GetCurrentPoolSize() const; - TAsyncCreateSessionResult CreateSession(const TCreateSessionSettings& settings, bool standalone, - std::string preferredLocation = std::string()); + TAsyncCreateSessionResult CreateSession(const TCreateSessionSettings& settings, + const TRpcRequestSettings& rpcSettings, + bool standalone); TAsyncKeepAliveResult KeepAlive(const TSession::TImpl* session, const TKeepAliveSettings& settings); TFuture CreateTable(Ydb::Table::CreateTableRequest&& request, const TCreateTableSettings& settings); diff --git a/src/client/table/table.cpp b/src/client/table/table.cpp index 3b00f587551..884b69141c5 100644 --- a/src/client/table/table.cpp +++ b/src/client/table/table.cpp @@ -1430,18 +1430,20 @@ TSessionInspectorFn TSession::TImpl::GetSessionInspector( NThreading::TPromise& promise, std::shared_ptr client, const TCreateSessionSettings& settings, + const TRpcRequestSettings& rpcSettings, ui32 counter, bool needUpdateActiveSessionCounter) { - return [promise, client, settings, counter, needUpdateActiveSessionCounter](TAsyncCreateSessionResult future) mutable { + return [promise, client, settings, rpcSettings, counter, needUpdateActiveSessionCounter](TAsyncCreateSessionResult future) mutable { Y_ASSERT(future.HasValue()); auto session = future.ExtractValue(); if (IsSessionStatusRetriable(session) && counter < client->GetSessionRetryLimit()) { counter++; - client->CreateSession(settings, false) + client->CreateSession(settings, rpcSettings, false) .Subscribe(GetSessionInspector( promise, client, settings, + rpcSettings, counter, needUpdateActiveSessionCounter) ); @@ -1461,7 +1463,9 @@ TTableClient::TTableClient(const TDriver& driver, const TClientSettings& setting TAsyncCreateSessionResult TTableClient::CreateSession(const TCreateSessionSettings& settings) { // Returns standalone session - return Impl_->CreateSession(settings, true); + auto rpcSettings = TRpcRequestSettings::Make(settings); + rpcSettings.Header.push_back({NYdb::YDB_CLIENT_CAPABILITIES, NYdb::YDB_CLIENT_CAPABILITY_SESSION_BALANCER}); + return Impl_->CreateSession(settings, rpcSettings, true); } TAsyncCreateSessionResult TTableClient::GetSession(const TCreateSessionSettings& settings) { diff --git a/src/client/topic/impl/write_session_impl.cpp b/src/client/topic/impl/write_session_impl.cpp index 0f31c09fb5d..7c2b0bca1fb 100644 --- a/src/client/topic/impl/write_session_impl.cpp +++ b/src/client/topic/impl/write_session_impl.cpp @@ -296,7 +296,7 @@ void TWriteSessionImpl::ConnectToPreferredPartitionLocation(const TDuration& del context); }; - Connections->ScheduleOneTimeTask(std::move(callback), TDeadline::SafeDurationCast(delay)); + Connections->ScheduleDelayedTask(std::move(callback), TDeadline::SafeDurationCast(delay)); } void TWriteSessionImpl::OnDescribePartition(const TStatus& status, const Ydb::Topic::DescribePartitionResult& proto, const NYdbGrpc::IQueueClientContextPtr& describePartitionContext) diff --git a/src/library/grpc/client/grpc_client_low.cpp b/src/library/grpc/client/grpc_client_low.cpp index d23ef4d3087..a8fa3c820cc 100644 --- a/src/library/grpc/client/grpc_client_low.cpp +++ b/src/library/grpc/client/grpc_client_low.cpp @@ -626,7 +626,18 @@ grpc_socket_mutator* NImpl::CreateGRpcKeepAliveSocketMutator(const TTcpKeepAlive return nullptr; } -gpr_timespec DurationToTimespec(const NYdb::TDeadline::Duration& duration) noexcept { +} + +grpc::TimePoint::TimePoint(const NYdb::TDeadline& deadline) + : time_(DeadlineToTimespec(deadline)) +{ +} + +gpr_timespec grpc::TimePoint::raw_time() const noexcept { + return time_; +} + +gpr_timespec grpc::TimePoint::DurationToTimespec(const NYdb::TDeadline::Duration& duration) noexcept { const auto secs = std::chrono::floor(duration); if (duration == NYdb::TDeadline::Duration::max() || secs.count() >= gpr_inf_future(GPR_CLOCK_MONOTONIC).tv_sec) { return gpr_inf_future(GPR_TIMESPAN); @@ -643,10 +654,8 @@ gpr_timespec DurationToTimespec(const NYdb::TDeadline::Duration& duration) noexc return t; } -gpr_timespec DeadlineToTimespec(const NYdb::TDeadline& deadline) { +gpr_timespec grpc::TimePoint::DeadlineToTimespec(const NYdb::TDeadline& deadline) { gpr_timespec t = DurationToTimespec(deadline.GetTimePoint() != NYdb::TDeadline::TimePoint::max() ? deadline.GetTimePoint() - NYdb::TDeadline::Clock::now() : NYdb::TDeadline::Duration::max()); return gpr_convert_clock_type(t, GPR_CLOCK_MONOTONIC); } - -} diff --git a/src/library/grpc/client/grpc_client_low.h b/src/library/grpc/client/grpc_client_low.h index 1b190087f86..32fff617aee 100644 --- a/src/library/grpc/client/grpc_client_low.h +++ b/src/library/grpc/client/grpc_client_low.h @@ -1412,23 +1412,18 @@ class TGRpcClientLow std::mutex JoinMutex_; }; -gpr_timespec DurationToTimespec(const NYdb::TDeadline::Duration& duration) noexcept; - -gpr_timespec DeadlineToTimespec(const NYdb::TDeadline& deadline); - } template <> class grpc::TimePoint { public: - TimePoint(const NYdb::TDeadline& deadline) - : time_(NYdbGrpc::DeadlineToTimespec(deadline)) - {} + TimePoint(const NYdb::TDeadline& deadline); - gpr_timespec raw_time() const noexcept { - return time_; - } + gpr_timespec raw_time() const noexcept; private: + static gpr_timespec DurationToTimespec(const NYdb::TDeadline::Duration& duration) noexcept; + static gpr_timespec DeadlineToTimespec(const NYdb::TDeadline& deadline); + gpr_timespec time_; }; diff --git a/src/library/time/time.cpp b/src/library/time/time.cpp index eba86aa66fd..cb430141030 100644 --- a/src/library/time/time.cpp +++ b/src/library/time/time.cpp @@ -2,15 +2,6 @@ namespace NYdb::inline V3 { -namespace { - TDeadline::TimePoint SafeSum(TDeadline::TimePoint timePoint, TDeadline::Duration duration) { - if (duration > TDeadline::TimePoint::max() - timePoint) { - return TDeadline::TimePoint::max(); - } - return timePoint + duration; - } -} - TDeadline TDeadline::Now() { return TDeadline(Clock::now()); } @@ -43,8 +34,33 @@ bool TDeadline::operator<(const TDeadline& other) const noexcept { return TimePoint_ < other.TimePoint_; } +bool TDeadline::operator<=(const TDeadline& other) const noexcept { + return TimePoint_ <= other.TimePoint_; +} + bool TDeadline::operator==(const TDeadline& other) const noexcept { return TimePoint_ == other.TimePoint_; } +TDeadline TDeadline::operator+(const Duration& duration) const noexcept { + return TDeadline(SafeSum(TimePoint_, duration)); +} + +TDeadline TDeadline::operator-(const Duration& duration) const noexcept { + return TDeadline(SafeSum(TimePoint_, -duration)); +} + +TDeadline::TimePoint TDeadline::SafeSum(TDeadline::TimePoint timePoint, TDeadline::Duration duration) noexcept { + if (duration > TDeadline::Duration::zero()) { + if (timePoint > TDeadline::TimePoint::max() - duration) { + return TDeadline::TimePoint::max(); + } + } else { + if (TDeadline::TimePoint::min() - duration > timePoint) { + return TDeadline::TimePoint::min(); + } + } + return timePoint + duration; +} + } // namespace NYdb diff --git a/src/library/time/time.h b/src/library/time/time.h index a4ece9db770..fc851195de7 100644 --- a/src/library/time/time.h +++ b/src/library/time/time.h @@ -53,13 +53,19 @@ class TDeadline { TimePoint GetTimePoint() const noexcept; bool operator<(const TDeadline& other) const noexcept; + bool operator<=(const TDeadline& other) const noexcept; bool operator==(const TDeadline& other) const noexcept; + TDeadline operator+(const Duration& duration) const noexcept; + TDeadline operator-(const Duration& duration) const noexcept; + private: explicit TDeadline(TimePoint timePoint) noexcept : TimePoint_(timePoint) {} + static TimePoint SafeSum(TimePoint timePoint, Duration duration) noexcept; + TimePoint TimePoint_; }; From 8b2c8bf3d5483f1052b8f271f8ce4fe4f9f9fce2 Mon Sep 17 00:00:00 2001 From: Stanislav Tebloev Date: Tue, 25 Nov 2025 16:31:51 +0000 Subject: [PATCH 13/19] Add snowball filter for fulltext index (#28827) --- .github/last_commit.txt | 2 +- src/api/protos/ydb_table.proto | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index c1e79ac7873..0f8180cd3d7 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -9bbd29fb8a7d7a06303ce368a18641108124a8e1 +0f75010b9746c9e24ea49b8c1bf8b8b45e5c5692 diff --git a/src/api/protos/ydb_table.proto b/src/api/protos/ydb_table.proto index 953dca9b3b0..b03daa1c21d 100644 --- a/src/api/protos/ydb_table.proto +++ b/src/api/protos/ydb_table.proto @@ -174,7 +174,7 @@ message FulltextIndexSettings { // See Tokenizer enum optional Tokenizer tokenizer = 1; - // Language used for language-sensitive operations like stopword filtering + // Language used for language-sensitive operations like stopword filtering and stemming // Example: language = "english" // By default is not specified and no language-specific logic is applied optional string language = 2; @@ -229,6 +229,13 @@ message FulltextIndexSettings { // Maximum token length to keep (inclusive) // Must be used with use_filter_length optional int32 filter_length_max = 132 [(Ydb.value) = ">= 0"]; + + // Whether to apply snowball stemming to each token + // Must be used with language option + // Example: language = "english" + // Tokens: ["cars", "beautifully", "conspired"] + // Output: ["car", "beauti", "conspir"] + optional bool use_filter_snowball = 140; } // Represents text analyzers settings for a specific column From acd50a0d6a632f6f176d1f9daf618859144cb0b3 Mon Sep 17 00:00:00 2001 From: Nikolay Perfilov Date: Tue, 25 Nov 2025 16:31:57 +0000 Subject: [PATCH 14/19] Fix keepalive time (#29235) --- .github/last_commit.txt | 2 +- src/client/impl/internal/grpc_connections/grpc_connections.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 0f8180cd3d7..ad472f7addb 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -0f75010b9746c9e24ea49b8c1bf8b8b45e5c5692 +b97b42caffd37e4a5462751d2336b9df9adc84ab diff --git a/src/client/impl/internal/grpc_connections/grpc_connections.cpp b/src/client/impl/internal/grpc_connections/grpc_connections.cpp index 496e183788e..77667ee42c1 100644 --- a/src/client/impl/internal/grpc_connections/grpc_connections.cpp +++ b/src/client/impl/internal/grpc_connections/grpc_connections.cpp @@ -321,7 +321,7 @@ void TGRpcConnectionsImpl::Stop(bool wait) { void TGRpcConnectionsImpl::SetGrpcKeepAlive(NYdbGrpc::TGRpcClientConfig& config, const TDeadline::Duration& timeout, bool permitWithoutCalls) { std::uint64_t timeoutMs = std::chrono::duration_cast(timeout).count(); - config.IntChannelParams[GRPC_ARG_KEEPALIVE_TIME_MS] = timeoutMs >> 3; + config.IntChannelParams[GRPC_ARG_KEEPALIVE_TIME_MS] = timeoutMs; config.IntChannelParams[GRPC_ARG_KEEPALIVE_TIMEOUT_MS] = timeoutMs; config.IntChannelParams[GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA] = 0; config.IntChannelParams[GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS] = permitWithoutCalls ? 1 : 0; From 97498b01782757a7e9000c2b9f01402755714581 Mon Sep 17 00:00:00 2001 From: Bulat Date: Tue, 25 Nov 2025 16:32:03 +0000 Subject: [PATCH 15/19] [C++ SDK] Fixed uninitialized value in balancing policy (#29249) --- .github/last_commit.txt | 2 +- .../impl/internal/common/balancing_policies.cpp | 14 +++----------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index ad472f7addb..c9830f165d9 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -b97b42caffd37e4a5462751d2336b9df9adc84ab +fa69a03d9174062f49aaacfee16836b2d0c76ffd diff --git a/src/client/impl/internal/common/balancing_policies.cpp b/src/client/impl/internal/common/balancing_policies.cpp index 22ec50a6226..69f474742e1 100644 --- a/src/client/impl/internal/common/balancing_policies.cpp +++ b/src/client/impl/internal/common/balancing_policies.cpp @@ -4,23 +4,15 @@ namespace NYdb::inline V3 { TBalancingPolicy::TImpl TBalancingPolicy::TImpl::UseAllNodes() { - TBalancingPolicy::TImpl impl; - impl.PolicyType = EPolicyType::UseAllNodes; - return impl; + return {EPolicyType::UseAllNodes, std::nullopt, EPileState::UNSPECIFIED}; } TBalancingPolicy::TImpl TBalancingPolicy::TImpl::UsePreferableLocation(const std::optional& location) { - TBalancingPolicy::TImpl impl; - impl.PolicyType = EPolicyType::UsePreferableLocation; - impl.Location = location; - return impl; + return {EPolicyType::UsePreferableLocation, location, EPileState::UNSPECIFIED}; } TBalancingPolicy::TImpl TBalancingPolicy::TImpl::UsePreferablePileState(EPileState pileState) { - TBalancingPolicy::TImpl impl; - impl.PolicyType = EPolicyType::UsePreferablePileState; - impl.PileState = pileState; - return impl; + return {EPolicyType::UsePreferablePileState, std::nullopt, pileState}; } } From 97e5e464332b8e44777391b8f75ec5fe214223f1 Mon Sep 17 00:00:00 2001 From: Alek5andr-Kotov Date: Tue, 25 Nov 2025 16:32:10 +0000 Subject: [PATCH 16/19] Entries remain in the TxWrites after the immediate transaction (#29285) --- .github/last_commit.txt | 2 +- src/client/topic/ut/topic_to_table_ut.cpp | 26 ++++++++++++++ .../topic/ut/ut_utils/txusage_fixture.cpp | 36 +++++++++++++++++++ .../topic/ut/ut_utils/txusage_fixture.h | 2 ++ 4 files changed, 65 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index c9830f165d9..3022bb2af3b 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -fa69a03d9174062f49aaacfee16836b2d0c76ffd +3b818c91c3229212519212286faa60a6177afbc5 diff --git a/src/client/topic/ut/topic_to_table_ut.cpp b/src/client/topic/ut/topic_to_table_ut.cpp index 38a5c89ba38..448a56bbfce 100644 --- a/src/client/topic/ut/topic_to_table_ut.cpp +++ b/src/client/topic/ut/topic_to_table_ut.cpp @@ -83,6 +83,32 @@ Y_UNIT_TEST_WITH_REBOOTS(WriteToTopic_Demo_21, 0, 2, 10); Y_UNIT_TEST_WITH_REBOOTS(WriteToTopic_Demo_22, 0, 0, 10); Y_UNIT_TEST_WITH_REBOOTS(WriteToTopic_Demo_23, 0, 2, 0); +Y_UNIT_TEST_F(The_TxWriteInfo_Is_Deleted_After_The_Immediate_Transaction, TFixtureTable) +{ + CreateTopic("topic_A"); + + auto session = CreateSession(); + + auto tx = session->BeginTx(); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", tx.get()); + RestartPQTablet("topic_A", 0); + session->CommitTx(*tx, EStatus::SUCCESS); + + tx = session->BeginTx(); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", tx.get()); + RestartPQTablet("topic_A", 0); + session->CommitTx(*tx, EStatus::SUCCESS); + + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #1"); + UNIT_ASSERT_VALUES_EQUAL(messages[1], "message #2"); + + CheckTabletKeys("topic_A"); + + WaitForTheTabletToDeleteTheWriteInfo("topic_A", 0); +} + Y_UNIT_TEST_F(WriteToTopic_Demo_24_Table, TFixtureTable) { TestWriteToTopic24(); diff --git a/src/client/topic/ut/ut_utils/txusage_fixture.cpp b/src/client/topic/ut/ut_utils/txusage_fixture.cpp index 994e7969d2e..9ff1610dae8 100644 --- a/src/client/topic/ut/ut_utils/txusage_fixture.cpp +++ b/src/client/topic/ut/ut_utils/txusage_fixture.cpp @@ -1149,6 +1149,42 @@ void TFixture::SendLongTxLockStatus(const NActors::TActorId& actorId, runtime.SendToPipe(tabletId, actorId, event.release()); } +void TFixture::WaitForTheTabletToDeleteTheWriteInfo(const std::string& topicName, + std::uint32_t partition) +{ + auto& runtime = Setup->GetRuntime(); + NActors::TActorId edge = runtime.AllocateEdgeActor(); + std::uint64_t tabletId = GetTopicTabletId(edge, "/Root/" + topicName, partition); + + for (int i = 0; i < 20; ++i) { + auto request = std::make_unique(); + request->Record.SetCookie(12345); + request->Record.AddCmdRead()->SetKey("_txinfo"); + + auto& runtime = Setup->GetRuntime(); + + runtime.SendToPipe(tabletId, edge, request.release()); + auto response = runtime.GrabEdgeEvent(); + + UNIT_ASSERT(response->Record.HasCookie()); + UNIT_ASSERT_VALUES_EQUAL(response->Record.GetCookie(), 12345); + UNIT_ASSERT_VALUES_EQUAL(response->Record.ReadResultSize(), 1); + + auto& read = response->Record.GetReadResult(0); + + NKikimrPQ::TTabletTxInfo info; + UNIT_ASSERT(info.ParseFromString(read.GetValue())); + + if (info.TxWritesSize() == 0) { + return; + } + + std::this_thread::sleep_for(100ms); + } + + UNIT_FAIL("TTabletTxInfo.TxWrites is expected to be empty"); +} + void TFixture::WaitForTheTabletToDeleteTheWriteInfo(const NActors::TActorId& actorId, std::uint64_t tabletId, const NPQ::TWriteId& writeId) diff --git a/src/client/topic/ut/ut_utils/txusage_fixture.h b/src/client/topic/ut/ut_utils/txusage_fixture.h index 1c953bc6cc3..28354a4db57 100644 --- a/src/client/topic/ut/ut_utils/txusage_fixture.h +++ b/src/client/topic/ut/ut_utils/txusage_fixture.h @@ -143,6 +143,8 @@ class TFixture : public NUnitTest::TBaseFixture { void DeleteSupportivePartition(const std::string& topicName, std::uint32_t partition); + void WaitForTheTabletToDeleteTheWriteInfo(const std::string& topicName, + std::uint32_t partition); struct TTableRecord { TTableRecord() = default; From 48277b297357ffb3e616092d5f4199bc1c45ced7 Mon Sep 17 00:00:00 2001 From: stanislav_shchetinin Date: Tue, 25 Nov 2025 16:32:16 +0000 Subject: [PATCH 17/19] Encryption settings in API for backups to fs (#28305) --- .github/last_commit.txt | 2 +- src/api/grpc/ydb_export_v1.proto | 2 +- src/api/grpc/ydb_import_v1.proto | 3 ++ src/api/protos/ydb_export.proto | 28 ++++++++--- src/api/protos/ydb_import.proto | 85 +++++++++++++++++++++++++++++--- 5 files changed, 105 insertions(+), 15 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 3022bb2af3b..105e13645ed 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -3b818c91c3229212519212286faa60a6177afbc5 +dedf25c4b041c86ae94e81c561196b7b684e8223 diff --git a/src/api/grpc/ydb_export_v1.proto b/src/api/grpc/ydb_export_v1.proto index 6e4a37f09a4..752824ce4c1 100644 --- a/src/api/grpc/ydb_export_v1.proto +++ b/src/api/grpc/ydb_export_v1.proto @@ -17,5 +17,5 @@ service ExportService { // Exports data to file system. // Method starts an asynchronous operation that can be cancelled while it is in progress. - rpc ExportToFs(ExportToFsRequest) returns (ExportToFsResponse); + rpc ExportToFs(Export.ExportToFsRequest) returns (Export.ExportToFsResponse); } diff --git a/src/api/grpc/ydb_import_v1.proto b/src/api/grpc/ydb_import_v1.proto index 49c69424977..f630a4deba9 100644 --- a/src/api/grpc/ydb_import_v1.proto +++ b/src/api/grpc/ydb_import_v1.proto @@ -18,6 +18,9 @@ service ImportService { // List objects from existing export stored in S3 bucket rpc ListObjectsInS3Export(Import.ListObjectsInS3ExportRequest) returns (Import.ListObjectsInS3ExportResponse); + // List objects from existing export stored in FS + rpc ListObjectsInFsExport(Import.ListObjectsInFsExportRequest) returns (Import.ListObjectsInFsExportResponse); + // Writes data to a table. // Method accepts serialized data in the selected format and writes it non-transactionally. rpc ImportData(Import.ImportDataRequest) returns (Import.ImportDataResponse); diff --git a/src/api/protos/ydb_export.proto b/src/api/protos/ydb_export.proto index 6deda814dcc..62c45c8e17e 100644 --- a/src/api/protos/ydb_export.proto +++ b/src/api/protos/ydb_export.proto @@ -126,7 +126,7 @@ message ExportToS3Settings { // it is especially useful for custom s3 implementations bool disable_virtual_addressing = 12; - // Database root if not provided. + // Defaults to database root if not provided. // All object names are calculated and written relative to this path. string source_path = 13; @@ -185,18 +185,23 @@ message EncryptionSettings { /// File system (FS) message ExportToFsSettings { message Item { - // Database path to a table to be exported + // Database path to a table/directory to be exported string source_path = 1 [(required) = true]; - /* The tables are exported to one or more files in FS. - The files are saved to the destination_path directory. */ - string destination_path = 2 [(required) = true]; + /* Tables are exported to one or more files in FS. + The path begins with 'destination_path'. + If not specified, actual FS path is the default `base_path` concatenated with: + * The object path relative to the global `source_path` for a non-encrypted export + * The anonymized path for an encrypted export + */ + string destination_path = 2; } - // Base path on FS where to write export - // Path to the mounted directory in the case of NFS + // Base path on FS where to write all export items + // In the case of NFS, one of the directories in the path must be mounted // Must be an absolute path // Example: /mnt/exports + // SchemaMapping file with the list of objects is written to this path string base_path = 1 [(required) = true]; // List of items to export @@ -212,6 +217,15 @@ message ExportToFsSettings { // - zstd. // - zstd-N, where N is compression level, e.g. zstd-3. string compression = 5; + + // Settings for data encryption. + // If encryption_settings field is not specified, + // the resulting data will not be encrypted. + EncryptionSettings encryption_settings = 6; + + // Defaults to database root if not provided. + // All object names are calculated and written relative to this path. + string source_path = 7; } message ExportToFsResult { diff --git a/src/api/protos/ydb_import.proto b/src/api/protos/ydb_import.proto index 96afd5f141a..d603850e13e 100644 --- a/src/api/protos/ydb_import.proto +++ b/src/api/protos/ydb_import.proto @@ -131,19 +131,23 @@ message ImportFromFsSettings { * '/scheme.pb' - object with information about scheme, indexes, etc; * '/permissions.pb' - object with information about ACL and owner; * '/metadata.json' - object with metadata about the backup. - The path in FS is specified relative to base_path. - Example: "my_export/table1" + The FS path can be either provided explicitly (relative to base_path) + Or, if the export contains the database objects list, you may specify the database object name, + and the FS prefix will be looked up in the database objects list by the import procedure */ - string source_path = 1 [(required) = true]; + string source_path = 1; - // Database path to a table to import to. - string destination_path = 2 [(required) = true]; + // Database path to a database object to import the item to + // Resolved relative to the default destination_path + // May be omitted if the item's source_path is specified, in this case will be taken equal to it + string destination_path = 2; } // Base path on FS where the export is located - // Path to the mounted directory in the case of NFS + // In the case of NFS, one of the directories in the path must be mounted // Must be an absolute path // Example: /mnt/exports + // SchemaMapping file with the list of objects is read from this path string base_path = 1 [(required) = true]; repeated Item items = 2; // Empty collection means import of all export objects @@ -160,6 +164,15 @@ message ImportFromFsSettings { // Skip checksum validation during import bool skip_checksum_validation = 6; + + // Destination path to restore paths inside database + // Default value is database root + string destination_path = 7; + + // Settings how data is encrypted. + // If encryption_settings field is not specified, + // the resulting data is considered not encrypted. + Ydb.Export.EncryptionSettings encryption_settings = 8; } message ImportFromFsResult { @@ -263,6 +276,66 @@ message ListObjectsInS3ExportResponse { Ydb.Operations.Operation operation = 1; } +message ListObjectsInFsExportSettings { + message Item { + // Database object path + // Recursive for directories + string path = 1; + } + + string base_path = 1 [(required) = true]; + repeated Item items = 2; + uint32 number_of_retries = 3; + + // Settings how data is encrypted. + // If encryption_settings field is not specified, + // the resulting data is considered not encrypted. + Ydb.Export.EncryptionSettings encryption_settings = 4; +} + +message ListObjectsInFsExportResult { + message Item { + /* YDB database objects in S3 are stored in one or more S3 objects (see ydb_export.proto). + The S3 object name begins with a prefix, followed by: + * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero; + * '/scheme.pb' - object with information about scheme, indexes, etc; + * '/permissions.pb' - object with information about ACL and owner; + * '/metadata.json' - object with metadata about the backup. + */ + string fs_path = 1; + string db_path = 2; + } + + repeated Item items = 1; + + // This token allows you to get the next page of results for ListObjectsInFsExport requests, + // if the number of results is larger than `page_size` specified in the request. + // To get the next page, specify the value of `next_page_token` as a value for + // the `page_token` parameter in the next ListObjectsInFsExport request. Subsequent ListObjectsInFsExport + // requests will have their own `next_page_token` to continue paging through the results. + string next_page_token = 2; +} + +message ListObjectsInFsExportRequest { + Ydb.Operations.OperationParams operation_params = 1; + ListObjectsInFsExportSettings settings = 2 [(required) = true]; + + // The maximum number of results per page that should be returned. If the number of available + // results is larger than `page_size`, the service returns a `next_page_token` that can be used + // to get the next page of results in subsequent ListObjectsInFsExport requests. + // 0 means that server returns all objects. + int64 page_size = 3 [(value) = "<= 10000"]; + + // Page token. Set `page_token` to the `next_page_token` returned by a previous ListObjectsInFsExport + // request to get the next page of results. + string page_token = 4; +} + +message ListObjectsInFsExportResponse { + // operation.result = ListObjectsInFsExportResult + Ydb.Operations.Operation operation = 1; +} + /// Data message YdbDumpFormat { repeated string columns = 1; From aed6d216b4cdc063efe55f47a712cd8d1e286d48 Mon Sep 17 00:00:00 2001 From: Evgenik2 Date: Tue, 25 Nov 2025 16:32:23 +0000 Subject: [PATCH 18/19] Compress cluster state content (#29467) --- .github/last_commit.txt | 2 +- src/api/protos/ydb_monitoring.proto | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 105e13645ed..23856360b9d 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -dedf25c4b041c86ae94e81c561196b7b684e8223 +8f09fc5ca428fadaf61de1abdea31c06a664f147 diff --git a/src/api/protos/ydb_monitoring.proto b/src/api/protos/ydb_monitoring.proto index 03bb95362da..eed160c36f9 100644 --- a/src/api/protos/ydb_monitoring.proto +++ b/src/api/protos/ydb_monitoring.proto @@ -249,11 +249,11 @@ message ClusterStateResponse { message ClusterStateResultBlock { string name = 1; - string content = 2; + bytes content = 2; google.protobuf.Timestamp timestamp = 3; } message ClusterStateResult { - string result = 1; // deprecated + reserved 1; // deprecated repeated ClusterStateResultBlock blocks = 2; } From 3ad4ff4c2069d7f773a8c7a6e188a44bf0b96ef6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 25 Nov 2025 16:32:23 +0000 Subject: [PATCH 19/19] Update import generation: 29 --- .github/import_generation.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/import_generation.txt b/.github/import_generation.txt index f04c001f3f7..64bb6b746dc 100644 --- a/.github/import_generation.txt +++ b/.github/import_generation.txt @@ -1 +1 @@ -29 +30