Import YDB C++ SDK 30

.github/import_generation.txt

@@ -1 +1 @@
-30
+31

.github/last_commit.txt

@@ -1 +1 @@
-8f09fc5ca428fadaf61de1abdea31c06a664f147
+2e149af5c09f010d0c8fc01f73b86b724a2e8e98

examples/time/main.cpp

@@ -27,6 +27,7 @@ void TimeExample(const std::string& endpoint, const std::string& database) {
             auto ts2 = parser.ColumnParser("ts2").GetTimestamp64();
             auto interval = parser.ColumnParser("interval").GetInterval64();
 
+            // Standard output and std::format can overflow for large timestamps
             std::cout << "ts1: " << ts1 << std::endl;
             std::cout << "ts2: " << ts2 << std::endl;
             std::cout << "interval: " << std::chrono::duration_cast<std::chrono::milliseconds>(interval) << std::endl;

include/ydb-cpp-sdk/client/driver/driver.h

@@ -65,6 +65,9 @@ class TDriverConfig {
     //! Set database, this option can be overridden for client by ClientSettings
     TDriverConfig& SetDatabase(const std::string& database);
 
+    //! Get database path, returns the database path set via connection string or SetDatabase()
+    const std::string& GetDatabase() const;
+
     //! Set credentials data, this option can be overridden for client by ClientSettings
     TDriverConfig& SetCredentialsProviderFactory(std::shared_ptr<ICredentialsProviderFactory> credentialsProviderFactory);
 

include/ydb-cpp-sdk/client/export/export.h

@@ -26,6 +26,12 @@ struct TExportItemProgress {
     TInstant EndTime;
 };
 
+struct TEncryptionAlgorithm {
+    static const std::string AES_128_GCM;
+    static const std::string AES_256_GCM;
+    static const std::string CHACHA_20_POLY_1305;
+};
+
 /// YT
 struct TExportToYtSettings : public TOperationRequestSettings<TExportToYtSettings> {
     struct TItem {
@@ -79,11 +85,8 @@ struct TExportToS3Settings : public TOperationRequestSettings<TExportToS3Setting
         UNKNOWN = std::numeric_limits<int>::max(),
     };
 
-    struct TEncryptionAlgorithm {
-        static const std::string AES_128_GCM;
-        static const std::string AES_256_GCM;
-        static const std::string CHACHA_20_POLY_1305;
-    };
+    // For backward compatibility
+    using TEncryptionAlgorithm = NExport::TEncryptionAlgorithm;
 
     struct TItem {
         std::string Src;
@@ -140,6 +143,16 @@ struct TExportToFsSettings : public TOperationRequestSettings<TExportToFsSetting
     FLUENT_SETTING_OPTIONAL(std::string, Description);
     FLUENT_SETTING_OPTIONAL(uint32_t, NumberOfRetries);
     FLUENT_SETTING_OPTIONAL(std::string, Compression);
+    FLUENT_SETTING_OPTIONAL(std::string, SourcePath);
+
+    TSelf& SymmetricEncryption(const std::string& algorithm, const std::string& key) {
+        EncryptionAlgorithm_ = algorithm;
+        SymmetricKey_ = key;
+        return *this;
+    }
+
+    std::string EncryptionAlgorithm_;
+    std::string SymmetricKey_;
 };
 
 class TExportToFsResponse : public TOperation {

include/ydb-cpp-sdk/client/monitoring/monitoring.h

@@ -38,6 +38,7 @@ struct TSelfCheckSettings : public TOperationRequestSettings<TSelfCheckSettings>
 struct TClusterStateSettings : public TOperationRequestSettings<TClusterStateSettings> {
     FLUENT_SETTING_OPTIONAL(uint32_t, DurationSeconds);
     FLUENT_SETTING_OPTIONAL(uint32_t, PeriodSeconds);
+    FLUENT_SETTING_OPTIONAL(bool, NoSanitize);
 };
 
 

include/ydb-cpp-sdk/client/query/client.h

@@ -215,6 +215,7 @@ class TTxControl {
         return TTxControl(settings);
     }
 
+    // Do not explicitly set the transaction mode. YDB determines the behavior automatically
     static TTxControl NoTx() {
         return TTxControl();
     }

include/ydb-cpp-sdk/client/table/table.h

@@ -301,6 +301,7 @@ struct TFulltextIndexSettings {
     enum class ELayout {
         Unspecified = 0,
         Flat,
+        FlatRelevance,
     };
 
     enum class ETokenizer {

include/ydb-cpp-sdk/client/topic/control_plane.h

@@ -48,11 +48,10 @@ enum class EDeadLetterAction {
     Move = 2,
 };
 
-// 0 - unspecified
-// 1 - disabeld
-// 2 - database level metrics
-// 3 - object level metrics
-// 4 - detailed metrics
+// 0 - disabled
+// 1 - database level metrics
+// 2 - object level metrics
+// 3 - detailed metrics
 using EMetricsLevel = uint32_t;
 
 class TDeadLetterPolicyCondition {

include/ydb-cpp-sdk/client/types/exceptions/exceptions.h

@@ -12,6 +12,11 @@ class TYdbException : public yexception {
     TYdbException(const std::string& reason);
 };
 
+class TAuthenticationError : public TYdbException {
+public:
+    TAuthenticationError(const std::string& reason);
+};
+
 class TContractViolation : public TYdbException {
 public:
     TContractViolation(const std::string& reason);

src/api/protos/ydb_export.proto

@@ -139,6 +139,12 @@ message ExportToS3Settings {
     // If encryption_settings field is not specified,
     // the resulting data will not be encrypted.
     EncryptionSettings encryption_settings = 15;
+
+    // Materialization of index table data.
+    // By default, only index metadata is uploaded and indexes are built during import — it saves space
+    // and reduces export time, but it can potentially increase the import time.
+    // Indexes can be materialized, then their data will be uploaded during export and downloaded during import.
+    bool materialize_indexes = 16;
 }
 
 message ExportToS3Result {

src/api/protos/ydb_import.proto

@@ -62,6 +62,17 @@ message ImportFromS3Settings {
         string destination_path = 2;
     }
 
+    enum IndexFillingMode {
+        // If unspecified, use default - Build
+        INDEX_FILLING_MODE_UNSPECIFIED = 0;
+        // Build index
+        INDEX_FILLING_MODE_BUILD = 1;
+        // Import materialized index
+        INDEX_FILLING_MODE_IMPORT = 2;
+        // Try to import materialized index, build otherwise
+        INDEX_FILLING_MODE_AUTO = 3;
+    }
+
     string endpoint = 1 [(required) = true];
     Scheme scheme = 2; // HTTPS if not specified
     string bucket = 3 [(required) = true];
@@ -100,6 +111,10 @@ message ImportFromS3Settings {
     // If encryption_settings field is not specified,
     // the resulting data is considered not encrypted.
     Ydb.Export.EncryptionSettings encryption_settings = 15;
+
+    // Index filling mode.
+    // If not specified, indexes will be built.
+    IndexFillingMode index_filling_mode = 16;
 }
 
 message ImportFromS3Result {

src/api/protos/ydb_monitoring.proto

@@ -240,6 +240,7 @@ message ClusterStateRequest {
     Ydb.Operations.OperationParams operation_params = 1; // basic operation params, including timeout
     uint32 duration_seconds = 2;
     uint32 period_seconds = 3;
+    bool no_sanitize = 4;
 }
 
 message ClusterStateResponse {

src/api/protos/ydb_table.proto

@@ -118,7 +118,9 @@ message FulltextIndexSettings {
     enum Layout {
         LAYOUT_UNSPECIFIED = 0;
 
-        // Uses a single flat inverted index table (indexImplTable)
+        // Simplest possible layout without support for relevance.
+        // Uses a single flat inverted index table (indexImplTable).
+        // Supports a single column only.
         // Example source table:
         //     ┌────┬────────────────────────────┐
         //     │ id │ text                       │
@@ -130,17 +132,66 @@ message FulltextIndexSettings {
         //     ┌──────────────┬────┐
         //     │ __ydb_token  │ id │
         //     ├──────────────┼────┤
+        //     │ "The"        │ 1  │
+        //     │ "The"        │ 2  │
         //     │ "blue"       │ 2  │
         //     │ "brown"      │ 1  │
         //     │ "fox"        │ 1  │
         //     │ "hare"       │ 2  │
         //     │ "quick"      │ 1  │
         //     │ "quick"      │ 2  │
-        //     │ "The"        │ 1  │
-        //     │ "The"        │ 2  │
         //     └──────────────┴────┘
-        // Supports a single column only
         FLAT = 1;
+
+        // Simple layout like FLAT, but with support for relevance.
+        // Uses multiple index tables. Supports a single column only.
+        // Example source table:
+        //     ┌────┬───────────────────────────────────────┐
+        //     │ id │ text                                  │
+        //     ├────┼───────────────────────────────────────┤
+        //     │ 1  │ "A quick fox catches quick hare"      │
+        //     │ 2  │ "The quick blue hare"                 │
+        //     └────┴───────────────────────────────────────┘
+        // Example inverted index table with term frequencies (indexImplTable):
+        //     ┌──────────────┬────┬────────────┐
+        //     │ __ydb_token  │ id │ __ydb_freq │
+        //     ├──────────────┼────┼────────────┤
+        //     │ "A"          │ 1  │ 1          │
+        //     │ "The"        │ 2  │ 1          │
+        //     │ "blue"       │ 2  │ 1          │
+        //     │ "catches"    │ 1  │ 1          │
+        //     │ "fox"        │ 1  │ 1          │
+        //     │ "hare"       │ 1  │ 1          │
+        //     │ "hare"       │ 2  │ 1          │
+        //     │ "quick"      │ 1  │ 2          │
+        //     │ "quick"      │ 2  │ 1          │
+        //     └──────────────┴────┴────────────┘
+        // Term dictionary table (indexImplDictTable):
+        //     ┌──────────────┬────────────┐
+        //     │ __ydb_token  │ __ydb_freq │
+        //     ├──────────────┼────────────┤
+        //     │ "A"          │ 1          │
+        //     │ "The"        │ 1          │
+        //     │ "blue"       │ 1          │
+        //     │ "catches"    │ 1          │
+        //     │ "fox"        │ 1          │
+        //     │ "hare"       │ 2          │
+        //     │ "quick"      │ 2          │
+        //     └──────────────┴────────────┘
+        // Document statistics table (indexImplDocsTable):
+        //     ┌────┬──────────────┐
+        //     │ id │ __ydb_length │
+        //     ├────┼──────────────┤
+        //     │ 1  │ 6            │
+        //     │ 2  │ 4            │
+        //     └────┴──────────────┘
+        // Global statistics table (indexImplStatsTable):
+        //     ┌──────────┬─────────────────┬────────────────────┐
+        //     │ __ydb_id │ __ydb_doc_count │ __ydb_total_length │
+        //     ├──────────┼─────────────────┼────────────────────┤
+        //     │ 1        │ 2               │ 10                 │
+        //     └──────────┴─────────────────┴────────────────────┘
+        FLAT_RELEVANCE = 2;
     }
 
     // Specifies how text is tokenized during indexing

src/client/driver/driver.cpp

@@ -139,6 +139,10 @@ TDriverConfig& TDriverConfig::SetDatabase(const std::string& database) {
     return *this;
 }
 
+const std::string& TDriverConfig::GetDatabase() const {
+    return Impl_->Database;
+}
+
 TDriverConfig& TDriverConfig::SetCredentialsProviderFactory(std::shared_ptr<ICredentialsProviderFactory> credentialsProviderFactory) {
     Impl_->CredentialsProviderFactory = credentialsProviderFactory;
     return *this;

src/client/export/export.cpp

@@ -21,9 +21,9 @@ namespace NExport {
 using namespace NThreading;
 using namespace Ydb::Export;
 
-const std::string TExportToS3Settings::TEncryptionAlgorithm::AES_128_GCM = "AES-128-GCM";
-const std::string TExportToS3Settings::TEncryptionAlgorithm::AES_256_GCM = "AES-256-GCM";
-const std::string TExportToS3Settings::TEncryptionAlgorithm::CHACHA_20_POLY_1305 = "ChaCha20-Poly1305";
+const std::string TEncryptionAlgorithm::AES_128_GCM = "AES-128-GCM";
+const std::string TEncryptionAlgorithm::AES_256_GCM = "AES-256-GCM";
+const std::string TEncryptionAlgorithm::CHACHA_20_POLY_1305 = "ChaCha20-Poly1305";
 
 /// Common
 namespace {
@@ -283,6 +283,19 @@ TFuture<TExportToFsResponse> TExportClient::ExportToFs(const TExportToFsSettings
         request.mutable_settings()->set_compression(TStringType{settings.Compression_.value()});
     }
 
+    if (settings.SourcePath_) {
+        request.mutable_settings()->set_source_path(settings.SourcePath_.value());
+    }
+
+    if (settings.EncryptionAlgorithm_.empty() != settings.SymmetricKey_.empty()) {
+        throw TContractViolation("Encryption algorithm and symmetric key must be set together");
+    }
+
+    if (!settings.EncryptionAlgorithm_.empty() && !settings.SymmetricKey_.empty()) {
+        request.mutable_settings()->mutable_encryption_settings()->set_encryption_algorithm(settings.EncryptionAlgorithm_);
+        request.mutable_settings()->mutable_encryption_settings()->mutable_symmetric_key()->set_key(settings.SymmetricKey_);
+    }
+
     return Impl_->ExportToFs(std::move(request), settings);
 }
 

src/client/impl/internal/grpc_connections/grpc_connections.cpp

@@ -8,18 +8,25 @@ namespace NYdb::inline V3 {
 
 bool IsTokenCorrect(const std::string& in) {
     for (char c : in) {
-        if (!(IsAsciiAlnum(c) || IsAsciiPunct(c) || c == ' '))
+        if (!(IsAsciiAlnum(c) || IsAsciiPunct(c) || c == ' ')) {
             return false;
+        }
     }
     return true;
 }
 
 std::string GetAuthInfo(TDbDriverStatePtr p) {
-    auto token = p->CredentialsProvider->GetAuthInfo();
-    if (!IsTokenCorrect(token)) {
-        throw TContractViolation("token is incorrect, illegal characters found");
+    try {
+        auto token = p->CredentialsProvider->GetAuthInfo();
+        if (!IsTokenCorrect(token)) {
+            throw TAuthenticationError("token is incorrect, illegal characters found");
+        }
+        return token;
+    } catch (const TAuthenticationError& e) {
+        throw e;
+    } catch (const std::exception& e) {
+        throw TAuthenticationError(TStringBuilder() << "Can't get Authentication info from CredentialsProvider. " << e.what());
     }
-    return token;
 }
 
 void SetDatabaseHeader(TCallMeta& meta, const std::string& database) {
@@ -442,4 +449,40 @@ void TGRpcConnectionsImpl::EnqueueResponse(IObjectInQueue* action) {
     });
 }
 
+TCallMeta TGRpcConnectionsImpl::MakeCallMeta(const TRpcRequestSettings& requestSettings, const TDbDriverStatePtr& dbState) const {
+    TCallMeta meta;
+    meta.Timeout = requestSettings.Deadline;
+#ifndef YDB_GRPC_UNSECURE_AUTH
+    meta.CallCredentials = dbState->CallCredentials;
+#else
+    if (requestSettings.UseAuth && dbState->CredentialsProvider && dbState->CredentialsProvider->IsValid()) {
+        meta.Aux.push_back({YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState)});
+    }
+#endif
+    if (!requestSettings.TraceId.empty()) {
+        meta.Aux.push_back({YDB_TRACE_ID_HEADER, requestSettings.TraceId});
+    }
+
+    if (!requestSettings.RequestType.empty()) {
+        meta.Aux.push_back({YDB_REQUEST_TYPE_HEADER, requestSettings.RequestType});
+    }
+
+    if (!requestSettings.TraceParent.empty()) {
+        meta.Aux.push_back({OTEL_TRACE_HEADER, requestSettings.TraceParent});
+    }
+
+    if (!dbState->Database.empty()) {
+        // See TDbDriverStateTracker::GetDriverState to find place where we do quote non ASCII characters
+        meta.Aux.push_back({YDB_DATABASE_HEADER, dbState->Database});
+    }
+
+    static const std::string clientPid = GetClientPIDHeaderValue();
+
+    meta.Aux.push_back({YDB_SDK_BUILD_INFO_HEADER, CreateSDKBuildInfo()});
+    meta.Aux.push_back({YDB_CLIENT_PID, clientPid});
+    meta.Aux.insert(meta.Aux.end(), requestSettings.Header.begin(), requestSettings.Header.end());
+
+    return meta;
+}
+
 } // namespace NYdb