seq-ui multicluster

api/errorgroups/v1/errorgroups.proto

@@ -78,7 +78,7 @@ message GetDetailsResponse {
         string value = 1;
         uint64 percent = 2;
     }
-    
+
     message Distributions {
         repeated Distribution by_env = 1;
         repeated Distribution by_release = 2;

api/seqapi/v1/seq_api.proto

@@ -35,6 +35,8 @@ service SeqAPIService {
   rpc CancelAsyncSearch(CancelAsyncSearchRequest) returns (CancelAsyncSearchResponse) {}
 
   rpc DeleteAsyncSearch(DeleteAsyncSearchRequest) returns (DeleteAsyncSearchResponse) {}
+
+  rpc GetEnvs(GetEnvsRequest) returns (GetEnvsResponse) {}
 }
 
 enum ErrorCode {
@@ -320,3 +322,17 @@ message DeleteAsyncSearchRequest {
 }
 
 message DeleteAsyncSearchResponse {}
+
+message GetEnvsRequest {}
+
+message GetEnvsResponse {
+    message Env {
+        string env = 1;
+        uint32 max_search_limit = 2;
+        uint32 max_export_limit = 3;
+        uint32 max_parallel_export_requests = 4;
+        uint32 max_aggregations_per_request = 5;
+        uint32 seq_cli_max_search_limit = 6;
+    }
+    repeated Env envs = 1;
+}

cmd/seq-ui/main.go

@@ -103,15 +103,27 @@ func run(ctx context.Context) {
 }
 
 func initApp(ctx context.Context, cfg config.Config) *api.Registrar {
-	logger.Info("initializing seq-db client")
-	seqDBClient, err := initSeqDBClient(ctx, cfg)
+	logger.Info("initializing seq-db clients")
+	seqDBClients, err := initSeqDBClients(ctx, cfg)
 	if err != nil {
 		logger.Fatal("failed to init seq-db client", zap.Error(err))
 	}
 
+	defaultClientID := config.DefaultSeqDBClientID
+	if len(cfg.Handlers.SeqAPI.Envs) > 0 {
+		defaultClientID = cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv].SeqDB
+	}
+
+	defaultClient, exists := seqDBClients[defaultClientID]
+	if !exists {
+		logger.Fatal("default seq-db client not found",
+			zap.String("clientID", defaultClientID),
+		)
+	}
+
 	var massExportV1 *massexport_v1.MassExport
 	if cfg.Handlers.MassExport != nil {
-		exportServer, err := initExportService(ctx, *cfg.Handlers.MassExport, seqDBClient)
+		exportServer, err := initExportService(ctx, *cfg.Handlers.MassExport, defaultClient)
 		if err != nil {
 			logger.Fatal("can't init export server", zap.Error(err))
 		}
@@ -151,10 +163,10 @@ func initApp(ctx context.Context, cfg config.Config) *api.Registrar {
 		userProfileV1 = userprofile_v1.New(svc, p)
 		dashboardsV1 = dashboards_v1.New(svc, p)
 
-		asyncSearchesService = asyncsearches.New(ctx, repo, seqDBClient, cfg.Handlers.AsyncSearch.AdminUsers)
+		asyncSearchesService = asyncsearches.New(ctx, repo, defaultClient, cfg.Handlers.AsyncSearch.AdminUsers)
 	}
 
-	seqApiV1 := seqapi_v1.New(cfg.Handlers.SeqAPI, seqDBClient, inmemWithRedisCache, redisCache, asyncSearchesService, p)
+	seqApiV1 := seqapi_v1.New(cfg.Handlers.SeqAPI, seqDBClients, inmemWithRedisCache, redisCache, asyncSearchesService, p)
 
 	logger.Info("initializing clickhouse")
 	ch, err := initClickHouse(ctx, cfg.Server.CH)
@@ -173,25 +185,38 @@ func initApp(ctx context.Context, cfg config.Config) *api.Registrar {
 	return api.NewRegistrar(seqApiV1, userProfileV1, dashboardsV1, massExportV1, errorGroupsV1)
 }
 
-func initSeqDBClient(ctx context.Context, cfg config.Config) (seqdb.Client, error) {
-	clientMaxRecvMsgSize := cfg.Clients.SeqDBAvgDocSize * 1024 * int(cfg.Handlers.SeqAPI.MaxSearchLimit)
+func initSeqDBClients(ctx context.Context, cfg config.Config) (map[string]seqdb.Client, error) {
+	clients := make(map[string]seqdb.Client)
+	for _, clientCfg := range cfg.Clients.SeqDB {
+		client, err := createSeqBDClient(ctx, clientCfg, cfg.Handlers.SeqAPI)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create seq_db client %s: %w", clientCfg.ID, err)
+		}
+		clients[clientCfg.ID] = client
+	}
+
+	return clients, nil
+}
+
+func createSeqBDClient(ctx context.Context, cfg config.SeqDBClient, seqAPI config.SeqAPI) (seqdb.Client, error) {
+	clientMaxRecvMsgSize := cfg.AvgDocSize * 1024 * int(seqAPI.MaxSearchLimit)
 	if clientMaxRecvMsgSize < defaultClientMaxRecvMsgSize {
 		clientMaxRecvMsgSize = defaultClientMaxRecvMsgSize
 	}
 
 	clientParams := seqdb.ClientParams{
-		Addrs:               cfg.Clients.SeqDBAddrs,
-		Timeout:             cfg.Clients.SeqDBTimeout,
-		MaxRetries:          cfg.Clients.RequestRetries,
-		InitialRetryBackoff: cfg.Clients.InitialRetryBackoff,
-		MaxRetryBackoff:     cfg.Clients.MaxRetryBackoff,
+		Addrs:               cfg.Addrs,
+		Timeout:             cfg.Timeout,
+		MaxRetries:          cfg.RequestRetries,
+		InitialRetryBackoff: cfg.InitialRetryBackoff,
+		MaxRetryBackoff:     cfg.MaxRetryBackoff,
 		MaxRecvMsgSize:      clientMaxRecvMsgSize,
 	}
-	if cfg.Clients.GRPCKeepaliveParams != nil {
+	if cfg.GRPCKeepaliveParams != nil {
 		clientParams.GRPCKeepaliveParams = &seqdb.GRPCKeepaliveParams{
-			Time:                cfg.Clients.GRPCKeepaliveParams.Time,
-			Timeout:             cfg.Clients.GRPCKeepaliveParams.Timeout,
-			PermitWithoutStream: cfg.Clients.GRPCKeepaliveParams.PermitWithoutStream,
+			Time:                cfg.GRPCKeepaliveParams.Time,
+			Timeout:             cfg.GRPCKeepaliveParams.Timeout,
+			PermitWithoutStream: cfg.GRPCKeepaliveParams.PermitWithoutStream,
 		}
 	}
 

config/config.multicluster.yaml

@@ -0,0 +1,48 @@
+server:
+  http_addr: ":5555"
+  grpc_addr: ":5556"
+  debug_addr: ":5557"
+
+clients:
+  seq_db:
+    - id: "default-seq-db"
+      addrs:
+        - "seq-db-proxy:9004"
+      timeout: 15s
+    - id: "test-seq-db"
+      addrs:
+        - "seq-db-proxy:9005"
+      timeout: 15s
+
+handlers:
+  seq_api:
+    max_search_limit: 10
+    max_export_limit: 11
+    max_parallel_export_requests: 12
+    max_aggregations_per_request: 5
+    seq_cli_max_search_limit: 6
+    pinned_fields:
+      - name: level
+        type: keyword
+      - name: message
+        type: text
+    default_env: "production"
+    envs:
+      production:
+        seq_db_id: "default-seq-db"
+
+      staging:
+        seq_db_id: "test-seq-db"
+        options:
+          max_search_limit: 20
+          max_export_limit: 22
+          max_parallel_export_requests: 24
+          max_aggregations_per_request: 10
+          seq_cli_max_search_limit: 12
+          pinned_fields:
+            - name: level
+              type: keyword
+            - name: message
+              type: text
+            - name: service
+              type: keyword

docs/en/02-configuration.md

@@ -372,48 +372,65 @@ clients:
   initial_retry_backoff:
   max_retry_backoff:
   grpc_keepalive_params:
+  seq_db:
 ```
 
 **`seq_db_addrs`** *`[]string`* *`required`*
 
 List of seq-db proxy hosts to be used in client calls. If there are more than one host, for each request random host will be chosen.
 
+> Deprecated. Use `seq_db.addrs` instead.
+
 **`proxy_client_mode`** *`string`* *`default="grpc"`* *`options="grpc"`*
 
 Allow choosing how to send requests to seq-db.
 
+> Deprecated. Use `seq_db.client_mode` instead.
+
 **`seq_db_timeout`** *`string`* *`default="0"`*
 
 Timeout for requests made by the client. A zero value means no timeout.
 
 > The value must be passed in the duration format: `<number>(ms|s|m|h)`.
 
+> Deprecated. Use `seq_db.timeout` instead.
+
 **`seq_db_avg_doc_size`** *`int`* *`default=0`*
 
 Specifies the average documents size in `KB` that the client calls returns. It's used in combination with `handlers.seq_api.max_search_limit` to calculate the maximum response size per client request.
 
 > Regardless of `seq_db_avg_doc_size`, the minimum response size per client request is `4MB`.
 
+> Deprecated. Use `seq_db.avg_doc_size` instead.
+
 **`request_retries`** *`int`* *`default=0`*
 
 The number of retries to send a request to client after the first attempt. For each retry, the entire `seq_db_addrs` list will be searched, choosing a random host. A zero value means no retries. If set to negative value, then it will be reset to `default`.
 
+> Deprecated. Use `seq_db.request_retries` instead.
+
 **`initial_retry_backoff`** *`string`* *`default="0"`*
 
 Initial duration interval value to be used in backoff with retries. If set to `0`, disables backoff.
 
 > The value must be passed in the duration format: `<number>(ms|s|m|h)`.
 
+> Deprecated. Use `seq_db.initial_retry_backoff` instead.
+
 **`max_retry_backoff`** *`string`* *`default="0"`*
 
 Max duration interval value to be used in backoff with retries. If set to `0`, only value from `initial_retry_backoff` is used for calculating backoff and the backoff is not higher than `initial_retry_backoff * 2`.
 
 > The value must be passed in the duration format: `<number>(ms|s|m|h)`.
 
+> Deprecated. Use `seq_db.max_retry_backoff` instead.
+
 **`grpc_keepalive_params`** *`GRPCKeepaliveParams`* *`optional`*
 
 If gRPC keepalive params are not set, no keepalive params are applied to gRPC client.
 
+> Deprecated. Use `seq_db.grpc_keepalive_params` instead.
+
 `GRPCKeepaliveParams` fields:
 
 + **`time`** *`string`* *`default="10s"`*
@@ -432,6 +449,56 @@ If gRPC keepalive params are not set, no keepalive params are applied to gRPC cl
 
   If set to `true`, client sends keepalive pings even with no active RPCs. Otherwise, when there are no active RPCs, `time` and `timeout` will be ignored and no keepalive pings will be sent.
 
+**`seq_db`** *`[]SeqDBClient`* *`optional`*
+
+List of SeqDB client configurations.
+
+`SeqDBClient` fields:
+
++ **`id`** *`string`* *`required`*
+
+  Unique client identifier.
+
++ **`addrs`** *`[]string`* *`required`*
+
+  List of seq-db proxy hosts to be used in client calls. If there are more than one host, for each request random host will be chosen.
+
++ **`timeout`** *`string`* *`default="0"`*
+
+  Timeout for requests made by the client. A zero value means no timeout.
+
+  > The value must be passed in the duration format: `<number>(ms|s|m|h)`.
+
++ **`avg_doc_size`** *`int`* *`default=0`*
+
+  Specifies the average documents size in `KB` that the client calls returns. It's used in combination with `handlers.seq_api.max_search_limit` to calculate the maximum response size per client request.
+
+  > Regardless of `avg_doc_size`, the minimum response size per client request is `4MB`.
+
++ **`request_retries`** *`int`* *`default=0`*
+
+  The number of retries to send a request to client after the first attempt. For each retry, the entire `addrs` list will be searched, choosing a random host. A zero value means no retries. If set to negative value, then it will be reset to `default`.
+
++ **`initial_retry_backoff`** *`string`* *`default="0"`*
+
+  Initial duration interval value to be used in backoff with retries. If set to `0`, disables backoff.
+
+  > The value must be passed in the duration format: `<number>(ms|s|m|h)`.
+
++ **`max_retry_backoff`** *`string`* *`default="0"`*
+
+  Max duration interval value to be used in backoff with retries. If set to `0`, only value from `initial_retry_backoff` is used for calculating backoff and the backoff is not higher than `initial_retry_backoff * 2`.
+
+  > The value must be passed in the duration format: `<number>(ms|s|m|h)`.
+
++ **`client_mode`** *`string`* *`default="grpc"`* *`options="grpc"`
+
+  Allow choosing how to send requests to seq-db.
+
++ **`grpc_keepalive_params`** *`GRPCKeepaliveParams`* *`optional`*
+
+  If gRPC keepalive params are not set, no keepalive params are applied to gRPC client.
+
 ## Handlers
 
 ```yaml
@@ -448,7 +515,7 @@ handlers:
 Config for `/seqapi` API handlers.
 
 `SeqAPI` fields:
-
+ 
 + **`max_search_limit`** *`int`* *`default=0`*
 
   Max value for `limit` field in search requests.
@@ -605,6 +672,24 @@ Config for `/seqapi` API handlers.
   + **`values`** *`[]string`* *`required`*
 
     List of event field values to filter.
+
+  + **`default_env`** *`string`* *`optional`*
+
+    Environment name used by default to process requests when no explicit environment is specified.
+
++ **`envs`** *`map[string]SeqAPIEnv`* *`optional`*
+
+  Environment-specific configurations
+
+  `SeqAPIEnv` fields:
+
+  + **`seq_db_id`** *`string`* *`required`*
+
+    SeqDB client identifier from `clients.seq_db` configuration.
+
+  + **`options`** *`SeqAPIOptions`* *`optional`*
+
+  Environment-specific API parameters. These settings override the corresponding global values defined in the root `seq_api` section.
 
 ### Error groups