feat(frac): new active fraction implementation based in mini-indexes

.github/CONTRIBUTING.md

@@ -86,6 +86,10 @@ WIP
 
 **Code that does not comply with these standards will require revision before acceptance.**
 
+## General Coding Guidelines
+
+**All metrics** must follow the [OpenMetrics](https://openmetrics.io/) standard convention.
+
 ## Mandatory Branch Naming Convention
 
 Branch names must use `kebab-case` and strictly follow this format:

.seqbench/baseline.env

@@ -0,0 +1,18 @@
+GOGC=100
+
+SEQDB_STORAGE_FRAC_SIZE=16MiB
+SEQDB_STORAGE_TOTAL_SIZE=10GiB
+
+SEQDB_LIMITS_QUERY_RATE=1024
+SEQDB_LIMITS_SEARCH_REQUESTS=1024
+SEQDB_LIMITS_BULK_REQUESTS=128
+SEQDB_LIMITS_INFLIGHT_BULK=128
+
+# DO NOT CHANGE FOLLOWING VALUES
+# SEQBAZOOKA RELIES ON THEM
+SEQDB_STORAGE_DATA_DIR=/var/seqdb
+SEQDB_MAPPING_PATH=/etc/mapping.yaml
+
+SEQDB_RESOURCES_SKIP_FSYNC=1
+SEQDB_COMPRESSION_DOCS_ZSTD_COMPRESSION_LEVEL=1
+SEQDB_COMPRESSION_METAS_ZSTD_COMPRESSION_LEVEL=1

.seqbench/comparison.env

@@ -0,0 +1,18 @@
+GOGC=50
+
+SEQDB_STORAGE_FRAC_SIZE=16MiB
+SEQDB_STORAGE_TOTAL_SIZE=10GiB
+
+SEQDB_LIMITS_QUERY_RATE=1024
+SEQDB_LIMITS_SEARCH_REQUESTS=1024
+SEQDB_LIMITS_BULK_REQUESTS=128
+SEQDB_LIMITS_INFLIGHT_BULK=128
+
+# DO NOT CHANGE FOLLOWING VALUES
+# SEQBAZOOKA RELIES ON THEM
+SEQDB_STORAGE_DATA_DIR=/var/seqdb
+SEQDB_MAPPING_PATH=/etc/mapping.yaml
+
+SEQDB_RESOURCES_SKIP_FSYNC=1
+SEQDB_COMPRESSION_DOCS_ZSTD_COMPRESSION_LEVEL=1
+SEQDB_COMPRESSION_METAS_ZSTD_COMPRESSION_LEVEL=1

asyncsearcher/async_searcher.go

@@ -47,22 +47,25 @@ var (
 		Namespace: "seq_db_store",
 		Subsystem: "async_search",
 		Name:      "in_progress",
-		Help:      "Amount of active async searches in progress",
+		Help:      "Number of active async searches in progress",
 	})
 	asyncSearchDiskUsage = promauto.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: "seq_db_store",
 		Subsystem: "async_search",
-		Name:      "disk_usage_bytes_total",
+		Name:      "disk_usage_bytes",
+		Help:      "Disk space used by async search files in bytes by file type",
 	}, []string{"file_type"})
 	asyncSearchStoredRequests = promauto.NewGauge(prometheus.GaugeOpts{
 		Namespace: "seq_db_store",
 		Subsystem: "async_search",
 		Name:      "stored_requests",
+		Help:      "Number of stored async search requests",
 	})
 	asyncSearchReadOnly = promauto.NewGauge(prometheus.GaugeOpts{
 		Namespace: "seq_db_store",
 		Subsystem: "async_search",
 		Name:      "read_only",
+		Help:      "Indicates if store is in async search read-only mode (can not start a new async search) due to disk usage limits",
 	})
 )
 

asyncsearcher/async_searcher_test.go

@@ -9,19 +9,18 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/ozontech/seq-db/frac"
-	"github.com/ozontech/seq-db/frac/common"
 	"github.com/ozontech/seq-db/frac/processor"
 	"github.com/ozontech/seq-db/mappingprovider"
 	"github.com/ozontech/seq-db/seq"
 )
 
 type fakeFrac struct {
 	frac.Fraction
-	info common.Info
+	info frac.Info
 	dp   fakeDP
 }
 
-func (f *fakeFrac) Info() *common.Info {
+func (f *fakeFrac) Info() *frac.Info {
 	return &f.info
 }
 
@@ -51,7 +50,7 @@ func TestAsyncSearcherMaintain(t *testing.T) {
 		Retention: time.Hour,
 	}
 	fracs := []frac.Fraction{
-		&fakeFrac{info: common.Info{Path: "1"}},
+		&fakeFrac{info: frac.Info{Path: "1"}},
 	}
 	r.NoError(as.StartSearch(req, fracs))
 

benchmarks/docker-compose-seqdb.yml

@@ -7,7 +7,7 @@ services:
         limits:
           cpus: "4"
           memory: "8GB"
-    image: ghcr.io/ozontech/seq-db:v0.61.0
+          image: 'gitlab-registry.ozon.ru/sre/images/seq-db:che@sha256:82d0dd34cb5d6db9e0450bc8d2cd1d9e29414ec2ba81dc8c4ae643dea6eb1bd0'
     ports:
       - '9002:9002'
     volumes:

bytespool/bytespool.go

@@ -35,27 +35,27 @@ var (
 		Namespace: "seq_db",
 		Subsystem: "common",
 		Name:      "bytes_pool_get_hits_total",
-		Help:      "",
+		Help:      "Number of hits while obtaining a buffer from the bytes pool",
 	}, []string{"capacity"})
 	missCounter = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: "seq_db",
 		Subsystem: "common",
 		Name:      "bytes_pool_get_misses_total",
-		Help:      "",
+		Help:      "Number of misses while obtaining a buffer from the bytes pool",
 	}, []string{"capacity"})
 
 	// put metrics
 	putCounter = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: "seq_db",
 		Subsystem: "common",
 		Name:      "bytes_pool_puts_total",
-		Help:      "",
+		Help:      "Number of buffers returned to the bytes pool",
 	}, []string{"capacity"})
 	putOversizeCounter = promauto.NewCounter(prometheus.CounterOpts{
 		Namespace: "seq_db",
 		Subsystem: "common",
 		Name:      "bytes_pool_put_oversizes_total",
-		Help:      "",
+		Help:      "Number of buffers rejected on return due to exceeding capacity",
 	})
 )
 

cmd/distribution/main.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/ozontech/seq-db/cache"
 	"github.com/ozontech/seq-db/consts"
-	"github.com/ozontech/seq-db/frac/common"
+	"github.com/ozontech/seq-db/frac"
 	"github.com/ozontech/seq-db/frac/sealed"
 	"github.com/ozontech/seq-db/fracmanager"
 	"github.com/ozontech/seq-db/logger"
@@ -59,7 +59,7 @@ func readBlock(reader storage.IndexReader, blockIndex uint32) ([]byte, error) {
 	return data, nil
 }
 
-func loadInfo(path string) *common.Info {
+func loadInfo(path string) *frac.Info {
 	indexReader, f := getReader(path)
 	defer f.Close()
 
@@ -87,7 +87,7 @@ func loadInfo(path string) *common.Info {
 	return b.Info
 }
 
-func buildDist(dist *seq.MIDsDistribution, path string, _ *common.Info) {
+func buildDist(dist *seq.MIDsDistribution, path string, _ *frac.Info) {
 	blocksReader, f := getReader(path)
 	defer f.Close()
 

cmd/seq-db/seq-db.go

@@ -23,7 +23,6 @@ import (
 	"github.com/ozontech/seq-db/config"
 	"github.com/ozontech/seq-db/consts"
 	"github.com/ozontech/seq-db/frac"
-	"github.com/ozontech/seq-db/frac/common"
 	"github.com/ozontech/seq-db/fracmanager"
 	"github.com/ozontech/seq-db/logger"
 	"github.com/ozontech/seq-db/mappingprovider"
@@ -259,7 +258,8 @@ func startStore(
 			MaintenanceDelay:  0,
 			CacheGCDelay:      0,
 			CacheCleanupDelay: 0,
-			SealParams: common.SealParams{
+			MinSealFracSize:   uint64(cfg.Storage.TotalSize) * consts.DefaultMinSealPercent / 100,
+			SealParams: frac.SealParams{
 				IDsZstdLevel:           cfg.Compression.SealedZstdCompressionLevel,
 				LIDsZstdLevel:          cfg.Compression.SealedZstdCompressionLevel,
 				TokenListZstdLevel:     cfg.Compression.SealedZstdCompressionLevel,

config/shared.go

@@ -8,7 +8,7 @@ var (
 	ReaderWorkers int
 
 	CaseSensitive = false
-	SkipFsync     = false
+	SkipFsync     = true
 
 	MaxFetchSizeBytes = 4 * units.MiB