Skip to content

Commit 9aaa9b1

Browse files
coderabbitai[bot]coderabbitai[bot]
authored
📝 Add docstrings to jt/dbmigrate
Docstrings generation was requested by @JayT106. * #1908 (comment) The following files were modified: * `cmd/cronosd/cmd/database.go` * `cmd/cronosd/cmd/migrate_db.go` * `cmd/cronosd/cmd/patch_db.go` * `cmd/cronosd/cmd/root.go` * `cmd/cronosd/dbmigrate/height_filter.go` * `cmd/cronosd/dbmigrate/migrate.go` * `cmd/cronosd/dbmigrate/migrate_no_rocksdb.go` * `cmd/cronosd/dbmigrate/migrate_rocksdb.go` * `cmd/cronosd/dbmigrate/patch.go` * `cmd/cronosd/dbmigrate/swap-migrated-db.sh` * `x/e2ee/client/cli/encrypt.go`
1 parent ae678b5 commit 9aaa9b1

File tree

11 files changed

+3819
-3
lines changed

11 files changed

+3819
-3
lines changed

‎cmd/cronosd/cmd/database.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package cmd
2+
3+
import (
4+
"github.com/spf13/cobra"
5+
)
6+
7+
// DatabaseCmd constructs the top-level "database" Cobra command and registers its migrate and patch subcommands.
8+
// The command is named "database" with alias "db" and provides help text describing available subcommands.
9+
func DatabaseCmd() *cobra.Command {
10+
cmd := &cobra.Command{
11+
Use: "database",
12+
Short: "Database management commands",
13+
Long: `Commands for managing Cronos databases.
14+
15+
Available subcommands:
16+
migrate - Migrate databases between different backend types
17+
patch - Patch specific block heights into existing databases
18+
19+
Use "cronosd database [command] --help" for more information about a command.`,
20+
Aliases: []string{"db"},
21+
}
22+
23+
// Add subcommands
24+
cmd.AddCommand(
25+
MigrateCmd(), // migrate-db -> database migrate
26+
PatchCmd(), // patchdb -> database patch
27+
)
28+
29+
return cmd
30+
}

‎cmd/cronosd/cmd/migrate_db.go‎

Lines changed: 361 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,361 @@
1+
package cmd
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
7+
dbm "github.com/cosmos/cosmos-db"
8+
"github.com/crypto-org-chain/cronos/cmd/cronosd/dbmigrate"
9+
"github.com/spf13/cobra"
10+
11+
"github.com/cosmos/cosmos-sdk/client/flags"
12+
"github.com/cosmos/cosmos-sdk/server"
13+
)
14+
15+
const (
16+
flagSourceBackend = "source-backend"
17+
flagTargetBackend = "target-backend"
18+
flagTargetHome = "target-home"
19+
flagBatchSize = "batch-size"
20+
flagVerify = "verify"
21+
flagDBType = "db-type"
22+
flagDatabases = "databases"
23+
flagHeight = "height"
24+
)
25+
26+
// Database type constants
27+
const (
28+
DBTypeApp = "app"
29+
DBTypeCometBFT = "cometbft"
30+
DBTypeAll = "all"
31+
)
32+
33+
// Valid database names
34+
var validDatabaseNames = map[string]bool{
35+
"application": true,
36+
"blockstore": true,
37+
"state": true,
38+
"tx_index": true,
39+
"evidence": true,
40+
}
41+
42+
// MigrateDBCmd returns the legacy migrate-db command (for backward compatibility)
43+
func MigrateDBCmd() *cobra.Command {
44+
cmd := &cobra.Command{
45+
Use: "migrate-db",
46+
Short: "Migrate databases from one backend to another (e.g., leveldb to rocksdb)",
47+
Deprecated: "Use 'database migrate' or 'db migrate' instead",
48+
Long: `Migrate databases from one backend to another.
49+
50+
This command migrates databases from a source backend to a target backend.
51+
It can migrate the application database, CometBFT databases, or both.
52+
53+
The migration process:
54+
1. Opens the source database(s) in read-only mode
55+
2. Creates new temporary target database(s)
56+
3. Copies all key-value pairs in batches
57+
4. Optionally verifies the migration
58+
5. Creates the target database(s) in a temporary location
59+
60+
Database types (--db-type):
61+
- app: Application database only (application.db)
62+
- cometbft: CometBFT databases only (blockstore.db, state.db, tx_index.db, evidence.db)
63+
- all: Both application and CometBFT databases
64+
65+
Specific databases (--databases):
66+
You can also specify individual databases as a comma-separated list:
67+
- application: Chain state
68+
- blockstore: Block data
69+
- state: Latest state
70+
- tx_index: Transaction indexing
71+
- evidence: Misbehavior evidence
72+
73+
Height Filtering (--height):
74+
For blockstore.db and tx_index.db, you can specify heights to migrate:
75+
- Range: --height 10000-20000 (migrate heights 10000 to 20000)
76+
- Single: --height 123456 (migrate only height 123456)
77+
- Multiple: --height 123456,234567,999999 (migrate specific heights)
78+
- Only applies to blockstore and tx_index databases
79+
- Other databases will ignore height filtering
80+
81+
IMPORTANT:
82+
- Always backup your databases before migration
83+
- The source databases are opened in read-only mode and are not modified
84+
- The target databases are created with a .migrate-temp suffix
85+
- After successful migration, you need to manually replace the original databases
86+
- Stop your node before running this command
87+
88+
Examples:
89+
# Migrate application database only (using --db-type)
90+
cronosd migrate-db --source-backend goleveldb --target-backend rocksdb --db-type app --home ~/.cronos
91+
92+
# Migrate CometBFT databases only (using --db-type)
93+
cronosd migrate-db --source-backend goleveldb --target-backend rocksdb --db-type cometbft --home ~/.cronos
94+
95+
# Migrate all databases (using --db-type)
96+
cronosd migrate-db --source-backend goleveldb --target-backend rocksdb --db-type all --home ~/.cronos
97+
98+
# Migrate specific databases (using --databases)
99+
cronosd migrate-db --source-backend goleveldb --target-backend rocksdb --databases blockstore,tx_index --home ~/.cronos
100+
101+
# Migrate multiple specific databases
102+
cronosd migrate-db --source-backend goleveldb --target-backend rocksdb --databases application,blockstore,state --home ~/.cronos
103+
104+
# Migrate with verification
105+
cronosd migrate-db --source-backend goleveldb --target-backend rocksdb --db-type all --verify --home ~/.cronos
106+
107+
# Migrate blockstore with height range
108+
cronosd migrate-db --source-backend goleveldb --target-backend rocksdb --databases blockstore --height 1000000-2000000 --home ~/.cronos
109+
110+
# Migrate single block height
111+
cronosd migrate-db --source-backend goleveldb --target-backend rocksdb --databases blockstore --height 123456 --home ~/.cronos
112+
113+
# Migrate specific heights
114+
cronosd migrate-db --source-backend goleveldb --target-backend rocksdb --databases blockstore,tx_index --height 100000,200000,300000 --home ~/.cronos
115+
`,
116+
RunE: func(cmd *cobra.Command, args []string) error {
117+
ctx := server.GetServerContextFromCmd(cmd)
118+
logger := ctx.Logger
119+
120+
homeDir := ctx.Viper.GetString(flags.FlagHome)
121+
sourceBackend := ctx.Viper.GetString(flagSourceBackend)
122+
targetBackend := ctx.Viper.GetString(flagTargetBackend)
123+
targetHome := ctx.Viper.GetString(flagTargetHome)
124+
batchSize := ctx.Viper.GetInt(flagBatchSize)
125+
verify := ctx.Viper.GetBool(flagVerify)
126+
dbType := ctx.Viper.GetString(flagDBType)
127+
databases := ctx.Viper.GetString(flagDatabases)
128+
heightFlag := ctx.Viper.GetString(flagHeight)
129+
130+
// Parse backend types
131+
sourceBackendType, err := parseBackendType(sourceBackend)
132+
if err != nil {
133+
return fmt.Errorf("invalid source backend: %w", err)
134+
}
135+
136+
targetBackendType, err := parseBackendType(targetBackend)
137+
if err != nil {
138+
return fmt.Errorf("invalid target backend: %w", err)
139+
}
140+
141+
if sourceBackendType == targetBackendType {
142+
return fmt.Errorf("source and target backends must be different")
143+
}
144+
145+
if targetHome == "" {
146+
targetHome = homeDir
147+
logger.Info("Target home not specified, using source home directory", "target_home", targetHome)
148+
}
149+
150+
// Determine which databases to migrate
151+
var dbNames []string
152+
153+
// If --databases flag is provided, use it (takes precedence over --db-type)
154+
if databases != "" {
155+
var err error
156+
dbNames, err = parseDatabaseNames(databases)
157+
if err != nil {
158+
return err
159+
}
160+
} else {
161+
// Fall back to --db-type flag
162+
var err error
163+
dbNames, err = getDBNamesFromType(dbType)
164+
if err != nil {
165+
return err
166+
}
167+
}
168+
169+
// Parse height flag
170+
heightRange, err := dbmigrate.ParseHeightFlag(heightFlag)
171+
if err != nil {
172+
return fmt.Errorf("invalid height flag: %w", err)
173+
}
174+
175+
// Validate height range
176+
if err := heightRange.Validate(); err != nil {
177+
return fmt.Errorf("invalid height specification: %w", err)
178+
}
179+
180+
// Warn if height specification is provided but not applicable
181+
if !heightRange.IsEmpty() {
182+
hasHeightSupport := false
183+
for _, dbName := range dbNames {
184+
if dbName == "blockstore" || dbName == "tx_index" {
185+
hasHeightSupport = true
186+
break
187+
}
188+
}
189+
if !hasHeightSupport {
190+
logger.Warn("Height specification provided but will be ignored (only applies to blockstore and tx_index databases)",
191+
"databases", dbNames,
192+
"height", heightRange.String(),
193+
)
194+
}
195+
}
196+
197+
logArgs := []interface{}{
198+
"source_home", homeDir,
199+
"target_home", targetHome,
200+
"source_backend", sourceBackend,
201+
"target_backend", targetBackend,
202+
"databases", dbNames,
203+
"batch_size", batchSize,
204+
"verify", verify,
205+
}
206+
if !heightRange.IsEmpty() {
207+
logArgs = append(logArgs, "height_range", heightRange.String())
208+
}
209+
logger.Info("Database migration configuration", logArgs...)
210+
211+
// Prepare RocksDB options if target is RocksDB
212+
var rocksDBOpts interface{}
213+
if targetBackendType == dbm.RocksDBBackend {
214+
// Use the same RocksDB options as the application (implemented in build-tagged files)
215+
rocksDBOpts = dbmigrate.PrepareRocksDBOptions()
216+
}
217+
218+
// Migrate each database
219+
var totalStats dbmigrate.MigrationStats
220+
for _, dbName := range dbNames {
221+
logger.Info("Starting migration", "database", dbName)
222+
223+
opts := dbmigrate.MigrateOptions{
224+
SourceHome: homeDir,
225+
TargetHome: targetHome,
226+
SourceBackend: sourceBackendType,
227+
TargetBackend: targetBackendType,
228+
BatchSize: batchSize,
229+
Logger: logger,
230+
RocksDBOptions: rocksDBOpts,
231+
Verify: verify,
232+
DBName: dbName,
233+
HeightRange: heightRange,
234+
}
235+
236+
stats, err := dbmigrate.Migrate(opts)
237+
if err != nil {
238+
logger.Error("Migration failed",
239+
"database", dbName,
240+
"error", err,
241+
"processed_keys", stats.ProcessedKeys.Load(),
242+
"total_keys", stats.TotalKeys.Load(),
243+
"duration", stats.Duration(),
244+
)
245+
return fmt.Errorf("failed to migrate %s: %w", dbName, err)
246+
}
247+
248+
logger.Info("Database migration completed",
249+
"database", dbName,
250+
"total_keys", stats.TotalKeys.Load(),
251+
"processed_keys", stats.ProcessedKeys.Load(),
252+
"errors", stats.ErrorCount.Load(),
253+
"duration", stats.Duration(),
254+
)
255+
256+
// Accumulate stats
257+
totalStats.TotalKeys.Add(stats.TotalKeys.Load())
258+
totalStats.ProcessedKeys.Add(stats.ProcessedKeys.Load())
259+
totalStats.ErrorCount.Add(stats.ErrorCount.Load())
260+
}
261+
262+
fmt.Println("\n" + strings.Repeat("=", 80))
263+
fmt.Println("ALL MIGRATIONS COMPLETED SUCCESSFULLY")
264+
fmt.Println(strings.Repeat("=", 80))
265+
if databases != "" {
266+
fmt.Printf("Databases: %s\n", strings.Join(dbNames, ", "))
267+
} else {
268+
fmt.Printf("Database Type: %s\n", dbType)
269+
}
270+
fmt.Printf("Total Keys: %d\n", totalStats.TotalKeys.Load())
271+
fmt.Printf("Processed Keys: %d\n", totalStats.ProcessedKeys.Load())
272+
fmt.Printf("Errors: %d\n", totalStats.ErrorCount.Load())
273+
fmt.Println("\nIMPORTANT NEXT STEPS:")
274+
fmt.Println("1. Backup your original databases")
275+
fmt.Println("2. Verify the migration was successful")
276+
fmt.Println("3. Migrated databases are located at:")
277+
for _, dbName := range dbNames {
278+
fmt.Printf(" %s/data/%s.db.migrate-temp\n", targetHome, dbName)
279+
}
280+
fmt.Println("4. Replace the original databases with the migrated ones")
281+
fmt.Println("5. Update your config.toml to use the new backend type")
282+
fmt.Println(strings.Repeat("=", 80))
283+
284+
return nil
285+
},
286+
}
287+
288+
cmd.Flags().StringP(flagSourceBackend, "s", "goleveldb", "Source database backend type (goleveldb, rocksdb)")
289+
cmd.Flags().StringP(flagTargetBackend, "t", "rocksdb", "Target database backend type (goleveldb, rocksdb)")
290+
cmd.Flags().StringP(flagTargetHome, "o", "", "Target home directory (default: same as --home)")
291+
cmd.Flags().IntP(flagBatchSize, "b", dbmigrate.DefaultBatchSize, "Number of key-value pairs to process in a batch")
292+
cmd.Flags().BoolP(flagVerify, "v", true, "Verify migration by comparing source and target databases")
293+
cmd.Flags().StringP(flagDBType, "y", DBTypeApp, "Database type to migrate: app (application.db only), cometbft (CometBFT databases only), all (both)")
294+
cmd.Flags().StringP(flagDatabases, "d", "", "Comma-separated list of specific databases to migrate (e.g., 'blockstore,tx_index'). Valid names: application, blockstore, state, tx_index, evidence. If specified, this flag takes precedence over --db-type")
295+
cmd.Flags().StringP(flagHeight, "H", "", "Height specification for blockstore/tx_index: range (10000-20000), single (123456), or multiple (123456,234567,999999). Only applies to blockstore and tx_index databases")
296+
297+
return cmd
298+
}
299+
300+
// MigrateCmd returns the migrate subcommand for the database command group.
301+
// The returned command is configured with the "migrate" usage and is not marked as deprecated.
302+
func MigrateCmd() *cobra.Command {
303+
cmd := MigrateDBCmd()
304+
cmd.Use = "migrate"
305+
cmd.Deprecated = ""
306+
return cmd
307+
}
308+
309+
// parseBackendType maps a backend name to the corresponding dbm.BackendType.
310+
// It accepts `goleveldb` or `leveldb` for dbm.GoLevelDBBackend and `rocksdb` for dbm.RocksDBBackend; returns an error for any other value.
311+
func parseBackendType(backend string) (dbm.BackendType, error) {
312+
switch backend {
313+
case "goleveldb", "leveldb":
314+
return dbm.GoLevelDBBackend, nil
315+
case "rocksdb":
316+
return dbm.RocksDBBackend, nil
317+
default:
318+
return "", fmt.Errorf("unsupported backend type: %s (supported: goleveldb, rocksdb)", backend)
319+
}
320+
}
321+
322+
// parseDatabaseNames parses and validates a comma-separated list of database names.
323+
// It returns a slice of validated database names in the order provided, or an error if
324+
// the input is empty, contains no valid names, or includes an unsupported name.
325+
func parseDatabaseNames(databases string) ([]string, error) {
326+
if databases == "" {
327+
return nil, fmt.Errorf("no databases specified")
328+
}
329+
330+
dbList := strings.Split(databases, ",")
331+
var dbNames []string
332+
for _, dbName := range dbList {
333+
dbName = strings.TrimSpace(dbName)
334+
if dbName == "" {
335+
continue
336+
}
337+
if !validDatabaseNames[dbName] {
338+
return nil, fmt.Errorf("invalid database name: %s (valid names: application, blockstore, state, tx_index, evidence)", dbName)
339+
}
340+
dbNames = append(dbNames, dbName)
341+
}
342+
if len(dbNames) == 0 {
343+
return nil, fmt.Errorf("no valid databases specified in --databases flag")
344+
}
345+
return dbNames, nil
346+
}
347+
348+
// getDBNamesFromType maps a db-type string to the corresponding list of database names to migrate.
349+
// Supported db-type values are "app", "cometbft", and "all"; any other value returns an error describing valid options.
350+
func getDBNamesFromType(dbType string) ([]string, error) {
351+
switch dbType {
352+
case DBTypeApp:
353+
return []string{"application"}, nil
354+
case DBTypeCometBFT:
355+
return []string{"blockstore", "state", "tx_index", "evidence"}, nil
356+
case DBTypeAll:
357+
return []string{"application", "blockstore", "state", "tx_index", "evidence"}, nil
358+
default:
359+
return nil, fmt.Errorf("invalid db-type: %s (must be: app, cometbft, or all)", dbType)
360+
}
361+
}

0 commit comments

Comments
 (0)