diff --git a/.gitignore b/.gitignore index 03f99b00..bc2ca2d8 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,16 @@ pnpm-debug.log* .env.test.local .env.production.local +# Build info +*.tsbuildinfo + +# Claude Code +.claude/settings.local.json + +# BetterDB context +.betterdb_context.md +**/.betterdb_context.md + # Turbo .turbo diff --git a/Dockerfile b/Dockerfile index 74b0d890..eb6a506f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,6 +76,18 @@ ENV NODE_ENV=production ENV PORT=3001 ENV STORAGE_TYPE=memory +# Install RedisShake binary for migration execution (with checksum verification) +ARG TARGETARCH +ARG REDISSHAKE_VERSION=4.6.0 +RUN REDISSHAKE_SHA256_AMD64="6ccab1ff2ba3c200950f8ada811f0c6fe6e2f5e6bd3b8e92b4d9444dc0aff4df" && \ + REDISSHAKE_SHA256_ARM64="653298efa83ef3d495ae2ec21b40c773f36eb15e507f8b3f2931660509d09690" && \ + if [ "${TARGETARCH}" = "amd64" ]; then EXPECTED_SHA256="${REDISSHAKE_SHA256_AMD64}"; else EXPECTED_SHA256="${REDISSHAKE_SHA256_ARM64}"; fi && \ + wget -qO /tmp/redis-shake.tar.gz "https://github.com/tair-opensource/RedisShake/releases/download/v${REDISSHAKE_VERSION}/redis-shake-v${REDISSHAKE_VERSION}-linux-${TARGETARCH}.tar.gz" && \ + echo "${EXPECTED_SHA256} /tmp/redis-shake.tar.gz" | sha256sum -c - && \ + tar -xzf /tmp/redis-shake.tar.gz --strip-components=0 -C /usr/local/bin ./redis-shake && \ + chmod +x /usr/local/bin/redis-shake && \ + rm /tmp/redis-shake.tar.gz + # Create non-root user for security (Docker Scout compliance) RUN addgroup --system --gid 1001 nodejs && \ adduser --system --uid 1001 --ingroup nodejs betterdb diff --git a/Dockerfile.prod b/Dockerfile.prod index bc5bbf87..c843ecdd 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -154,6 +154,20 @@ ENV DB_USERNAME=default ENV STORAGE_TYPE=memory ENV AI_ENABLED=false +# Install RedisShake binary for migration execution (with checksum verification) +ARG TARGETARCH +ARG REDISSHAKE_VERSION=4.6.0 +RUN apk add --no-cache wget && \ + REDISSHAKE_SHA256_AMD64="6ccab1ff2ba3c200950f8ada811f0c6fe6e2f5e6bd3b8e92b4d9444dc0aff4df" && \ + REDISSHAKE_SHA256_ARM64="653298efa83ef3d495ae2ec21b40c773f36eb15e507f8b3f2931660509d09690" && \ + if [ "${TARGETARCH}" = "amd64" ]; then EXPECTED_SHA256="${REDISSHAKE_SHA256_AMD64}"; else EXPECTED_SHA256="${REDISSHAKE_SHA256_ARM64}"; fi && \ + wget -qO /tmp/redis-shake.tar.gz "https://github.com/tair-opensource/RedisShake/releases/download/v${REDISSHAKE_VERSION}/redis-shake-v${REDISSHAKE_VERSION}-linux-${TARGETARCH}.tar.gz" && \ + echo "${EXPECTED_SHA256} /tmp/redis-shake.tar.gz" | sha256sum -c - && \ + tar -xzf /tmp/redis-shake.tar.gz --strip-components=0 -C /usr/local/bin ./redis-shake && \ + chmod +x /usr/local/bin/redis-shake && \ + rm /tmp/redis-shake.tar.gz && \ + apk del wget + # Create non-root user for security (Docker Scout compliance) RUN addgroup --system --gid 1001 nodejs && \ adduser --system --uid 1001 --ingroup nodejs betterdb diff --git a/apps/api/package.json b/apps/api/package.json index fb8db27a..5d54a58f 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -26,6 +26,7 @@ "test:integration:redis": "TEST_DB_PORT=6392 jest test/database-compatibility.e2e-spec.ts", "test:integration:valkey": "TEST_DB_PORT=6390 jest --testRegex='.e2e-spec.ts$'", "test:cluster": "jest test/api-cluster.e2e-spec.ts", + "test:migration-topology": "RUN_TOPOLOGY_TESTS=true jest test/migration-topology.e2e-spec.ts", "test:cluster:unit": "jest src/cluster/*.spec.ts", "test:integration:cluster": "TEST_DB_HOST=localhost TEST_DB_PORT=7001 jest test/api-cluster.e2e-spec.ts", "test:unit:parsers": "jest src/database/parsers/*.spec.ts", diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts index 260e9391..31d38dcb 100644 --- a/apps/api/src/app.module.ts +++ b/apps/api/src/app.module.ts @@ -16,6 +16,7 @@ import { SettingsModule } from './settings/settings.module'; import { WebhooksModule } from './webhooks/webhooks.module'; import { TelemetryModule } from './telemetry/telemetry.module'; import { VectorSearchModule } from './vector-search/vector-search.module'; +import { MigrationModule } from './migration/migration.module'; import { CloudAuthModule } from './auth/cloud-auth.module'; import { McpModule } from './mcp/mcp.module'; import { MetricForecastingModule } from './metric-forecasting/metric-forecasting.module'; @@ -120,6 +121,7 @@ const baseImports = [ WebhooksModule, McpModule, VectorSearchModule, + MigrationModule, MetricForecastingModule, ]; diff --git a/apps/api/src/migration/__tests__/commandlog-analyzer.spec.ts b/apps/api/src/migration/__tests__/commandlog-analyzer.spec.ts new file mode 100644 index 00000000..fb91e819 --- /dev/null +++ b/apps/api/src/migration/__tests__/commandlog-analyzer.spec.ts @@ -0,0 +1,133 @@ +import { analyzeCommands } from '../analysis/commandlog-analyzer'; +import type { DatabasePort, DatabaseCapabilities } from '../../common/interfaces/database-port.interface'; + +function createMockAdapter(options: { + hasCommandLog?: boolean; + commandLogEntries?: Array<{ command: string[] }>; + slowLogEntries?: Array<{ command: string[] }>; + commandLogError?: boolean; + slowLogError?: boolean; +} = {}): DatabasePort { + const { + hasCommandLog = false, + commandLogEntries = [], + slowLogEntries = [], + commandLogError = false, + slowLogError = false, + } = options; + + return { + getCapabilities: jest.fn().mockReturnValue({ + hasCommandLog, + } as Partial), + getCommandLog: jest.fn().mockImplementation(() => { + if (commandLogError) return Promise.reject(new Error('COMMANDLOG failed')); + return Promise.resolve(commandLogEntries); + }), + getSlowLog: jest.fn().mockImplementation(() => { + if (slowLogError) return Promise.reject(new Error('SLOWLOG failed')); + return Promise.resolve(slowLogEntries); + }), + } as unknown as DatabasePort; +} + +describe('analyzeCommands', () => { + it('should return top commands from COMMANDLOG when available', async () => { + const adapter = createMockAdapter({ + hasCommandLog: true, + commandLogEntries: [ + { command: ['SET', 'key1', 'val'] }, + { command: ['SET', 'key2', 'val'] }, + { command: ['GET', 'key1'] }, + { command: ['SET', 'key3', 'val'] }, + ], + }); + + const result = await analyzeCommands(adapter); + + expect(result.sourceUsed).toBe('commandlog'); + expect(result.topCommands).toHaveLength(2); + expect(result.topCommands[0]).toEqual({ command: 'SET', count: 3 }); + expect(result.topCommands[1]).toEqual({ command: 'GET', count: 1 }); + }); + + it('should fall back to SLOWLOG when COMMANDLOG is not available', async () => { + const adapter = createMockAdapter({ + hasCommandLog: false, + slowLogEntries: [ + { command: ['HGETALL', 'myhash'] }, + { command: ['HGETALL', 'myhash2'] }, + { command: ['ZADD', 'myset', '1', 'a'] }, + ], + }); + + const result = await analyzeCommands(adapter); + + expect(result.sourceUsed).toBe('slowlog'); + expect(result.topCommands[0]).toEqual({ command: 'HGETALL', count: 2 }); + expect(result.topCommands[1]).toEqual({ command: 'ZADD', count: 1 }); + }); + + it('should fall back to SLOWLOG when COMMANDLOG errors', async () => { + const adapter = createMockAdapter({ + hasCommandLog: true, + commandLogError: true, + slowLogEntries: [ + { command: ['INFO', 'all'] }, + ], + }); + + const result = await analyzeCommands(adapter); + + expect(result.sourceUsed).toBe('slowlog'); + expect(result.topCommands).toHaveLength(1); + expect(result.topCommands[0].command).toBe('INFO'); + }); + + it('should return empty topCommands when both logs are empty', async () => { + const adapter = createMockAdapter({ + hasCommandLog: true, + commandLogEntries: [], + }); + + const result = await analyzeCommands(adapter); + + expect(result.sourceUsed).toBe('commandlog'); + expect(result.topCommands).toEqual([]); + }); + + it('should return unavailable when both sources fail', async () => { + const adapter = createMockAdapter({ + hasCommandLog: false, + slowLogError: true, + }); + + const result = await analyzeCommands(adapter); + + expect(result.sourceUsed).toBe('unavailable'); + expect(result.topCommands).toEqual([]); + }); + + it('should sort commands by count descending', async () => { + const adapter = createMockAdapter({ + hasCommandLog: true, + commandLogEntries: [ + { command: ['GET', 'a'] }, + { command: ['SET', 'a', '1'] }, + { command: ['SET', 'b', '2'] }, + { command: ['SET', 'c', '3'] }, + { command: ['GET', 'b'] }, + { command: ['DEL', 'a'] }, + ], + }); + + const result = await analyzeCommands(adapter); + + expect(result.topCommands[0].command).toBe('SET'); + expect(result.topCommands[0].count).toBe(3); + expect(result.topCommands[1].command).toBe('GET'); + expect(result.topCommands[1].count).toBe(2); + expect(result.topCommands[2].command).toBe('DEL'); + expect(result.topCommands[2].count).toBe(1); + }); +}); diff --git a/apps/api/src/migration/__tests__/compatibility-checker.spec.ts b/apps/api/src/migration/__tests__/compatibility-checker.spec.ts new file mode 100644 index 00000000..411bb59c --- /dev/null +++ b/apps/api/src/migration/__tests__/compatibility-checker.spec.ts @@ -0,0 +1,221 @@ +import { buildInstanceMeta, checkCompatibility, InstanceMeta } from '../analysis/compatibility-checker'; +import type { DatabaseCapabilities } from '../../common/interfaces/database-port.interface'; + +function makeMeta(overrides: Partial = {}): InstanceMeta { + return { + dbType: 'valkey', + version: '8.1.0', + capabilities: { dbType: 'valkey', version: '8.1.0' } as DatabaseCapabilities, + clusterEnabled: false, + databases: [0], + modules: [], + maxmemoryPolicy: 'noeviction', + hasAclUsers: false, + persistenceMode: 'rdb', + ...overrides, + }; +} + +describe('compatibility-checker', () => { + describe('buildInstanceMeta', () => { + it('should parse INFO fields correctly', () => { + const info: Record = { + cluster_enabled: '1', + db0: 'keys=100,expires=10', + db3: 'keys=50,expires=5', + maxmemory_policy: 'allkeys-lru', + aof_enabled: '1', + }; + const capabilities: DatabaseCapabilities = { + dbType: 'valkey', + version: '8.1.0', + hasCommandLog: true, + hasSlotStats: false, + hasClusterSlotStats: false, + hasLatencyMonitor: true, + hasAclLog: true, + hasMemoryDoctor: true, + hasConfig: true, + hasVectorSearch: false, + }; + + const meta = buildInstanceMeta(info, capabilities, ['default', 'admin'], '3600 1 300 100'); + + expect(meta.dbType).toBe('valkey'); + expect(meta.version).toBe('8.1.0'); + expect(meta.clusterEnabled).toBe(true); + expect(meta.databases).toEqual(expect.arrayContaining([0, 3])); + expect(meta.maxmemoryPolicy).toBe('allkeys-lru'); + expect(meta.hasAclUsers).toBe(true); + expect(meta.persistenceMode).toBe('rdb+aof'); + }); + + it('should default to db0 when no keyspace databases found', () => { + const meta = buildInstanceMeta( + {}, + { dbType: 'redis', version: '7.2.0' } as DatabaseCapabilities, + ['default'], + ); + expect(meta.databases).toEqual([0]); + }); + + it('should detect RDB-only persistence via CONFIG save schedule', () => { + const meta = buildInstanceMeta( + { aof_enabled: '0' }, + { dbType: 'valkey', version: '8.0.0' } as DatabaseCapabilities, + [], + '3600 1 300 100', + ); + expect(meta.persistenceMode).toBe('rdb'); + }); + + it('should not detect RDB when CONFIG save is empty', () => { + const meta = buildInstanceMeta( + { aof_enabled: '0' }, + { dbType: 'valkey', version: '8.0.0' } as DatabaseCapabilities, + [], + '', + ); + expect(meta.persistenceMode).toBe('none'); + }); + + it('should fall back to rdb_bgsave_in_progress when CONFIG not available', () => { + const meta = buildInstanceMeta( + { rdb_bgsave_in_progress: '1', aof_enabled: '0' }, + { dbType: 'valkey', version: '8.0.0' } as DatabaseCapabilities, + [], + ); + expect(meta.persistenceMode).toBe('rdb'); + }); + + it('should detect AOF-only persistence', () => { + const meta = buildInstanceMeta( + { aof_enabled: '1' }, + { dbType: 'valkey', version: '8.0.0' } as DatabaseCapabilities, + [], + ); + expect(meta.persistenceMode).toBe('aof'); + }); + + it('should detect no persistence', () => { + const meta = buildInstanceMeta( + {}, + { dbType: 'valkey', version: '8.0.0' } as DatabaseCapabilities, + [], + ); + expect(meta.persistenceMode).toBe('none'); + }); + + it('should not flag ACL users when only default exists', () => { + const meta = buildInstanceMeta( + {}, + { dbType: 'valkey', version: '8.0.0' } as DatabaseCapabilities, + ['default'], + ); + expect(meta.hasAclUsers).toBe(false); + }); + }); + + describe('checkCompatibility', () => { + it('should return no issues for Valkey→Valkey same version', () => { + const source = makeMeta(); + const target = makeMeta(); + const issues = checkCompatibility(source, target, false); + expect(issues).toEqual([]); + }); + + it('should return HFE warning when hfeDetected and target does not support HFE', () => { + const source = makeMeta({ dbType: 'redis', version: '7.2.0', capabilities: { dbType: 'redis', version: '7.2.0' } as DatabaseCapabilities }); + const target = makeMeta({ dbType: 'valkey', version: '7.2.0', capabilities: { dbType: 'valkey', version: '7.2.0' } as DatabaseCapabilities }); + const issues = checkCompatibility(source, target, true); + const hfeIssue = issues.find(i => i.category === 'hfe'); + expect(hfeIssue).toBeDefined(); + expect(hfeIssue!.severity).toBe('blocking'); + }); + + it('should not flag HFE when target supports it (Valkey 8.1+)', () => { + const source = makeMeta(); + const target = makeMeta({ dbType: 'valkey', version: '8.1.0' }); + const issues = checkCompatibility(source, target, true); + const hfeIssue = issues.find(i => i.category === 'hfe'); + expect(hfeIssue).toBeUndefined(); + }); + + it('should return modules blocking issue when source has modules target does not', () => { + const source = makeMeta({ modules: ['search', 'json'] }); + const target = makeMeta({ modules: [] }); + const issues = checkCompatibility(source, target, false); + const moduleIssues = issues.filter(i => i.category === 'modules'); + expect(moduleIssues).toHaveLength(2); + expect(moduleIssues[0].severity).toBe('blocking'); + }); + + it('should return multi_db blocking issue when source is multi-DB and target is cluster', () => { + const source = makeMeta({ databases: [0, 1, 2] }); + const target = makeMeta({ clusterEnabled: true }); + const issues = checkCompatibility(source, target, false); + const multiDbIssue = issues.find(i => i.category === 'multi_db' && i.severity === 'blocking'); + expect(multiDbIssue).toBeDefined(); + }); + + it('should return multi_db warning when source multi-DB target standalone without matching DBs', () => { + const source = makeMeta({ databases: [0, 1] }); + const target = makeMeta({ databases: [0] }); + const issues = checkCompatibility(source, target, false); + const multiDbWarn = issues.find(i => i.category === 'multi_db' && i.severity === 'warning'); + expect(multiDbWarn).toBeDefined(); + }); + + it('should return cluster_topology blocking issue for cluster→standalone', () => { + const source = makeMeta({ clusterEnabled: true }); + const target = makeMeta({ clusterEnabled: false }); + const issues = checkCompatibility(source, target, false); + const clusterIssue = issues.find(i => i.category === 'cluster_topology' && i.severity === 'blocking'); + expect(clusterIssue).toBeDefined(); + }); + + it('should return cluster_topology warning for standalone→cluster', () => { + const source = makeMeta({ clusterEnabled: false }); + const target = makeMeta({ clusterEnabled: true }); + const issues = checkCompatibility(source, target, false); + const clusterIssue = issues.find(i => i.category === 'cluster_topology' && i.severity === 'warning'); + expect(clusterIssue).toBeDefined(); + }); + + it('should return acl warning when source has ACL users and target does not', () => { + const source = makeMeta({ hasAclUsers: true }); + const target = makeMeta({ hasAclUsers: false }); + const issues = checkCompatibility(source, target, false); + const aclIssue = issues.find(i => i.category === 'acl'); + expect(aclIssue).toBeDefined(); + expect(aclIssue!.severity).toBe('warning'); + }); + + it('should return persistence info when modes differ', () => { + const source = makeMeta({ persistenceMode: 'rdb' }); + const target = makeMeta({ persistenceMode: 'aof' }); + const issues = checkCompatibility(source, target, false); + const persistIssue = issues.find(i => i.category === 'persistence'); + expect(persistIssue).toBeDefined(); + expect(persistIssue!.severity).toBe('info'); + }); + + it('should return type_direction blocking issue for Valkey→Redis', () => { + const source = makeMeta({ dbType: 'valkey' }); + const target = makeMeta({ dbType: 'redis', capabilities: { dbType: 'redis', version: '7.2.0' } as DatabaseCapabilities }); + const issues = checkCompatibility(source, target, false); + const dirIssue = issues.find(i => i.category === 'type_direction'); + expect(dirIssue).toBeDefined(); + expect(dirIssue!.severity).toBe('blocking'); + }); + + it('should return eviction policy mismatch warning', () => { + const source = makeMeta({ maxmemoryPolicy: 'noeviction' }); + const target = makeMeta({ maxmemoryPolicy: 'allkeys-lru' }); + const issues = checkCompatibility(source, target, false); + const evictionIssue = issues.find(i => i.category === 'maxmemory_policy'); + expect(evictionIssue).toBeDefined(); + expect(evictionIssue!.severity).toBe('warning'); + }); + }); +}); diff --git a/apps/api/src/migration/__tests__/hfe-detector.spec.ts b/apps/api/src/migration/__tests__/hfe-detector.spec.ts new file mode 100644 index 00000000..11ea9e34 --- /dev/null +++ b/apps/api/src/migration/__tests__/hfe-detector.spec.ts @@ -0,0 +1,130 @@ +import { detectHfe } from '../analysis/hfe-detector'; + +function createMockClient(options: { + hlens?: Record; + hrandfields?: Record; + hexpiretimes?: Record; + hexpireError?: Error; +} = {}) { + const { hlens = {}, hrandfields = {}, hexpiretimes = {}, hexpireError } = options; + + const pipelineMock = (): Record => { + const calls: Array<{ method: string; args: unknown[] }> = []; + + const self: Record = { + hlen: jest.fn((...args: unknown[]) => { calls.push({ method: 'hlen', args }); return self; }), + call: jest.fn((...args: unknown[]) => { calls.push({ method: 'call', args }); return self; }), + exec: jest.fn().mockImplementation(() => { + const results = calls.map(c => { + if (c.method === 'hlen') { + const key = c.args[0] as string; + return [null, hlens[key] ?? 5]; + } + if (c.method === 'call') { + const cmd = String(c.args[0]).toUpperCase(); + if (cmd === 'HRANDFIELD') { + const key = c.args[1] as string; + return [null, hrandfields[key] ?? ['field1']]; + } + if (cmd === 'HEXPIRETIME') { + if (hexpireError) return [hexpireError, null]; + const key = c.args[1] as string; + const field = c.args[4] as string; + const times = hexpiretimes[key]; + if (times) { + // Return the time for the specific field (just use first available) + return [null, [times[0] ?? -1]]; + } + return [null, [-1]]; + } + } + return [null, null]; + }); + return Promise.resolve(results); + }), + }; + return self; + }; + + return { + pipeline: jest.fn().mockImplementation(pipelineMock), + } as any; +} + +describe('detectHfe', () => { + it('should return hfeDetected: false when no hash keys provided', async () => { + const client = createMockClient(); + const result = await detectHfe(client, [], 0); + + expect(result.hfeDetected).toBe(false); + expect(result.sampledHashCount).toBe(0); + }); + + it('should detect HFE when HEXPIRETIME returns > 0', async () => { + const client = createMockClient({ + hlens: { 'hash:1': 5 }, + hrandfields: { 'hash:1': ['f1'] }, + hexpiretimes: { 'hash:1': [1700000000] }, + }); + + const result = await detectHfe(client, ['hash:1'], 100); + + expect(result.hfeDetected).toBe(true); + expect(result.hfeSupported).toBe(true); + expect(result.sampledHashCount).toBe(1); + }); + + it('should return hfeDetected: false when HEXPIRETIME returns -1', async () => { + const client = createMockClient({ + hlens: { 'hash:1': 5 }, + hrandfields: { 'hash:1': ['f1'] }, + hexpiretimes: { 'hash:1': [-1] }, + }); + + const result = await detectHfe(client, ['hash:1'], 100); + + expect(result.hfeDetected).toBe(false); + expect(result.hfeSupported).toBe(true); + }); + + it('should skip oversized hashes (> 10K fields)', async () => { + const client = createMockClient({ + hlens: { 'hash:big': 20_000, 'hash:small': 5 }, + hrandfields: { 'hash:small': ['f1'] }, + hexpiretimes: { 'hash:small': [-1] }, + }); + + const result = await detectHfe(client, ['hash:big', 'hash:small'], 200); + + expect(result.hfeOversizedHashesSkipped).toBe(1); + expect(result.sampledHashCount).toBe(1); + }); + + it('should set hfeSupported: false when HEXPIRETIME command errors', async () => { + const client = createMockClient({ + hlens: { 'hash:1': 5 }, + hrandfields: { 'hash:1': ['f1'] }, + hexpireError: new Error('ERR unknown command `HEXPIRETIME`'), + }); + + const result = await detectHfe(client, ['hash:1'], 100); + + expect(result.hfeSupported).toBe(false); + expect(result.hfeDetected).toBe(false); + }); + + it('should estimate hfeKeyCount from sample ratio', async () => { + const client = createMockClient({ + hlens: { 'h:1': 5, 'h:2': 5 }, + hrandfields: { 'h:1': ['f1'], 'h:2': ['f2'] }, + hexpiretimes: { 'h:1': [1700000000], 'h:2': [-1] }, + }); + + const result = await detectHfe(client, ['h:1', 'h:2'], 1000); + + expect(result.hfeDetected).toBe(true); + // 1 out of 2 valid keys had HFE, total estimated = 1000 + // hfeKeyCount = round((1/2) * 1000) = 500 + expect(result.hfeKeyCount).toBe(500); + }); +}); diff --git a/apps/api/src/migration/__tests__/migration-execution.service.spec.ts b/apps/api/src/migration/__tests__/migration-execution.service.spec.ts new file mode 100644 index 00000000..bafca5e4 --- /dev/null +++ b/apps/api/src/migration/__tests__/migration-execution.service.spec.ts @@ -0,0 +1,219 @@ +import { MigrationExecutionService } from '../migration-execution.service'; +import { BadRequestException, NotFoundException, ServiceUnavailableException } from '@nestjs/common'; + +jest.mock('../execution/redisshake-runner', () => ({ + findRedisShakeBinary: jest.fn().mockReturnValue('/usr/local/bin/redis-shake'), +})); + +jest.mock('../execution/toml-builder', () => ({ + buildScanReaderToml: jest.fn().mockReturnValue('[scan_reader]\naddress = "127.0.0.1:6379"\n'), +})); + +jest.mock('child_process', () => ({ + spawn: jest.fn().mockReturnValue({ + stdout: { on: jest.fn() }, + stderr: { on: jest.fn() }, + on: jest.fn().mockImplementation((event: string, cb: (code: number) => void) => { + if (event === 'exit') setTimeout(() => cb(0), 10); + }), + kill: jest.fn(), + pid: 12345, + }), +})); + +jest.mock('fs', () => ({ + writeFileSync: jest.fn(), + unlinkSync: jest.fn(), + existsSync: jest.fn().mockReturnValue(true), +})); + +jest.mock('../execution/command-migration-worker', () => ({ + runCommandMigration: jest.fn().mockResolvedValue(undefined), +})); + +function createMockRegistry(overrides?: { sourceClusterEnabled?: boolean; targetClusterEnabled?: boolean }) { + const sourceCluster = overrides?.sourceClusterEnabled ?? false; + const targetCluster = overrides?.targetClusterEnabled ?? false; + + const mockSourceAdapter = { + getCapabilities: jest.fn().mockReturnValue({ dbType: 'valkey', version: '8.1.0' }), + getInfo: jest.fn().mockResolvedValue({ cluster: { cluster_enabled: sourceCluster ? '1' : '0' } }), + getClient: jest.fn().mockReturnValue({ quit: jest.fn() }), + }; + const mockTargetAdapter = { + getCapabilities: jest.fn().mockReturnValue({ dbType: 'valkey', version: '8.1.0' }), + getInfo: jest.fn().mockResolvedValue({ cluster: { cluster_enabled: targetCluster ? '1' : '0' } }), + getClient: jest.fn().mockReturnValue({ quit: jest.fn() }), + }; + + const adapters: Record = { + 'conn-1': mockSourceAdapter, + 'conn-2': mockTargetAdapter, + }; + + return { + get: jest.fn().mockImplementation((id: string) => adapters[id] ?? mockSourceAdapter), + getConfig: jest.fn().mockReturnValue({ + id: 'conn-1', + name: 'Test', + host: '127.0.0.1', + port: 6379, + createdAt: Date.now(), + }), + mockSourceAdapter, + mockTargetAdapter, + }; +} + +describe('MigrationExecutionService', () => { + let service: MigrationExecutionService; + let registry: ReturnType; + + beforeEach(() => { + registry = createMockRegistry(); + service = new MigrationExecutionService(registry as any); + }); + + describe('startExecution', () => { + it('should return a job ID with pending status', async () => { + const result = await service.startExecution({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + + expect(result.id).toBeDefined(); + expect(result.status).toBe('pending'); + }); + + it('should make the job retrievable via getExecution', async () => { + const { id } = await service.startExecution({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + + const exec = service.getExecution(id); + expect(exec).toBeDefined(); + expect(exec!.id).toBe(id); + }); + + it('should reject same source and target', async () => { + await expect( + service.startExecution({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-1', + }), + ).rejects.toThrow(BadRequestException); + }); + + it('should throw NotFoundException when connection does not exist', async () => { + registry.get.mockImplementation((id: string) => { + if (id === 'missing') throw new NotFoundException(); + return { getCapabilities: jest.fn(), getInfo: jest.fn().mockResolvedValue({}), getClient: jest.fn() }; + }); + + await expect( + service.startExecution({ + sourceConnectionId: 'missing', + targetConnectionId: 'conn-2', + }), + ).rejects.toThrow(NotFoundException); + }); + + it('should pass targetIsCluster: true when target reports cluster_enabled=1', async () => { + const { runCommandMigration } = require('../execution/command-migration-worker'); + + const clusterRegistry = createMockRegistry({ targetClusterEnabled: true }); + const clusterService = new MigrationExecutionService(clusterRegistry as any); + + await clusterService.startExecution({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + mode: 'command', + }); + + // Wait a tick for the async runCommandMode to call runCommandMigration + await new Promise(r => setTimeout(r, 20)); + + expect(runCommandMigration).toHaveBeenCalledWith( + expect.objectContaining({ targetIsCluster: true }), + ); + }); + + it('should pass targetIsCluster: false when target is standalone', async () => { + const { runCommandMigration } = require('../execution/command-migration-worker'); + (runCommandMigration as jest.Mock).mockClear(); + + await service.startExecution({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + mode: 'command', + }); + + await new Promise(r => setTimeout(r, 20)); + + expect(runCommandMigration).toHaveBeenCalledWith( + expect.objectContaining({ targetIsCluster: false }), + ); + }); + }); + + describe('stopExecution', () => { + it('should cancel a running job', async () => { + const { id } = await service.startExecution({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + + const result = service.stopExecution(id); + expect(result).toBe(true); + + const exec = service.getExecution(id); + expect(exec!.status).toBe('cancelled'); + }); + + it('should return false for unknown job ID', () => { + expect(service.stopExecution('nonexistent')).toBe(false); + }); + + it('should be idempotent for terminal states', async () => { + const { id } = await service.startExecution({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + service.stopExecution(id); + + // Call again — should still return true + expect(service.stopExecution(id)).toBe(true); + }); + }); + + describe('getExecution', () => { + it('should return undefined for unknown job ID', () => { + expect(service.getExecution('nonexistent')).toBeUndefined(); + }); + }); + + describe('job eviction', () => { + it('should evict oldest completed jobs when MAX_JOBS (10) reached', async () => { + const ids: string[] = []; + for (let i = 0; i < 10; i++) { + const { id } = await service.startExecution({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + ids.push(id); + service.stopExecution(id); // Mark as cancelled (terminal) + } + + // One more should trigger eviction + const { id: newId } = await service.startExecution({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + + expect(service.getExecution(newId)).toBeDefined(); + // Oldest should be evicted + expect(service.getExecution(ids[0])).toBeUndefined(); + }); + }); +}); diff --git a/apps/api/src/migration/__tests__/migration-validation.service.spec.ts b/apps/api/src/migration/__tests__/migration-validation.service.spec.ts new file mode 100644 index 00000000..09af6881 --- /dev/null +++ b/apps/api/src/migration/__tests__/migration-validation.service.spec.ts @@ -0,0 +1,245 @@ +import { MigrationValidationService } from '../migration-validation.service'; +import { BadRequestException, NotFoundException } from '@nestjs/common'; + +jest.mock('iovalkey', () => { + const mockClient = () => ({ + connect: jest.fn().mockResolvedValue(undefined), + ping: jest.fn().mockResolvedValue('PONG'), + quit: jest.fn().mockResolvedValue(undefined), + dbsize: jest.fn().mockResolvedValue(100), + scan: jest.fn().mockResolvedValue(['0', []]), + pipeline: jest.fn().mockReturnValue({ + type: jest.fn().mockReturnThis(), + exec: jest.fn().mockResolvedValue([]), + }), + }); + const Valkey = jest.fn().mockImplementation(mockClient); + (Valkey as any).Cluster = jest.fn().mockImplementation(mockClient); + return Valkey; +}); + +jest.mock('../validation/key-count-comparator', () => ({ + compareKeyCounts: jest.fn().mockResolvedValue({ + sourceKeys: 100, + targetKeys: 100, + discrepancy: 0, + discrepancyPercent: 0, + }), +})); + +jest.mock('../validation/sample-validator', () => ({ + validateSample: jest.fn().mockResolvedValue({ + sampledKeys: 100, + matched: 100, + missing: 0, + typeMismatches: 0, + valueMismatches: 0, + issues: [], + }), +})); + +jest.mock('../validation/baseline-comparator', () => ({ + compareBaseline: jest.fn().mockResolvedValue({ + available: true, + snapshotCount: 10, + baselineWindowMs: 3600000, + metrics: [], + }), +})); + +function createMockRegistry(overrides?: { targetClusterEnabled?: boolean }) { + const targetCluster = overrides?.targetClusterEnabled ?? false; + const mockAdapter = { + getCapabilities: jest.fn().mockReturnValue({ dbType: 'valkey', version: '8.1.0' }), + getInfo: jest.fn().mockResolvedValue({ cluster: { cluster_enabled: targetCluster ? '1' : '0' } }), + getClient: jest.fn().mockReturnValue({ quit: jest.fn() }), + }; + return { + get: jest.fn().mockReturnValue(mockAdapter), + getConfig: jest.fn().mockReturnValue({ + id: 'conn-1', + name: 'Test', + host: '127.0.0.1', + port: 6379, + createdAt: Date.now(), + }), + mockAdapter, + }; +} + +function createMockStorage() { + return { + getSnapshots: jest.fn().mockResolvedValue([]), + getLatestSnapshot: jest.fn().mockResolvedValue(null), + } as any; +} + +function createMockMigrationService() { + return { + getJob: jest.fn().mockReturnValue(undefined), + } as any; +} + +describe('MigrationValidationService', () => { + let service: MigrationValidationService; + let registry: ReturnType; + let storage: ReturnType; + let migrationService: ReturnType; + + beforeEach(() => { + registry = createMockRegistry(); + storage = createMockStorage(); + migrationService = createMockMigrationService(); + service = new MigrationValidationService(registry as any, storage, migrationService); + }); + + describe('startValidation', () => { + it('should return a job ID with pending status', async () => { + const result = await service.startValidation({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + + expect(result.id).toBeDefined(); + expect(result.status).toBe('pending'); + }); + + it('should make the job retrievable via getValidation', async () => { + const { id } = await service.startValidation({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + + const validation = service.getValidation(id); + expect(validation).toBeDefined(); + expect(validation!.id).toBe(id); + }); + + it('should reject same source and target', async () => { + await expect( + service.startValidation({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-1', + }), + ).rejects.toThrow(BadRequestException); + }); + + it('should throw NotFoundException when connection does not exist', async () => { + registry.get.mockImplementation((id: string) => { + if (id === 'missing') throw new NotFoundException(); + return registry.mockAdapter; + }); + + await expect( + service.startValidation({ + sourceConnectionId: 'missing', + targetConnectionId: 'conn-2', + }), + ).rejects.toThrow(NotFoundException); + }); + + it('should detect cluster target and complete validation', async () => { + const clusterRegistry = createMockRegistry({ targetClusterEnabled: true }); + const clusterStorage = createMockStorage(); + const clusterMigrationService = createMockMigrationService(); + const clusterService = new MigrationValidationService( + clusterRegistry as any, + clusterStorage, + clusterMigrationService, + ); + + const { id } = await clusterService.startValidation({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + + // Wait for async validation to complete + await new Promise(r => setTimeout(r, 100)); + + const validation = clusterService.getValidation(id); + expect(validation).toBeDefined(); + // Should query target adapter for cluster info + expect(clusterRegistry.mockAdapter.getInfo).toHaveBeenCalledWith(['cluster']); + }); + + it('should use Phase 1 analysis result when analysisId provided', async () => { + migrationService.getJob.mockReturnValue({ + status: 'completed', + dataTypeBreakdown: { string: { count: 50 } }, + }); + + const { id } = await service.startValidation({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + analysisId: 'analysis-1', + }); + + expect(migrationService.getJob).toHaveBeenCalledWith('analysis-1'); + expect(service.getValidation(id)).toBeDefined(); + }); + }); + + describe('cancelValidation', () => { + it('should cancel a running validation', async () => { + const { id } = await service.startValidation({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + + const result = service.cancelValidation(id); + expect(result).toBe(true); + + const validation = service.getValidation(id); + expect(validation!.status).toBe('cancelled'); + expect(validation!.error).toBe('Cancelled by user'); + }); + + it('should return false for unknown job ID', () => { + expect(service.cancelValidation('nonexistent')).toBe(false); + }); + + it('should be idempotent for terminal states', async () => { + const { id } = await service.startValidation({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + + // Wait a tick for the job to potentially complete + await new Promise(r => setTimeout(r, 50)); + + const validation = service.getValidation(id); + if (validation!.status === 'completed' || validation!.status === 'failed') { + expect(service.cancelValidation(id)).toBe(true); + } + }); + }); + + describe('getValidation', () => { + it('should return undefined for unknown job ID', () => { + expect(service.getValidation('nonexistent')).toBeUndefined(); + }); + }); + + describe('job eviction', () => { + it('should evict oldest jobs when MAX_JOBS (10) reached', async () => { + const ids: string[] = []; + for (let i = 0; i < 10; i++) { + const { id } = await service.startValidation({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + ids.push(id); + // Cancel to put in terminal state + service.cancelValidation(id); + } + + const { id: newId } = await service.startValidation({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + + expect(service.getValidation(newId)).toBeDefined(); + expect(service.getValidation(ids[0])).toBeUndefined(); + }); + }); +}); diff --git a/apps/api/src/migration/__tests__/migration.controller.spec.ts b/apps/api/src/migration/__tests__/migration.controller.spec.ts new file mode 100644 index 00000000..1e97552d --- /dev/null +++ b/apps/api/src/migration/__tests__/migration.controller.spec.ts @@ -0,0 +1,216 @@ +import { BadRequestException, NotFoundException } from '@nestjs/common'; +import { MigrationController } from '../migration.controller'; + +describe('MigrationController', () => { + let controller: MigrationController; + let migrationService: Record; + let executionService: Record; + let validationService: Record; + + beforeEach(() => { + migrationService = { + startAnalysis: jest.fn().mockResolvedValue({ id: 'job-1', status: 'pending' }), + getJob: jest.fn().mockReturnValue({ id: 'job-1', status: 'running', progress: 50 }), + cancelJob: jest.fn().mockReturnValue(true), + }; + + executionService = { + startExecution: jest.fn().mockResolvedValue({ id: 'exec-1', status: 'pending' }), + getExecution: jest.fn().mockReturnValue({ id: 'exec-1', status: 'running' }), + stopExecution: jest.fn().mockReturnValue(true), + }; + + validationService = { + startValidation: jest.fn().mockResolvedValue({ id: 'val-1', status: 'pending' }), + getValidation: jest.fn().mockReturnValue({ id: 'val-1', status: 'running' }), + cancelValidation: jest.fn().mockReturnValue(true), + }; + + controller = new MigrationController( + migrationService as any, + executionService as any, + validationService as any, + ); + }); + + describe('POST /analysis', () => { + it('should reject same source and target connection', async () => { + await expect( + controller.startAnalysis({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-1', + }), + ).rejects.toThrow(BadRequestException); + }); + + it('should reject scanSampleSize below 1000', async () => { + await expect( + controller.startAnalysis({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + scanSampleSize: 500, + }), + ).rejects.toThrow(BadRequestException); + }); + + it('should reject scanSampleSize above 50000', async () => { + await expect( + controller.startAnalysis({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + scanSampleSize: 100000, + }), + ).rejects.toThrow(BadRequestException); + }); + + it('should accept valid request and return job ID', async () => { + const result = await controller.startAnalysis({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + + expect(result).toEqual({ id: 'job-1', status: 'pending' }); + expect(migrationService.startAnalysis).toHaveBeenCalled(); + }); + + it('should accept valid scanSampleSize', async () => { + const result = await controller.startAnalysis({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + scanSampleSize: 5000, + }); + + expect(result).toEqual({ id: 'job-1', status: 'pending' }); + }); + + it('should reject missing sourceConnectionId', async () => { + await expect( + controller.startAnalysis({ + sourceConnectionId: '', + targetConnectionId: 'conn-2', + }), + ).rejects.toThrow(BadRequestException); + }); + + it('should reject missing targetConnectionId', async () => { + await expect( + controller.startAnalysis({ + sourceConnectionId: 'conn-1', + targetConnectionId: '', + }), + ).rejects.toThrow(BadRequestException); + }); + }); + + describe('GET /analysis/:id', () => { + it('should return 404 for unknown ID', () => { + migrationService.getJob.mockReturnValue(undefined); + + expect(() => controller.getJob('nonexistent')).toThrow(NotFoundException); + }); + + it('should return the job result for a valid ID', () => { + const result = controller.getJob('job-1'); + + expect(result).toEqual({ id: 'job-1', status: 'running', progress: 50 }); + }); + }); + + describe('DELETE /analysis/:id', () => { + it('should return 404 when job not found', () => { + migrationService.cancelJob.mockReturnValue(false); + + expect(() => controller.cancelJob('nonexistent')).toThrow(NotFoundException); + }); + + it('should return cancelled: true on success', () => { + const result = controller.cancelJob('job-1'); + expect(result).toEqual({ cancelled: true }); + }); + }); + + describe('POST /execution', () => { + it('should reject same source and target', async () => { + await expect( + controller.startExecution({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-1', + }), + ).rejects.toThrow(BadRequestException); + }); + + it('should reject invalid mode', async () => { + await expect( + controller.startExecution({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + mode: 'invalid' as any, + }), + ).rejects.toThrow(BadRequestException); + }); + + it('should accept valid request', async () => { + const result = await controller.startExecution({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + mode: 'command', + }); + + expect(result).toEqual({ id: 'exec-1', status: 'pending' }); + }); + + it('should reject missing sourceConnectionId', async () => { + await expect( + controller.startExecution({ + sourceConnectionId: '', + targetConnectionId: 'conn-2', + }), + ).rejects.toThrow(BadRequestException); + }); + }); + + describe('POST /validation', () => { + it('should accept valid body and return job ID', async () => { + const result = await controller.startValidation({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + + expect(result).toEqual({ id: 'val-1', status: 'pending' }); + }); + + it('should reject same source and target', async () => { + await expect( + controller.startValidation({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-1', + }), + ).rejects.toThrow(BadRequestException); + }); + + it('should reject missing sourceConnectionId', async () => { + await expect( + controller.startValidation({ + sourceConnectionId: '', + targetConnectionId: 'conn-2', + }), + ).rejects.toThrow(BadRequestException); + }); + }); + + describe('GET /validation/:id', () => { + it('should return 404 for unknown ID', () => { + validationService.getValidation.mockReturnValue(undefined); + + expect(() => controller.getValidation('nonexistent')).toThrow(NotFoundException); + }); + }); + + describe('DELETE /validation/:id', () => { + it('should return 404 when job not found', () => { + validationService.cancelValidation.mockReturnValue(false); + + expect(() => controller.cancelValidation('nonexistent')).toThrow(NotFoundException); + }); + }); +}); diff --git a/apps/api/src/migration/__tests__/migration.service.spec.ts b/apps/api/src/migration/__tests__/migration.service.spec.ts new file mode 100644 index 00000000..e1677223 --- /dev/null +++ b/apps/api/src/migration/__tests__/migration.service.spec.ts @@ -0,0 +1,160 @@ +import { MigrationService } from '../migration.service'; +import { NotFoundException } from '@nestjs/common'; + +// Mock dependencies to prevent actual analysis from running +jest.mock('../analysis/type-sampler', () => ({ + sampleKeyTypes: jest.fn().mockResolvedValue([]), +})); +jest.mock('../analysis/ttl-sampler', () => ({ + sampleTtls: jest.fn().mockResolvedValue({ + noExpiry: 0, expiresWithin1h: 0, expiresWithin24h: 0, + expiresWithin7d: 0, expiresAfter7d: 0, sampledKeyCount: 0, + }), +})); +jest.mock('../analysis/hfe-detector', () => ({ + detectHfe: jest.fn().mockResolvedValue({ + hfeDetected: false, hfeSupported: true, hfeKeyCount: 0, + hfeOversizedHashesSkipped: 0, sampledHashCount: 0, + }), +})); +jest.mock('../analysis/commandlog-analyzer', () => ({ + analyzeCommands: jest.fn().mockResolvedValue({ + sourceUsed: 'unavailable', topCommands: [], + }), +})); + +function createMockRegistry() { + const mockClient = { + call: jest.fn().mockResolvedValue(['default']), + quit: jest.fn().mockResolvedValue(undefined), + ping: jest.fn().mockResolvedValue('PONG'), + }; + const mockAdapter = { + getCapabilities: jest.fn().mockReturnValue({ + dbType: 'valkey', + version: '8.1.0', + hasCommandLog: false, + }), + getInfo: jest.fn().mockResolvedValue({ + keyspace: { db0: 'keys=100,expires=0,avg_ttl=0' }, + memory: { used_memory: '1000000' }, + cluster: { cluster_enabled: '0' }, + server: {}, + persistence: { rdb_last_save_time: '0', aof_enabled: '0' }, + }), + getClient: jest.fn().mockReturnValue(mockClient), + getClusterNodes: jest.fn().mockResolvedValue([]), + }; + return { + get: jest.fn().mockReturnValue(mockAdapter), + getConfig: jest.fn().mockReturnValue({ + id: 'conn-1', + name: 'Test', + host: '127.0.0.1', + port: 6379, + createdAt: Date.now(), + }), + mockAdapter, + }; +} + +describe('MigrationService', () => { + let service: MigrationService; + let registry: ReturnType; + + beforeEach(() => { + registry = createMockRegistry(); + service = new MigrationService(registry as any); + }); + + describe('startAnalysis', () => { + it('should return a job ID with pending status', async () => { + const result = await service.startAnalysis({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + + expect(result.id).toBeDefined(); + expect(result.status).toBe('pending'); + }); + + it('should make the job retrievable via getJob', async () => { + const { id } = await service.startAnalysis({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + + const job = service.getJob(id); + expect(job).toBeDefined(); + expect(job!.id).toBe(id); + }); + + it('should throw NotFoundException when source connection does not exist', async () => { + registry.get.mockImplementation((id: string) => { + if (id === 'bad') throw new NotFoundException(); + return registry.mockAdapter; + }); + + await expect( + service.startAnalysis({ + sourceConnectionId: 'bad', + targetConnectionId: 'conn-2', + }), + ).rejects.toThrow(NotFoundException); + }); + }); + + describe('cancelJob', () => { + it('should cancel a running job', async () => { + const { id } = await service.startAnalysis({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + + const success = service.cancelJob(id); + expect(success).toBe(true); + + const job = service.getJob(id); + expect(job!.status).toBe('cancelled'); + }); + + it('should return false for unknown job ID', () => { + const result = service.cancelJob('nonexistent'); + expect(result).toBe(false); + }); + }); + + describe('getJob', () => { + it('should return undefined for unknown job ID', () => { + expect(service.getJob('nonexistent')).toBeUndefined(); + }); + }); + + describe('job eviction', () => { + it('should evict oldest completed jobs when MAX_JOBS reached', async () => { + // Fill with 20 jobs (MAX_JOBS), then mark them all completed + const ids: string[] = []; + for (let i = 0; i < 20; i++) { + const { id } = await service.startAnalysis({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + ids.push(id); + // Cancel them so they are in terminal state + service.cancelJob(id); + } + + // Now start one more — it should trigger eviction + const { id: newId } = await service.startAnalysis({ + sourceConnectionId: 'conn-1', + targetConnectionId: 'conn-2', + }); + + // The new job should exist + expect(service.getJob(newId)).toBeDefined(); + + // At least the oldest should have been evicted + expect(service.getJob(ids[0])).toBeUndefined(); + }); + }); +}); diff --git a/apps/api/src/migration/__tests__/toml-builder.spec.ts b/apps/api/src/migration/__tests__/toml-builder.spec.ts new file mode 100644 index 00000000..09fab359 --- /dev/null +++ b/apps/api/src/migration/__tests__/toml-builder.spec.ts @@ -0,0 +1,159 @@ +import { buildScanReaderToml } from '../execution/toml-builder'; +import type { DatabaseConnectionConfig } from '@betterdb/shared'; + +function makeConfig(overrides: Partial = {}): DatabaseConnectionConfig { + return { + id: 'conn-1', + name: 'Test', + host: '127.0.0.1', + port: 6379, + createdAt: Date.now(), + ...overrides, + }; +} + +describe('buildScanReaderToml', () => { + it('should generate valid TOML for single-node source and target', () => { + const source = makeConfig({ host: '10.0.0.1', port: 6379, password: 'srcpass' }); + const target = makeConfig({ host: '10.0.0.2', port: 6380, password: 'tgtpass' }); + + const toml = buildScanReaderToml(source, target, false); + + expect(toml).toContain('[scan_reader]'); + expect(toml).toContain('address = "10.0.0.1:6379"'); + expect(toml).toContain('password = "srcpass"'); + expect(toml).toContain('[redis_writer]'); + expect(toml).toContain('address = "10.0.0.2:6380"'); + expect(toml).toContain('password = "tgtpass"'); + expect(toml).not.toContain('cluster = true'); + }); + + it('should include cluster = true for cluster source', () => { + const source = makeConfig(); + const target = makeConfig(); + + const toml = buildScanReaderToml(source, target, true); + + expect(toml).toContain('cluster = true'); + }); + + it('should escape special characters in passwords', () => { + const source = makeConfig({ password: 'pass"word\\with\nnewline' }); + const target = makeConfig({ password: 'simple' }); + + const toml = buildScanReaderToml(source, target, false); + + expect(toml).toContain('pass\\"word\\\\with\\nnewline'); + expect(toml).not.toContain('pass"word'); + }); + + it('should set tls = true when TLS enabled', () => { + const source = makeConfig({ tls: true }); + const target = makeConfig({ tls: true }); + + const toml = buildScanReaderToml(source, target, false); + + // Both sections should have tls = true + const scanSection = toml.split('[redis_writer]')[0]; + const writerSection = toml.split('[redis_writer]')[1]; + expect(scanSection).toContain('tls = true'); + expect(writerSection).toContain('tls = true'); + }); + + it('should set tls = false when TLS not enabled', () => { + const source = makeConfig({ tls: false }); + const target = makeConfig({ tls: false }); + + const toml = buildScanReaderToml(source, target, false); + + expect(toml).toContain('tls = false'); + }); + + it('should use empty string for "default" username', () => { + const source = makeConfig({ username: 'default' }); + const target = makeConfig({ username: 'default' }); + + const toml = buildScanReaderToml(source, target, false); + + expect(toml).toContain('username = ""'); + }); + + it('should include custom username when not "default"', () => { + const source = makeConfig({ username: 'admin' }); + const target = makeConfig({ username: 'reader' }); + + const toml = buildScanReaderToml(source, target, false); + + const scanSection = toml.split('[redis_writer]')[0]; + const writerSection = toml.split('[redis_writer]')[1]; + expect(scanSection).toContain('username = "admin"'); + expect(writerSection).toContain('username = "reader"'); + }); + + it('should include [advanced] section with log_level', () => { + const source = makeConfig(); + const target = makeConfig(); + + const toml = buildScanReaderToml(source, target, false); + + expect(toml).toContain('[advanced]'); + expect(toml).toContain('log_level = "info"'); + }); + + it('should reject control characters in password instead of silently stripping', () => { + const source = makeConfig({ password: 'pass\x00word' }); + const target = makeConfig(); + + expect(() => buildScanReaderToml(source, target, false)).toThrow('control characters'); + }); + + it('should reject invalid port', () => { + const source = makeConfig({ port: 99999 as any }); + const target = makeConfig(); + + expect(() => buildScanReaderToml(source, target, false)).toThrow('Invalid port'); + }); + + it('should reject host with whitespace', () => { + const source = makeConfig({ host: 'host name' }); + const target = makeConfig(); + + expect(() => buildScanReaderToml(source, target, false)).toThrow('Invalid host'); + }); + + it('should reject empty host', () => { + const source = makeConfig({ host: '' }); + const target = makeConfig(); + + expect(() => buildScanReaderToml(source, target, false)).toThrow('Invalid host'); + }); + + it('should wrap bare IPv6 addresses in brackets for Go net.Dial', () => { + const source = makeConfig({ host: '::1', port: 6379 }); + const target = makeConfig({ host: '2001:db8::1', port: 6380 }); + + const toml = buildScanReaderToml(source, target, false); + + expect(toml).toContain('address = "[::1]:6379"'); + expect(toml).toContain('address = "[2001:db8::1]:6380"'); + }); + + it('should not double-bracket already-bracketed IPv6 addresses', () => { + const source = makeConfig({ host: '[::1]', port: 6379 }); + const target = makeConfig(); + + const toml = buildScanReaderToml(source, target, false); + + expect(toml).toContain('address = "[::1]:6379"'); + expect(toml).not.toContain('[['); + }); + + it('should not bracket IPv4 addresses containing no colons', () => { + const source = makeConfig({ host: '127.0.0.1', port: 6379 }); + const target = makeConfig(); + + const toml = buildScanReaderToml(source, target, false); + + expect(toml).toContain('address = "127.0.0.1:6379"'); + }); +}); diff --git a/apps/api/src/migration/__tests__/ttl-sampler.spec.ts b/apps/api/src/migration/__tests__/ttl-sampler.spec.ts new file mode 100644 index 00000000..e9a89b22 --- /dev/null +++ b/apps/api/src/migration/__tests__/ttl-sampler.spec.ts @@ -0,0 +1,99 @@ +import { sampleTtls } from '../analysis/ttl-sampler'; + +function createMockClient(ttls: number[]) { + return { + pipeline: jest.fn().mockReturnValue({ + pttl: jest.fn().mockReturnThis(), + exec: jest.fn().mockResolvedValue( + ttls.map(t => [null, t]), + ), + }), + } as any; +} + +describe('sampleTtls', () => { + it('should categorize a mix of persistent and expiring keys', async () => { + const ttls = [ + -1, // no expiry + 1_800_000, // 30 min → expiresWithin1h + 7_200_000, // 2 hours → expiresWithin24h + 259_200_000, // 3 days → expiresWithin7d + 864_000_000, // 10 days → expiresAfter7d + ]; + const client = createMockClient(ttls); + const keys = ['k1', 'k2', 'k3', 'k4', 'k5']; + + const result = await sampleTtls(client, keys); + + expect(result.noExpiry).toBe(1); + expect(result.expiresWithin1h).toBe(1); + expect(result.expiresWithin24h).toBe(1); + expect(result.expiresWithin7d).toBe(1); + expect(result.expiresAfter7d).toBe(1); + expect(result.sampledKeyCount).toBe(5); + }); + + it('should count all persistent keys when TTL = -1', async () => { + const ttls = [-1, -1, -1]; + const client = createMockClient(ttls); + + const result = await sampleTtls(client, ['k1', 'k2', 'k3']); + + expect(result.noExpiry).toBe(3); + expect(result.expiresWithin1h).toBe(0); + expect(result.expiresWithin24h).toBe(0); + expect(result.expiresWithin7d).toBe(0); + expect(result.expiresAfter7d).toBe(0); + }); + + it('should count all keys expiring within 1h', async () => { + const ttls = [60_000, 120_000, 300_000]; // 1min, 2min, 5min + const client = createMockClient(ttls); + + const result = await sampleTtls(client, ['k1', 'k2', 'k3']); + + expect(result.expiresWithin1h).toBe(3); + expect(result.noExpiry).toBe(0); + }); + + it('should return all buckets zero for empty key list', async () => { + const client = createMockClient([]); + + const result = await sampleTtls(client, []); + + expect(result.noExpiry).toBe(0); + expect(result.expiresWithin1h).toBe(0); + expect(result.expiresWithin24h).toBe(0); + expect(result.expiresWithin7d).toBe(0); + expect(result.expiresAfter7d).toBe(0); + expect(result.sampledKeyCount).toBe(0); + }); + + it('should skip TTL = -2 (key gone) and exclude from sampledKeyCount', async () => { + const ttls = [-2, -1]; + const client = createMockClient(ttls); + + const result = await sampleTtls(client, ['gone', 'persistent']); + + expect(result.noExpiry).toBe(1); + expect(result.sampledKeyCount).toBe(1); + }); + + it('should skip pipeline errors and exclude from sampledKeyCount', async () => { + const client = { + pipeline: jest.fn().mockReturnValue({ + pttl: jest.fn().mockReturnThis(), + exec: jest.fn().mockResolvedValue([ + [new Error('ERR'), null], + [null, 60_000], + ]), + }), + } as any; + + const result = await sampleTtls(client, ['k1', 'k2']); + + expect(result.noExpiry).toBe(0); + expect(result.expiresWithin1h).toBe(1); + expect(result.sampledKeyCount).toBe(1); + }); +}); diff --git a/apps/api/src/migration/__tests__/type-handlers.spec.ts b/apps/api/src/migration/__tests__/type-handlers.spec.ts new file mode 100644 index 00000000..01090915 --- /dev/null +++ b/apps/api/src/migration/__tests__/type-handlers.spec.ts @@ -0,0 +1,224 @@ +import { migrateKey } from '../execution/type-handlers'; + +function createMockSource(overrides: Record = {}) { + return { + getBuffer: jest.fn().mockResolvedValue(Buffer.from('value')), + hlen: jest.fn().mockResolvedValue(3), + hgetallBuffer: jest.fn().mockResolvedValue({ f1: Buffer.from('v1'), f2: Buffer.from('v2') }), + hscanBuffer: jest.fn().mockResolvedValue(['0', [Buffer.from('f1'), Buffer.from('v1')]]), + llen: jest.fn().mockResolvedValue(2), + lrangeBuffer: jest.fn().mockResolvedValue([Buffer.from('a'), Buffer.from('b')]), + scard: jest.fn().mockResolvedValue(2), + smembersBuffer: jest.fn().mockResolvedValue([Buffer.from('m1'), Buffer.from('m2')]), + sscanBuffer: jest.fn().mockResolvedValue(['0', [Buffer.from('m1')]]), + zcard: jest.fn().mockResolvedValue(2), + pttl: jest.fn().mockResolvedValue(-1), + call: jest.fn().mockResolvedValue(['m1', '1', 'm2', '2']), + // callBuffer returns Buffers for binary-safe zset/stream migration + callBuffer: jest.fn().mockImplementation((cmd: string) => { + if (cmd === 'ZRANGE') { + return Promise.resolve([Buffer.from('m1'), Buffer.from('1'), Buffer.from('m2'), Buffer.from('2')]); + } + if (cmd === 'ZSCAN') { + return Promise.resolve([Buffer.from('0'), [Buffer.from('m1'), Buffer.from('1')]]); + } + if (cmd === 'XRANGE') { + return Promise.resolve([[Buffer.from('1-0'), [Buffer.from('field'), Buffer.from('value')]]]); + } + return Promise.resolve(null); + }), + pipeline: jest.fn().mockReturnValue({ + zadd: jest.fn().mockReturnThis(), + exec: jest.fn().mockResolvedValue([]), + }), + ...overrides, + } as any; +} + +function createMockTarget() { + return { + set: jest.fn().mockResolvedValue('OK'), + del: jest.fn().mockResolvedValue(1), + rename: jest.fn().mockResolvedValue('OK'), + llen: jest.fn().mockResolvedValue(2), + pexpire: jest.fn().mockResolvedValue(1), + call: jest.fn().mockResolvedValue('OK'), + callBuffer: jest.fn().mockResolvedValue(Buffer.from('OK')), + pipeline: jest.fn().mockReturnValue({ + zadd: jest.fn().mockReturnThis(), + exec: jest.fn().mockResolvedValue([]), + }), + } as any; +} + +describe('type-handlers / migrateKey', () => { + let source: ReturnType; + let target: ReturnType; + + beforeEach(() => { + source = createMockSource(); + target = createMockTarget(); + }); + + describe('string', () => { + it('should GET from source and SET on target', async () => { + const result = await migrateKey(source, target, 'str:1', 'string'); + + expect(result.ok).toBe(true); + expect(source.getBuffer).toHaveBeenCalledWith('str:1'); + expect(target.set).toHaveBeenCalledWith('str:1', expect.any(Buffer)); + }); + + it('should handle deleted key gracefully and skip write', async () => { + source.getBuffer.mockResolvedValue(null); + const result = await migrateKey(source, target, 'gone', 'string'); + + expect(result.ok).toBe(true); + expect(target.set).not.toHaveBeenCalled(); + }); + }); + + describe('hash', () => { + it('should use HSCAN, write to temp key, and RENAME', async () => { + source.hlen.mockResolvedValue(5); + + const result = await migrateKey(source, target, 'hash:1', 'hash'); + + expect(result.ok).toBe(true); + expect(source.hscanBuffer).toHaveBeenCalled(); + // Writes to temp key then renames atomically + expect(target.call).toHaveBeenCalledWith('HSET', expect.stringContaining('__betterdb_mig_'), expect.any(Buffer), expect.any(Buffer)); + expect(target.call).toHaveBeenCalledWith('EVAL', expect.any(String), '2', expect.stringContaining('__betterdb_mig_'), 'hash:1', '-1'); + }); + }); + + describe('list', () => { + it('should LRANGE, RPUSH to temp key, and RENAME', async () => { + const result = await migrateKey(source, target, 'list:1', 'list'); + + expect(result.ok).toBe(true); + expect(source.lrangeBuffer).toHaveBeenCalled(); + expect(target.call).toHaveBeenCalledWith('RPUSH', expect.stringContaining('__betterdb_mig_'), expect.any(Buffer), expect.any(Buffer)); + expect(target.call).toHaveBeenCalledWith('EVAL', expect.any(String), '2', expect.stringContaining('__betterdb_mig_'), 'list:1', '-1'); + }); + }); + + describe('set', () => { + it('should use SMEMBERS for small sets, write to temp key, and RENAME', async () => { + source.scard.mockResolvedValue(5); + + const result = await migrateKey(source, target, 'set:1', 'set'); + + expect(result.ok).toBe(true); + expect(source.smembersBuffer).toHaveBeenCalledWith('set:1'); + expect(target.call).toHaveBeenCalledWith('SADD', expect.stringContaining('__betterdb_mig_'), expect.any(Buffer), expect.any(Buffer)); + expect(target.call).toHaveBeenCalledWith('EVAL', expect.any(String), '2', expect.stringContaining('__betterdb_mig_'), 'set:1', '-1'); + }); + + it('should use SSCAN for large sets (>10K members)', async () => { + source.scard.mockResolvedValue(15_000); + + const result = await migrateKey(source, target, 'set:big', 'set'); + + expect(result.ok).toBe(true); + expect(source.sscanBuffer).toHaveBeenCalled(); + expect(target.call).toHaveBeenCalledWith('EVAL', expect.any(String), '2', expect.stringContaining('__betterdb_mig_'), 'set:big', '-1'); + }); + }); + + describe('zset', () => { + it('should use callBuffer ZRANGE WITHSCORES, write to temp key, and RENAME', async () => { + source.zcard.mockResolvedValue(5); + + const result = await migrateKey(source, target, 'zset:1', 'zset'); + + expect(result.ok).toBe(true); + expect(source.callBuffer).toHaveBeenCalledWith('ZRANGE', 'zset:1', '0', '-1', 'WITHSCORES'); + expect(target.call).toHaveBeenCalledWith('EVAL', expect.any(String), '2', expect.stringContaining('__betterdb_mig_'), 'zset:1', '-1'); + }); + + it('should use callBuffer ZSCAN for large sorted sets (>10K members)', async () => { + source.zcard.mockResolvedValue(15_000); + + const result = await migrateKey(source, target, 'zset:big', 'zset'); + + expect(result.ok).toBe(true); + expect(source.callBuffer).toHaveBeenCalledWith('ZSCAN', 'zset:big', '0', 'COUNT', '1000'); + expect(target.call).toHaveBeenCalledWith('EVAL', expect.any(String), '2', expect.stringContaining('__betterdb_mig_'), 'zset:big', '-1'); + }); + }); + + describe('stream', () => { + it('should use callBuffer XRANGE, XADD to temp key, and RENAME', async () => { + const result = await migrateKey(source, target, 'stream:1', 'stream'); + + expect(result.ok).toBe(true); + expect(source.callBuffer).toHaveBeenCalledWith('XRANGE', 'stream:1', '-', '+', 'COUNT', '1000'); + expect(target.callBuffer).toHaveBeenCalledWith( + 'XADD', expect.stringContaining('__betterdb_mig_'), '1-0', Buffer.from('field'), Buffer.from('value'), + ); + expect(target.call).toHaveBeenCalledWith('EVAL', expect.any(String), '2', expect.stringContaining('__betterdb_mig_'), 'stream:1', '-1'); + }); + }); + + describe('TTL preservation', () => { + it('should use atomic SET PX for strings when source TTL > 0', async () => { + source.pttl.mockResolvedValue(60000); + + const result = await migrateKey(source, target, 'str:ttl', 'string'); + + expect(result.ok).toBe(true); + expect(target.set).toHaveBeenCalledWith('str:ttl', expect.any(Buffer), 'PX', 60000); + expect(target.pexpire).not.toHaveBeenCalled(); + }); + + it('should apply TTL atomically via Lua EVAL for compound types when source TTL > 0', async () => { + source.pttl.mockResolvedValue(60000); + + const result = await migrateKey(source, target, 'hash:ttl', 'hash'); + + expect(result.ok).toBe(true); + // TTL is passed to Lua EVAL as the ARGV[1] parameter + expect(target.call).toHaveBeenCalledWith( + 'EVAL', expect.any(String), '2', + expect.stringContaining('__betterdb_mig_'), 'hash:ttl', '60000', + ); + }); + + it('should not call pexpire when source TTL is -1', async () => { + source.pttl.mockResolvedValue(-1); + + const result = await migrateKey(source, target, 'str:no-ttl', 'string'); + + expect(result.ok).toBe(true); + expect(target.pexpire).not.toHaveBeenCalled(); + }); + + it('should return ok: false for string when source TTL is -2 (expired)', async () => { + source.pttl.mockResolvedValue(-2); + + const result = await migrateKey(source, target, 'str:expired', 'string'); + + expect(result.ok).toBe(true); + expect(target.del).toHaveBeenCalledWith('str:expired'); + }); + }); + + describe('error handling', () => { + it('should return ok: false for unsupported type', async () => { + const result = await migrateKey(source, target, 'key', 'unknown_type'); + + expect(result.ok).toBe(false); + expect(result.error).toContain('Unsupported type'); + }); + + it('should capture errors and return ok: false', async () => { + source.getBuffer.mockRejectedValue(new Error('Connection lost')); + + const result = await migrateKey(source, target, 'key', 'string'); + + expect(result.ok).toBe(false); + expect(result.error).toBe('Connection lost'); + }); + }); +}); diff --git a/apps/api/src/migration/__tests__/type-sampler.spec.ts b/apps/api/src/migration/__tests__/type-sampler.spec.ts new file mode 100644 index 00000000..1e89cf8c --- /dev/null +++ b/apps/api/src/migration/__tests__/type-sampler.spec.ts @@ -0,0 +1,116 @@ +import { sampleKeyTypes } from '../analysis/type-sampler'; + +function createMockClient(keys: string[], types: Record = {}) { + let callCount = 0; + return { + scan: jest.fn().mockImplementation((_cursor: string, ..._args: unknown[]) => { + if (callCount === 0) { + callCount++; + return Promise.resolve(['0', keys]); + } + return Promise.resolve(['0', []]); + }), + pipeline: jest.fn().mockReturnValue({ + type: jest.fn().mockReturnThis(), + exec: jest.fn().mockResolvedValue( + keys.map(k => [null, types[k] ?? 'string']), + ), + }), + } as any; +} + +describe('sampleKeyTypes', () => { + it('should return sampled keys with types for a single client', async () => { + const client = createMockClient(['key:1', 'key:2', 'key:3'], { + 'key:1': 'string', + 'key:2': 'hash', + 'key:3': 'list', + }); + + const result = await sampleKeyTypes([client], 1000); + + expect(result).toHaveLength(3); + expect(result[0]).toEqual({ key: 'key:1', type: 'string', clientIndex: 0 }); + expect(result[1]).toEqual({ key: 'key:2', type: 'hash', clientIndex: 0 }); + expect(result[2]).toEqual({ key: 'key:3', type: 'list', clientIndex: 0 }); + }); + + it('should return empty array for empty database', async () => { + const client = createMockClient([]); + const result = await sampleKeyTypes([client], 1000); + expect(result).toEqual([]); + }); + + it('should sample from multiple clients and merge results', async () => { + const client1 = createMockClient(['a:1'], { 'a:1': 'string' }); + const client2 = createMockClient(['b:1'], { 'b:1': 'hash' }); + + const result = await sampleKeyTypes([client1, client2], 1000); + + expect(result).toHaveLength(2); + expect(result[0].clientIndex).toBe(0); + expect(result[1].clientIndex).toBe(1); + }); + + it('should respect maxKeysPerNode limit', async () => { + const keys = Array.from({ length: 100 }, (_, i) => `key:${i}`); + // Return keys in two batches via cursor + let callCount = 0; + const client = { + scan: jest.fn().mockImplementation(() => { + if (callCount === 0) { + callCount++; + return Promise.resolve(['1', keys.slice(0, 50)]); + } + callCount++; + return Promise.resolve(['0', keys.slice(50)]); + }), + pipeline: jest.fn().mockReturnValue({ + type: jest.fn().mockReturnThis(), + exec: jest.fn().mockImplementation(function (this: any) { + // Return types matching the number of .type() calls made + const typeCallCount = this.type.mock.calls.length; + return Promise.resolve( + Array.from({ length: typeCallCount }, () => [null, 'string']), + ); + }), + }), + } as any; + + const result = await sampleKeyTypes([client], 5); + + // Should stop at 5 keys (the maxKeysPerNode limit) + expect(result.length).toBeLessThanOrEqual(5); + }); + + it('should call onProgress callback with increasing values', async () => { + const client = createMockClient(['key:1', 'key:2']); + const onProgress = jest.fn(); + + await sampleKeyTypes([client], 1000, onProgress); + + expect(onProgress).toHaveBeenCalled(); + // onProgress is called with the count of keys scanned so far + const calls = onProgress.mock.calls.map((c: number[]) => c[0]); + for (let i = 1; i < calls.length; i++) { + expect(calls[i]).toBeGreaterThanOrEqual(calls[i - 1]); + } + }); + + it('should mark type as unknown on pipeline error', async () => { + const client = { + scan: jest.fn().mockResolvedValue(['0', ['key:1']]), + pipeline: jest.fn().mockReturnValue({ + type: jest.fn().mockReturnThis(), + exec: jest.fn().mockResolvedValue([ + [new Error('NOPERM'), null], + ]), + }), + } as any; + + const result = await sampleKeyTypes([client], 1000); + + expect(result).toHaveLength(1); + expect(result[0].type).toBe('unknown'); + }); +}); diff --git a/apps/api/src/migration/analysis/analysis-job.ts b/apps/api/src/migration/analysis/analysis-job.ts new file mode 100644 index 00000000..4c21a25c --- /dev/null +++ b/apps/api/src/migration/analysis/analysis-job.ts @@ -0,0 +1,14 @@ +import type { MigrationJobStatus, MigrationAnalysisResult } from '@betterdb/shared'; +import type Valkey from 'iovalkey'; + +export interface AnalysisJob { + id: string; + status: MigrationJobStatus; + progress: number; + createdAt: number; + completedAt?: number; + error?: string; + result: Partial; + cancelled: boolean; + nodeClients: Valkey[]; +} diff --git a/apps/api/src/migration/analysis/commandlog-analyzer.ts b/apps/api/src/migration/analysis/commandlog-analyzer.ts new file mode 100644 index 00000000..ffdfc6f8 --- /dev/null +++ b/apps/api/src/migration/analysis/commandlog-analyzer.ts @@ -0,0 +1,55 @@ +import type { DatabasePort } from '../../common/interfaces/database-port.interface'; +import type { CommandAnalysis } from '@betterdb/shared'; + +export async function analyzeCommands( + adapter: DatabasePort, +): Promise { + const result: CommandAnalysis = { + sourceUsed: 'unavailable', + topCommands: [], + }; + + const capabilities = adapter.getCapabilities(); + let commandNames: string[] = []; + + // Try COMMANDLOG first + if (capabilities.hasCommandLog) { + try { + const entries = await adapter.getCommandLog(200); + commandNames = entries.map(e => { + const args = e.command ?? []; + return args.length > 0 ? String(args[0]).toUpperCase() : ''; + }).filter(Boolean); + result.sourceUsed = 'commandlog'; + } catch { + // Fall through to slowlog + } + } + + // Fallback to SLOWLOG + if (result.sourceUsed === 'unavailable') { + try { + const entries = await adapter.getSlowLog(128); + commandNames = entries.map(e => { + const args = e.command ?? []; + return args.length > 0 ? String(args[0]).toUpperCase() : ''; + }).filter(Boolean); + result.sourceUsed = 'slowlog'; + } catch { + // Both unavailable + return result; + } + } + + // Top commands + const counts = new Map(); + for (const cmd of commandNames) { + counts.set(cmd, (counts.get(cmd) ?? 0) + 1); + } + result.topCommands = Array.from(counts.entries()) + .map(([command, count]) => ({ command, count })) + .sort((a, b) => b.count - a.count) + .slice(0, 50); + + return result; +} diff --git a/apps/api/src/migration/analysis/compatibility-checker.ts b/apps/api/src/migration/analysis/compatibility-checker.ts new file mode 100644 index 00000000..322cae85 --- /dev/null +++ b/apps/api/src/migration/analysis/compatibility-checker.ts @@ -0,0 +1,235 @@ +import type { Incompatibility } from '@betterdb/shared'; +import type { DatabaseCapabilities } from '../../common/interfaces/database-port.interface'; + +export interface InstanceMeta { + dbType: 'valkey' | 'redis'; + version: string; + capabilities: DatabaseCapabilities; + clusterEnabled: boolean; + databases: number[]; + modules: string[]; + maxmemoryPolicy: string; + hasAclUsers: boolean; + persistenceMode: string; +} + +/** + * Compare two semver strings: returns true if a >= b. + * Handles versions like "8.1.0", "7.2.4", etc. + */ +function semverGte(a: string, b: string): boolean { + const partsA = a.split('.').map(s => parseInt(s, 10) || 0); + const partsB = b.split('.').map(s => parseInt(s, 10) || 0); + const len = Math.max(partsA.length, partsB.length); + for (let i = 0; i < len; i++) { + const va = partsA[i] ?? 0; + const vb = partsB[i] ?? 0; + if (va > vb) return true; + if (va < vb) return false; + } + return true; // equal +} + +export function buildInstanceMeta( + info: Record, + capabilities: DatabaseCapabilities, + aclUsers: string[], + rdbSaveConfig?: string, +): InstanceMeta { + // clusterEnabled + const clusterEnabled = String(info['cluster_enabled'] ?? '0') === '1'; + + // databases: parse keys like 'db0', 'db1', etc. + const databases: number[] = []; + for (const key of Object.keys(info)) { + const match = key.match(/^db(\d+)$/); + if (match && typeof info[key] === 'string') { + databases.push(parseInt(match[1], 10)); + } + } + if (databases.length === 0) { + databases.push(0); + } + + // modules: will be populated by caller via client.call('MODULE', 'LIST') + // For now, default to empty — the caller sets this after construction if needed + const modules: string[] = []; + + // maxmemoryPolicy + const maxmemoryPolicy = (info['maxmemory_policy'] as string) ?? 'noeviction'; + + // hasAclUsers: more than just the 'default' user + const hasAclUsers = aclUsers.length > 1; + + // persistenceMode + let hasRdb = false; + let hasAof = false; + + // rdb_last_save_time > 0 is unreliable — it's set to server start time even when RDB is disabled. + // Use the CONFIG GET save schedule when available; fall back to rdb_bgsave_in_progress as a weak signal. + if (rdbSaveConfig !== undefined) { + // CONFIG GET save returns "" when RDB is disabled, non-empty when a schedule is set + hasRdb = rdbSaveConfig.length > 0; + } else { + // Fallback: if a BGSAVE is actively running, RDB is clearly configured + const bgsaveInProgress = String(info['rdb_bgsave_in_progress'] ?? '0'); + if (bgsaveInProgress === '1') { + hasRdb = true; + } + } + + const aofEnabled = String(info['aof_enabled'] ?? '0'); + if (aofEnabled === '1') { + hasAof = true; + } + + let persistenceMode: string; + if (hasRdb && hasAof) { + persistenceMode = 'rdb+aof'; + } else if (hasRdb) { + persistenceMode = 'rdb'; + } else if (hasAof) { + persistenceMode = 'aof'; + } else { + persistenceMode = 'none'; + } + + return { + dbType: capabilities.dbType, + version: capabilities.version, + capabilities, + clusterEnabled, + databases, + modules, + maxmemoryPolicy, + hasAclUsers, + persistenceMode, + }; +} + +export function checkCompatibility( + source: InstanceMeta, + target: InstanceMeta, + hfeDetected: boolean, +): Incompatibility[] { + const issues: Incompatibility[] = []; + + // 1. Valkey -> Redis direction + if (source.dbType === 'valkey' && target.dbType === 'redis') { + issues.push({ + severity: 'blocking', + category: 'type_direction', + title: 'Valkey \u2192 Redis migration', + detail: + 'Migrating from Valkey to Redis may lose Valkey-specific features and data structures. This direction is not recommended.', + }); + } + + // 2. HFE unsupported on target + if (hfeDetected) { + const targetSupportsHfe = + target.dbType === 'valkey' && semverGte(target.version, '8.1.0'); + if (!targetSupportsHfe) { + issues.push({ + severity: 'blocking', + category: 'hfe', + title: 'Hash Field Expiry unsupported', + detail: + 'Source uses Hash Field Expiry (HFE). Target does not support HFE \u2014 per-field TTLs will be lost during migration. Requires Valkey 8.1+.', + }); + } + } + + // 3. Missing modules + for (const mod of source.modules) { + if (!target.modules.includes(mod)) { + issues.push({ + severity: 'blocking', + category: 'modules', + title: `Missing module: ${mod}`, + detail: `Source uses the '${mod}' module which is not loaded on the target instance.`, + }); + } + } + + // 4. Cluster -> standalone mismatch + if (source.clusterEnabled && !target.clusterEnabled) { + issues.push({ + severity: 'blocking', + category: 'cluster_topology', + title: 'Cluster \u2192 standalone mismatch', + detail: + 'Source runs in cluster mode but target is standalone. Data spread across multiple slots cannot be directly migrated to a single-node instance.', + }); + } + + // 5. Standalone -> cluster migration + if (!source.clusterEnabled && target.clusterEnabled) { + issues.push({ + severity: 'warning', + category: 'cluster_topology', + title: 'Standalone \u2192 cluster migration', + detail: + 'Source is standalone, target is clustered. Migration is possible but keys will be resharded across target slots.', + }); + } + + // 6. Multi-DB to cluster unsupported + if (source.databases.some(db => db !== 0) && target.clusterEnabled) { + issues.push({ + severity: 'blocking', + category: 'multi_db', + title: 'Multi-DB to cluster unsupported', + detail: + 'Source uses multiple databases (db indices beyond 0). Cluster mode only supports db0.', + }); + } + + // 7. Multi-DB data may be lost (standalone target without matching DBs) + if ( + source.databases.some(db => db !== 0) && + !target.clusterEnabled && + !target.databases.some(db => db !== 0) + ) { + issues.push({ + severity: 'warning', + category: 'multi_db', + title: 'Multi-DB data may be lost', + detail: + 'Source uses databases beyond db0. Verify the target is configured to accept multiple databases.', + }); + } + + // 8. Eviction policy mismatch + if (source.maxmemoryPolicy !== target.maxmemoryPolicy) { + issues.push({ + severity: 'warning', + category: 'maxmemory_policy', + title: 'Eviction policy mismatch', + detail: `Source uses '${source.maxmemoryPolicy}', target uses '${target.maxmemoryPolicy}'. Mismatched eviction policies may cause unexpected key eviction after migration.`, + }); + } + + // 9. ACL users not configured + if (source.hasAclUsers && !target.hasAclUsers) { + issues.push({ + severity: 'warning', + category: 'acl', + title: 'ACL users not configured', + detail: + 'Source has custom ACL users configured. Target only has the default user. Recreate ACL rules on the target before migrating.', + }); + } + + // 10. Persistence mode differs + if (source.persistenceMode !== target.persistenceMode) { + issues.push({ + severity: 'info', + category: 'persistence', + title: 'Persistence mode differs', + detail: `Source uses '${source.persistenceMode}' persistence, target uses '${target.persistenceMode}'. Review target persistence settings to ensure durability requirements are met.`, + }); + } + + return issues; +} diff --git a/apps/api/src/migration/analysis/hfe-detector.ts b/apps/api/src/migration/analysis/hfe-detector.ts new file mode 100644 index 00000000..5864f643 --- /dev/null +++ b/apps/api/src/migration/analysis/hfe-detector.ts @@ -0,0 +1,133 @@ +import type Valkey from 'iovalkey'; + +export interface HfeResult { + hfeDetected: boolean; + hfeSupported: boolean; + hfeKeyCount: number; + hfeOversizedHashesSkipped: number; + sampledHashCount: number; +} + +const MAX_HASH_SAMPLE = 300; +const MAX_HASH_FIELDS = 10_000; + +export async function detectHfe( + client: Valkey, + hashKeys: string[], + totalEstimatedHashKeys: number, +): Promise { + const result: HfeResult = { + hfeDetected: false, + hfeSupported: true, + hfeKeyCount: 0, + hfeOversizedHashesSkipped: 0, + sampledHashCount: 0, + }; + + const candidates = hashKeys.slice(0, MAX_HASH_SAMPLE); + if (candidates.length === 0) { + return result; + } + + // Check HLEN for each candidate, skip oversized ones + const validKeys: string[] = []; + for (let i = 0; i < candidates.length; i += 1000) { + const batch = candidates.slice(i, i + 1000); + const pipeline = client.pipeline(); + for (const key of batch) { + pipeline.hlen(key); + } + const results = await pipeline.exec(); + if (!results) continue; + for (let j = 0; j < batch.length; j++) { + const [err, len] = results[j] ?? []; + if (err) { + // Pipeline error (key expired, permission denied, etc.) — skip without counting as oversized + continue; + } + if (Number(len) > MAX_HASH_FIELDS) { + result.hfeOversizedHashesSkipped++; + } else { + validKeys.push(batch[j]); + } + } + } + + if (validKeys.length === 0) { + result.sampledHashCount = 0; + return result; + } + + // HRANDFIELD to get up to 3 random fields per key + const keyFieldPairs: Array<{ key: string; field: string }> = []; + for (let i = 0; i < validKeys.length; i += 1000) { + const batch = validKeys.slice(i, i + 1000); + const pipeline = client.pipeline(); + for (const key of batch) { + pipeline.call('HRANDFIELD', key, '-3'); + } + const results = await pipeline.exec(); + if (!results) continue; + for (let j = 0; j < batch.length; j++) { + const [err, fields] = results[j] ?? []; + if (err || !fields) continue; + const fieldList = Array.isArray(fields) ? fields : [fields]; + for (const f of fieldList) { + keyFieldPairs.push({ key: batch[j], field: String(f) }); + } + } + } + + result.sampledHashCount = validKeys.length; + + if (keyFieldPairs.length === 0) { + return result; + } + + // Pipeline HEXPIRETIME — wrap in try/catch for Redis (unknown command) + try { + let hfePositiveKeys = 0; + const checkedKeys = new Set(); + + const pipeline = client.pipeline(); + for (const { key, field } of keyFieldPairs) { + pipeline.call('HEXPIRETIME', key, 'FIELDS', '1', field); + } + const results = await pipeline.exec(); + if (results) { + for (let i = 0; i < keyFieldPairs.length; i++) { + const [err, val] = results[i] ?? []; + if (err) { + const errMsg = String(err); + // Only mark unsupported for genuinely unknown command errors + if (errMsg.includes('unknown command') || errMsg.includes('unknown subcommand')) { + result.hfeSupported = false; + result.hfeDetected = false; + return result; + } + // Transient errors (overload, permission, etc.) — skip this field + continue; + } + // HEXPIRETIME returns an array with the expiry time, >0 means HFE in use + const expiry = Array.isArray(val) ? Number(val[0]) : Number(val); + if (expiry > 0 && !checkedKeys.has(keyFieldPairs[i].key)) { + checkedKeys.add(keyFieldPairs[i].key); + hfePositiveKeys++; + } + } + } + + if (hfePositiveKeys > 0) { + result.hfeDetected = true; + result.hfeKeyCount = candidates.length > 0 + ? Math.round((hfePositiveKeys / candidates.length) * totalEstimatedHashKeys) + : 0; + } + } catch { + // HEXPIRETIME not supported (Redis) + result.hfeSupported = false; + result.hfeDetected = false; + } + + return result; +} diff --git a/apps/api/src/migration/analysis/ttl-sampler.ts b/apps/api/src/migration/analysis/ttl-sampler.ts new file mode 100644 index 00000000..eb183547 --- /dev/null +++ b/apps/api/src/migration/analysis/ttl-sampler.ts @@ -0,0 +1,47 @@ +import type Valkey from 'iovalkey'; +import type { TtlDistribution } from '@betterdb/shared'; + +export async function sampleTtls( + client: Valkey, + keys: string[], +): Promise { + const dist: TtlDistribution = { + noExpiry: 0, + expiresWithin1h: 0, + expiresWithin24h: 0, + expiresWithin7d: 0, + expiresAfter7d: 0, + sampledKeyCount: 0, + }; + + for (let i = 0; i < keys.length; i += 1000) { + const batch = keys.slice(i, i + 1000); + const pipeline = client.pipeline(); + for (const key of batch) { + pipeline.pttl(key); + } + const results = await pipeline.exec(); + if (!results) continue; + for (const [err, ttl] of results) { + const ms = err ? -2 : Number(ttl); + if (ms < 0 && ms !== -1) { + // ms === -2: key expired between SCAN and PTTL, or pipeline error — skip + continue; + } + dist.sampledKeyCount++; + if (ms === -1) { + dist.noExpiry++; + } else if (ms < 3_600_000) { + dist.expiresWithin1h++; + } else if (ms < 86_400_000) { + dist.expiresWithin24h++; + } else if (ms < 604_800_000) { + dist.expiresWithin7d++; + } else { + dist.expiresAfter7d++; + } + } + } + + return dist; +} diff --git a/apps/api/src/migration/analysis/type-sampler.ts b/apps/api/src/migration/analysis/type-sampler.ts new file mode 100644 index 00000000..b6b2602a --- /dev/null +++ b/apps/api/src/migration/analysis/type-sampler.ts @@ -0,0 +1,52 @@ +import type Valkey from 'iovalkey'; + +export interface SampledKey { + key: string; + type: string; + clientIndex: number; +} + +/** + * SCAN each client up to maxKeysPerNode, pipeline TYPE in batches of 1000. + * Returns combined list of sampled keys with types. + */ +export async function sampleKeyTypes( + clients: Valkey[], + maxKeysPerNode: number, + onProgress?: (scannedSoFar: number) => void, +): Promise { + const allKeys: SampledKey[] = []; + + for (let ci = 0; ci < clients.length; ci++) { + const client = clients[ci]; + const nodeKeys: string[] = []; + let cursor = '0'; + do { + const [nextCursor, keys] = await client.scan(cursor, 'COUNT', 1000); + cursor = nextCursor; + for (const k of keys) { + if (nodeKeys.length >= maxKeysPerNode) break; + nodeKeys.push(k); + } + onProgress?.(allKeys.length + nodeKeys.length); + } while (cursor !== '0' && nodeKeys.length < maxKeysPerNode); + + // Pipeline TYPE in batches of 1000 + for (let i = 0; i < nodeKeys.length; i += 1000) { + const batch = nodeKeys.slice(i, i + 1000); + const pipeline = client.pipeline(); + for (const key of batch) { + pipeline.type(key); + } + const results = await pipeline.exec(); + if (results) { + for (let j = 0; j < batch.length; j++) { + const [err, type] = results[j] ?? []; + allKeys.push({ key: batch[j], type: err ? 'unknown' : String(type), clientIndex: ci }); + } + } + } + } + + return allKeys; +} diff --git a/apps/api/src/migration/execution/client-factory.ts b/apps/api/src/migration/execution/client-factory.ts new file mode 100644 index 00000000..9ea4fe2b --- /dev/null +++ b/apps/api/src/migration/execution/client-factory.ts @@ -0,0 +1,53 @@ +import Valkey, { Cluster } from 'iovalkey'; +import type { DatabaseConnectionConfig } from '@betterdb/shared'; + +/** + * Create a standalone Valkey client from a connection config. + */ +export function createClient(config: DatabaseConnectionConfig, name: string): Valkey { + return new Valkey({ + host: config.host, + port: config.port, + username: config.username || undefined, + password: config.password || undefined, + tls: config.tls ? {} : undefined, + lazyConnect: true, + connectTimeout: 10_000, + commandTimeout: 15_000, + connectionName: name, + }); +} + +/** + * Create a target client — Cluster or standalone depending on the topology. + * The Cluster client is cast to Valkey so callers can use the same Commander + * interface without branching. + */ +export function createTargetClient( + config: DatabaseConnectionConfig, + name: string, + isCluster: boolean, +): Valkey { + if (!isCluster) { + return createClient(config, name); + } + + const cluster = new Cluster( + [{ host: config.host, port: config.port }], + { + redisOptions: { + username: config.username || undefined, + password: config.password || undefined, + tls: config.tls ? {} : undefined, + connectTimeout: 10_000, + commandTimeout: 15_000, + connectionName: name, + }, + lazyConnect: true, + enableReadyCheck: true, + ...(config.tls ? { dnsLookup: (address: string, callback: (err: NodeJS.ErrnoException | null, address: string, family: number) => void) => callback(null, address, 4) } : {}), + }, + ); + + return cluster as unknown as Valkey; +} diff --git a/apps/api/src/migration/execution/command-migration-worker.ts b/apps/api/src/migration/execution/command-migration-worker.ts new file mode 100644 index 00000000..0d6b75db --- /dev/null +++ b/apps/api/src/migration/execution/command-migration-worker.ts @@ -0,0 +1,206 @@ +import Valkey from 'iovalkey'; +import type { DatabaseConnectionConfig } from '@betterdb/shared'; +import type { ExecutionJob } from './execution-job'; +import { migrateKey } from './type-handlers'; +import { createClient, createTargetClient } from './client-factory'; + +const SCAN_COUNT = 500; +const TYPE_BATCH = 500; +const MIGRATE_BATCH = 50; + +export interface CommandMigrationOptions { + sourceConfig: DatabaseConnectionConfig; + targetConfig: DatabaseConnectionConfig; + sourceIsCluster: boolean; + targetIsCluster: boolean; + job: ExecutionJob; + maxLogLines: number; +} + +/** + * Run a command-based migration: SCAN source → TYPE → type-specific read/write → TTL. + * Operates entirely in-process using iovalkey. No external binary needed. + */ +export async function runCommandMigration(opts: CommandMigrationOptions): Promise { + const { sourceConfig, targetConfig, sourceIsCluster, targetIsCluster, job, maxLogLines } = opts; + const sourceClients: Valkey[] = []; + const targetClient = createTargetClient(targetConfig, 'BetterDB-Migration-Target', targetIsCluster); + + try { + await targetClient.connect(); + log(job, maxLogLines, `Connected to target${targetIsCluster ? ' (cluster mode)' : ''}`); + + // Build source clients (one per cluster master, or single standalone) + if (sourceIsCluster) { + const discoveryClient = createClient(sourceConfig, 'BetterDB-Migration-Discovery'); + await discoveryClient.connect(); + try { + const nodesRaw = await discoveryClient.call('CLUSTER', 'NODES') as string; + const masters = parseClusterMasters(nodesRaw); + log(job, maxLogLines, `Cluster mode: ${masters.length} master(s) detected`); + for (const { host, port } of masters) { + const client = new Valkey({ + host, + port, + username: sourceConfig.username || undefined, + password: sourceConfig.password || undefined, + tls: sourceConfig.tls ? {} : undefined, + lazyConnect: true, + connectionName: 'BetterDB-Migration-Source', + }); + await client.connect(); + sourceClients.push(client); + } + } finally { + await discoveryClient.quit(); + } + } else { + const client = createClient(sourceConfig, 'BetterDB-Migration-Source'); + await client.connect(); + sourceClients.push(client); + } + + log(job, maxLogLines, `Connected to source (${sourceClients.length} node(s))`); + + // Count total keys across all source nodes for progress tracking + let totalKeys = 0; + for (const client of sourceClients) { + const dbsize = await client.dbsize(); + totalKeys += dbsize; + } + job.totalKeys = totalKeys; + log(job, maxLogLines, `Total keys to migrate: ${totalKeys.toLocaleString()}`); + + if (totalKeys === 0) { + log(job, maxLogLines, 'No keys to migrate'); + job.progress = 100; + return; + } + + // Scan and migrate each source node + let keysProcessed = 0; + let keysSkipped = 0; + + for (let nodeIdx = 0; nodeIdx < sourceClients.length; nodeIdx++) { + const sourceClient = sourceClients[nodeIdx]; + if (isCancelled(job)) return; + + if (sourceClients.length > 1) { + log(job, maxLogLines, `Scanning node ${nodeIdx + 1}/${sourceClients.length}...`); + } + + let cursor = '0'; + do { + if (isCancelled(job)) return; + + const [nextCursor, keys] = await sourceClient.scan(cursor, 'COUNT', SCAN_COUNT); + cursor = nextCursor; + + if (keys.length === 0) continue; + + // Batch TYPE lookup + const types = await batchType(sourceClient, keys); + + // Migrate keys in parallel batches for throughput + for (let batchStart = 0; batchStart < keys.length; batchStart += MIGRATE_BATCH) { + if (isCancelled(job)) return; + + const batchEnd = Math.min(batchStart + MIGRATE_BATCH, keys.length); + const batchPromises: Promise[] = []; + + for (let i = batchStart; i < batchEnd; i++) { + const key = keys[i]; + const type = types[i]; + + if (type === 'none') { + // Key expired between SCAN and TYPE + keysProcessed++; + continue; + } + + batchPromises.push( + migrateKey(sourceClient, targetClient, key, type).then(result => { + if (result.ok) { + job.keysTransferred++; + } else { + keysSkipped++; + job.keysSkipped = keysSkipped; + log(job, maxLogLines, `SKIP ${key} (${type}): ${result.error}`); + } + keysProcessed++; + }), + ); + } + + await Promise.all(batchPromises); + job.progress = Math.min(99, Math.round((keysProcessed / totalKeys) * 100)); + } + + // Periodic progress log + if (keysProcessed % 5000 < keys.length) { + log(job, maxLogLines, + `Progress: ${keysProcessed.toLocaleString()}/${totalKeys.toLocaleString()} keys ` + + `(${job.keysTransferred.toLocaleString()} transferred, ${keysSkipped} skipped)`); + } + } while (cursor !== '0'); + } + + job.progress = 100; + log(job, maxLogLines, + `Migration complete: ${job.keysTransferred.toLocaleString()} transferred, ${keysSkipped} skipped out of ${totalKeys.toLocaleString()} total`); + + } finally { + await Promise.allSettled([...sourceClients, targetClient].map(c => c.quit())); + } +} + +// ── Helpers ── + +function parseClusterMasters(nodesRaw: string): Array<{ host: string; port: number }> { + const results: Array<{ host: string; port: number }> = []; + for (const line of nodesRaw.split('\n')) { + if (!line.trim()) continue; + const parts = line.split(' '); + const flags = parts[2] ?? ''; + if (!flags.includes('master')) continue; + if (flags.includes('fail') || flags.includes('noaddr')) continue; + // address format: host:port@clusterport (host may be IPv6, e.g. [::1]:6379@16379) + const addrPart = (parts[1] ?? '').split('@')[0]; + const lastColon = addrPart.lastIndexOf(':'); + let host = lastColon > 0 ? addrPart.substring(0, lastColon) : ''; + const port = lastColon > 0 ? parseInt(addrPart.substring(lastColon + 1), 10) : NaN; + // Strip IPv6 brackets — iovalkey expects bare addresses + if (host.startsWith('[') && host.endsWith(']')) { + host = host.slice(1, -1); + } + if (host && !isNaN(port)) { + results.push({ host, port }); + } + } + return results; +} + +async function batchType(client: Valkey, keys: string[]): Promise { + const pipeline = client.pipeline(); + for (const key of keys) { + pipeline.type(key); + } + const results = await pipeline.exec(); + return (results ?? []).map(([err, val]) => { + if (err) return 'none'; + return String(val); + }); +} + +function isCancelled(job: ExecutionJob): boolean { + return (job.status as string) === 'cancelled'; +} + +function log(job: ExecutionJob, maxLines: number, message: string): void { + const timestamp = new Date().toISOString().replace('T', ' ').replace('Z', ''); + const line = `[${timestamp}] ${message}`; + job.logs.push(line); + if (job.logs.length > maxLines) { + job.logs.shift(); + } +} diff --git a/apps/api/src/migration/execution/execution-job.ts b/apps/api/src/migration/execution/execution-job.ts new file mode 100644 index 00000000..20dc3d3f --- /dev/null +++ b/apps/api/src/migration/execution/execution-job.ts @@ -0,0 +1,20 @@ +import type { ChildProcess } from 'child_process'; +import type { ExecutionJobStatus, ExecutionMode } from '@betterdb/shared'; + +export interface ExecutionJob { + id: string; + mode: ExecutionMode; + status: ExecutionJobStatus; + startedAt: number; + completedAt?: number; + error?: string; + keysTransferred: number; + bytesTransferred: number; + keysSkipped: number; + totalKeys: number; + logs: string[]; // rolling, capped at MAX_LOG_LINES = 500 + progress: number | null; + process: ChildProcess | null; // redis_shake mode only + tomlPath: string | null; // redis_shake mode only + pidPath: string | null; // redis_shake mode only — for orphan detection +} diff --git a/apps/api/src/migration/execution/log-parser.ts b/apps/api/src/migration/execution/log-parser.ts new file mode 100644 index 00000000..6d97e2d5 --- /dev/null +++ b/apps/api/src/migration/execution/log-parser.ts @@ -0,0 +1,71 @@ +export interface ParsedLogLine { + keysTransferred: number | null; + bytesTransferred: number | null; + progress: number | null; // 0–100 +} + +const NULL_RESULT: ParsedLogLine = { keysTransferred: null, bytesTransferred: null, progress: null }; + +export function parseLogLine(line: string): ParsedLogLine { + // Strategy 1: Try JSON parse + try { + const obj = JSON.parse(line); + if (typeof obj === 'object' && obj !== null) { + const scanned = + obj?.counts?.scanned ?? + obj?.key_counts?.scanned ?? + obj?.scanned ?? + null; + const total = + obj?.counts?.total ?? + obj?.key_counts?.total ?? + obj?.total ?? + null; + const bytes = + obj?.bytes ?? + obj?.bytes_transferred ?? + null; + + const keysTransferred = typeof scanned === 'number' ? scanned : null; + const bytesTransferred = typeof bytes === 'number' ? bytes : null; + let progress: number | null = null; + + if (typeof scanned === 'number' && typeof total === 'number' && total > 0) { + progress = Math.min(100, Math.round((scanned / total) * 100)); + } + + if (keysTransferred !== null || bytesTransferred !== null || progress !== null) { + return { keysTransferred, bytesTransferred, progress }; + } + } + } catch { + // Not JSON — fall through to regex + } + + // Strategy 2: Regex patterns + const result: ParsedLogLine = { keysTransferred: null, bytesTransferred: null, progress: null }; + + const scannedMatch = line.match(/scanned[=: ]+(\d+)/i); + if (scannedMatch) { + result.keysTransferred = parseInt(scannedMatch[1], 10); + } + + const totalMatch = line.match(/total[=: ]+(\d+)/i); + if (totalMatch && result.keysTransferred !== null) { + const total = parseInt(totalMatch[1], 10); + if (total > 0) { + result.progress = Math.min(100, Math.round((result.keysTransferred / total) * 100)); + } + } + + const percentMatch = line.match(/(\d+(?:\.\d+)?)\s*%/); + if (percentMatch && result.progress === null) { + result.progress = Math.min(100, Math.round(parseFloat(percentMatch[1]))); + } + + if (result.keysTransferred !== null || result.bytesTransferred !== null || result.progress !== null) { + return result; + } + + return NULL_RESULT; +} diff --git a/apps/api/src/migration/execution/redisshake-runner.ts b/apps/api/src/migration/execution/redisshake-runner.ts new file mode 100644 index 00000000..68377d83 --- /dev/null +++ b/apps/api/src/migration/execution/redisshake-runner.ts @@ -0,0 +1,24 @@ +import { existsSync } from 'fs'; +import { join } from 'path'; +import * as os from 'os'; + +export function findRedisShakeBinary(): string { + // 1. Explicit env override + if (process.env.REDIS_SHAKE_PATH && existsSync(process.env.REDIS_SHAKE_PATH)) { + return process.env.REDIS_SHAKE_PATH; + } + // 2. Docker image location + if (existsSync('/usr/local/bin/redis-shake')) { + return '/usr/local/bin/redis-shake'; + } + // 3. npx install location + const npxPath = join(os.homedir(), '.betterdb', 'bin', 'redis-shake'); + if (npxPath && existsSync(npxPath)) { + return npxPath; + } + throw new Error( + 'RedisShake binary not found. ' + + 'Set REDIS_SHAKE_PATH env var, or install it to ~/.betterdb/bin/redis-shake. ' + + 'See https://docs.betterdb.com/migration for instructions.', + ); +} diff --git a/apps/api/src/migration/execution/toml-builder.ts b/apps/api/src/migration/execution/toml-builder.ts new file mode 100644 index 00000000..30aaffcc --- /dev/null +++ b/apps/api/src/migration/execution/toml-builder.ts @@ -0,0 +1,83 @@ +import type { DatabaseConnectionConfig } from '@betterdb/shared'; + +// eslint-disable-next-line no-control-regex +const CONTROL_CHAR_RE = /[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/; + +function escapeTomlString(value: string): string { + if (CONTROL_CHAR_RE.test(value)) { + throw new Error(`Value contains disallowed control characters`); + } + return value + .replace(/\\/g, '\\\\') + .replace(/"/g, '\\"') + .replace(/\n/g, '\\n') + .replace(/\r/g, '\\r') + .replace(/\t/g, '\\t'); +} + +function validatePort(port: unknown): number { + const n = Number(port); + if (!Number.isInteger(n) || n < 1 || n > 65535) { + throw new Error(`Invalid port: ${port}`); + } + return n; +} + +function validateHost(host: string): string { + if (!host || host.length > 253) { + throw new Error('Invalid host: empty or too long'); + } + if (/[\s"\\]/.test(host)) { + throw new Error('Invalid host: contains whitespace, quotes, or backslashes'); + } + return host; +} + +function formatAddress(host: string, port: number): string { + // Bare IPv6 addresses must be wrapped in brackets for Go's net.Dial + // e.g. "::1" → "[::1]:6379" + if (host.includes(':') && !host.startsWith('[')) { + return `[${host}]:${port}`; + } + return `${host}:${port}`; +} + +export function buildScanReaderToml( + source: DatabaseConnectionConfig, + target: DatabaseConnectionConfig, + sourceIsCluster: boolean, +): string { + const srcHost = validateHost(source.host); + const srcPort = validatePort(source.port); + const tgtHost = validateHost(target.host); + const tgtPort = validatePort(target.port); + + const srcUsername = (!source.username || source.username === 'default') ? '' : source.username; + const srcPassword = source.password ?? ''; + const tgtUsername = (!target.username || target.username === 'default') ? '' : target.username; + const tgtPassword = target.password ?? ''; + + let toml = `[scan_reader] +address = "${escapeTomlString(formatAddress(srcHost, srcPort))}" +username = "${escapeTomlString(srcUsername)}" +password = "${escapeTomlString(srcPassword)}" +tls = ${source.tls ? 'true' : 'false'} +`; + + if (sourceIsCluster) { + toml += `cluster = true\n`; + } + + toml += ` +[redis_writer] +address = "${escapeTomlString(formatAddress(tgtHost, tgtPort))}" +username = "${escapeTomlString(tgtUsername)}" +password = "${escapeTomlString(tgtPassword)}" +tls = ${target.tls ? 'true' : 'false'} + +[advanced] +log_level = "info" +`; + + return toml; +} diff --git a/apps/api/src/migration/execution/type-handlers.ts b/apps/api/src/migration/execution/type-handlers.ts new file mode 100644 index 00000000..1e059048 --- /dev/null +++ b/apps/api/src/migration/execution/type-handlers.ts @@ -0,0 +1,392 @@ +import type Valkey from 'iovalkey'; +import { randomBytes } from 'crypto'; + +// Threshold above which we use cursor-based reads (HSCAN/SSCAN/ZSCAN) instead of bulk reads +const LARGE_KEY_THRESHOLD = 10_000; +const SCAN_BATCH = 1000; +const LIST_CHUNK = 1000; +const STREAM_CHUNK = 1000; + +/** + * Generate a unique temporary key that hashes to the same slot as the original key. + * In cluster mode, RENAME requires both keys to be in the same slot. + * + * Returns null for keys that contain braces but have no valid hash tag (e.g. + * `user:{}:1`). Valkey hashes the full key name for these, and we can't + * construct a temp key in the same slot without embedding `}` which would + * create a different hash tag. Callers must write directly to the final key + * when null is returned. + */ +function tempKey(key: string): string | null { + const suffix = randomBytes(8).toString('hex'); + const openBrace = key.indexOf('{'); + if (openBrace !== -1) { + const closeBrace = key.indexOf('}', openBrace + 1); + if (closeBrace > openBrace + 1) { + // Key has a valid hash tag — reuse it so temp key lands in the same slot + const tag = key.substring(openBrace, closeBrace + 1); + return `__betterdb_mig_${suffix}:${tag}`; + } + // Braces present but no valid tag (empty `{}` or unclosed `{`). + // Cannot safely construct a same-slot temp key. + return null; + } + // No braces — wrap the whole key as the tag + return `__betterdb_mig_${suffix}:{${key}}`; +} + +export interface MigratedKey { + key: string; + type: string; + ok: boolean; + error?: string; + warning?: string; +} + +/** + * Migrate a single key from source to target using type-specific commands. + * Returns success/failure per key. Never throws — errors are captured in the result. + */ +export async function migrateKey( + source: Valkey, + target: Valkey, + key: string, + type: string, +): Promise { + try { + let wrote: boolean; + switch (type) { + case 'string': + // String handles TTL atomically via SET PX + wrote = await migrateString(source, target, key); + break; + case 'hash': + wrote = await migrateHash(source, target, key); + break; + case 'list': + wrote = await migrateList(source, target, key); + break; + case 'set': + wrote = await migrateSet(source, target, key); + break; + case 'zset': + wrote = await migrateZset(source, target, key); + break; + case 'stream': + wrote = await migrateStream(source, target, key); + break; + default: + return { key, type, ok: false, error: `Unsupported type: ${type}` }; + } + // TTL is handled atomically in each handler: + // - String: SET PX + // - Compound types: Lua RENAME+PEXPIRE via atomicRenameWithTtl + const result: MigratedKey = { key, type, ok: true }; + + // Post-migration list length check: source list may have changed during migration + if (wrote && type === 'list') { + try { + const targetLen = await target.llen(key); + const sourceLen = await source.llen(key); + if (targetLen !== sourceLen) { + result.warning = `list length changed during migration (migrated: ${targetLen}, current source: ${sourceLen})`; + } + } catch { /* non-fatal check */ } + } + + return result; + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + return { key, type, ok: false, error: message }; + } +} + +// ── String ── + +async function migrateString(source: Valkey, target: Valkey, key: string): Promise { + const [value, pttl] = await Promise.all([ + source.getBuffer(key), + source.pttl(key), + ]); + if (value === null) return false; // key expired/deleted between SCAN and GET + if (pttl > 0) { + // Atomic SET with PX — no window where key exists without TTL + await target.set(key, value, 'PX', pttl); + } else if (pttl === -2 || pttl === 0) { + // pttl -2: expired between GET and PTTL; pttl 0: sub-ms remaining — treat as expired + await target.del(key); + return false; + } else { + // pttl -1: no expiry (persistent key) + await target.set(key, value); + } + return true; +} + +// ── Hash ── + +async function migrateHash(source: Valkey, target: Valkey, key: string): Promise { + const len = await source.hlen(key); + if (len === 0) return false; + + const tmp = tempKey(key); + const writeKey = tmp ?? key; + + try { + // DEL the target when writing directly (no temp key) + if (!tmp) await target.del(key); + + // Use HSCAN for all sizes so binary field names are preserved as Buffers + let cursor = '0'; + do { + const [next, fields] = await source.hscanBuffer(key, cursor, 'COUNT', SCAN_BATCH); + cursor = String(next); + if (fields.length === 0) continue; + const args: (string | Buffer | number)[] = [writeKey]; + for (let i = 0; i < fields.length; i += 2) { + args.push(fields[i], fields[i + 1]); + } + await target.call('HSET', ...args); + } while (cursor !== '0'); + const pttl = await source.pttl(key); + if (tmp) { + await atomicRenameWithTtl(target, tmp, key, pttl); + } else { + await applyTtl(target, key, pttl); + } + } catch (err) { + if (tmp) { try { await target.del(tmp); } catch { /* best-effort cleanup */ } } + throw err; + } + return true; +} + +// ── List ── + +async function migrateList(source: Valkey, target: Valkey, key: string): Promise { + const len = await source.llen(key); + if (len === 0) return false; + + const tmp = tempKey(key); + const writeKey = tmp ?? key; + + try { + if (!tmp) await target.del(key); + + for (let start = 0; start < len; start += LIST_CHUNK) { + const end = Math.min(start + LIST_CHUNK - 1, len - 1); + const items = await source.lrangeBuffer(key, start, end); + if (items.length === 0) break; + await target.call('RPUSH', writeKey, ...items); + } + const pttl = await source.pttl(key); + if (tmp) { + await atomicRenameWithTtl(target, tmp, key, pttl); + } else { + await applyTtl(target, key, pttl); + } + } catch (err) { + if (tmp) { try { await target.del(tmp); } catch { /* best-effort cleanup */ } } + throw err; + } + return true; +} + +// ── Set ── + +async function migrateSet(source: Valkey, target: Valkey, key: string): Promise { + const card = await source.scard(key); + if (card === 0) return false; + + const tmp = tempKey(key); + const writeKey = tmp ?? key; + + try { + if (!tmp) await target.del(key); + + if (card <= LARGE_KEY_THRESHOLD) { + const members = await source.smembersBuffer(key); + if (members.length === 0) { + if (tmp) { try { await target.del(tmp); } catch { /* best-effort cleanup */ } } + return false; // key expired between SCARD and SMEMBERS + } + await target.call('SADD', writeKey, ...members); + } else { + let cursor = '0'; + do { + const [next, members] = await source.sscanBuffer(key, cursor, 'COUNT', SCAN_BATCH); + cursor = String(next); + if (members.length === 0) continue; + await target.call('SADD', writeKey, ...members); + } while (cursor !== '0'); + } + const pttl = await source.pttl(key); + if (tmp) { + await atomicRenameWithTtl(target, tmp, key, pttl); + } else { + await applyTtl(target, key, pttl); + } + } catch (err) { + if (tmp) { try { await target.del(tmp); } catch { /* best-effort cleanup */ } } + throw err; + } + return true; +} + +// ── Sorted Set ── + +async function migrateZset(source: Valkey, target: Valkey, key: string): Promise { + const card = await source.zcard(key); + if (card === 0) return false; + + const tmp = tempKey(key); + const writeKey = tmp ?? key; + + try { + if (!tmp) await target.del(key); + + if (card <= LARGE_KEY_THRESHOLD) { + // Use callBuffer to preserve binary member data (call() decodes as UTF-8) + const raw = await source.callBuffer('ZRANGE', key, '0', '-1', 'WITHSCORES') as Buffer[]; + if (!raw || raw.length === 0) { + if (tmp) { try { await target.del(tmp); } catch { /* best-effort cleanup */ } } + return false; // key expired between ZCARD and ZRANGE + } + // raw is [member, score, member, score, ...] as Buffers + const pipeline = target.pipeline(); + for (let i = 0; i < raw.length; i += 2) { + // Score is always ASCII-safe, member stays as Buffer + pipeline.zadd(writeKey, raw[i + 1].toString(), raw[i]); + } + await pipeline.exec(); + } else { + // zscanBuffer not available — use callBuffer for ZSCAN to preserve binary members + let cursor = '0'; + do { + const result = await source.callBuffer('ZSCAN', key, cursor, 'COUNT', String(SCAN_BATCH)) as [Buffer, Buffer[]]; + cursor = result[0].toString(); + const entries = result[1]; + if (!entries || entries.length === 0) continue; + // entries is [member, score, member, score, ...] as Buffers + const pipeline = target.pipeline(); + for (let i = 0; i < entries.length; i += 2) { + pipeline.zadd(writeKey, entries[i + 1].toString(), entries[i]); + } + await pipeline.exec(); + } while (cursor !== '0'); + } + const pttl = await source.pttl(key); + if (tmp) { + await atomicRenameWithTtl(target, tmp, key, pttl); + } else { + await applyTtl(target, key, pttl); + } + } catch (err) { + if (tmp) { try { await target.del(tmp); } catch { /* best-effort cleanup */ } } + throw err; + } + return true; +} + +// ── Stream ── + +async function migrateStream(source: Valkey, target: Valkey, key: string): Promise { + const tmp = tempKey(key); + const writeKey = tmp ?? key; + let wrote = false; + + try { + if (!tmp) await target.del(key); + + let lastId = '-'; + let hasMore = true; + + while (hasMore) { + const start = lastId === '-' ? '-' : `(${lastId}`; + // Use callBuffer to preserve binary field names and values + const raw = await source.callBuffer( + 'XRANGE', key, start, '+', 'COUNT', String(STREAM_CHUNK), + ) as Buffer[][]; + if (!raw || raw.length === 0) { + hasMore = false; + break; + } + for (const entry of raw) { + // entry[0] = stream ID (always ASCII), entry[1] = [field, value, field, value, ...] + const id = entry[0].toString(); + const fields = entry[1] as unknown as Buffer[]; + await target.callBuffer('XADD', writeKey, id, ...fields); + lastId = id; + wrote = true; + } + if (raw.length < STREAM_CHUNK) { + hasMore = false; + } + } + if (wrote) { + const pttl = await source.pttl(key); + if (tmp) { + await atomicRenameWithTtl(target, tmp, key, pttl); + } else { + await applyTtl(target, key, pttl); + } + } + } catch (err) { + if (tmp) { try { await target.del(tmp); } catch { /* best-effort cleanup */ } } + throw err; + } + return wrote; +} + +// ── TTL ── + +// Lua script: atomically RENAME tmp→key and PEXPIRE in one round-trip. +// KEYS[1] = tmp, KEYS[2] = final key, ARGV[1] = pttl (or "-1" for no expiry, "-2" for expired) +const RENAME_WITH_TTL_LUA = ` +redis.call('RENAME', KEYS[1], KEYS[2]) +local pttl = tonumber(ARGV[1]) +if pttl > 0 then + redis.call('PEXPIRE', KEYS[2], pttl) +elseif pttl == -2 or pttl == 0 then + redis.call('DEL', KEYS[2]) +end +return 1 +`; + + +/** Apply TTL directly to a key (used when temp-key RENAME is not possible). */ +async function applyTtl(target: Valkey, key: string, pttl: number): Promise { + if (pttl > 0) { + await target.pexpire(key, pttl); + } else if (pttl === -2 || pttl === 0) { + await target.del(key); + } +} + +/** + * Atomically RENAME tmp→key and apply PTTL in a single Lua eval. + * Falls back to separate RENAME + PEXPIRE if EVAL is blocked (e.g. by ACL). + */ +async function atomicRenameWithTtl( + target: Valkey, + tmp: string, + key: string, + pttl: number, +): Promise { + try { + await target.call('EVAL', RENAME_WITH_TTL_LUA, '2', tmp, key, String(pttl)); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + // Only fall back for NOSCRIPT / unknown-command / ACL-denied errors. + // Transient errors (OOM, timeouts) should propagate. + if (!/NOSCRIPT|unknown command|DENIED|NOPERM/i.test(msg)) { + throw err; + } + await target.rename(tmp, key); + if (pttl > 0) { + await target.pexpire(key, pttl); + } else if (pttl === -2 || pttl === 0) { + await target.del(key); + } + } +} diff --git a/apps/api/src/migration/migration-execution.service.ts b/apps/api/src/migration/migration-execution.service.ts new file mode 100644 index 00000000..fec4fef9 --- /dev/null +++ b/apps/api/src/migration/migration-execution.service.ts @@ -0,0 +1,314 @@ +import { Injectable, Logger, BadRequestException, NotFoundException, ServiceUnavailableException } from '@nestjs/common'; +import { randomUUID } from 'crypto'; +import { spawn } from 'child_process'; +import { writeFileSync, unlinkSync, existsSync } from 'fs'; +import { join } from 'path'; +import * as os from 'os'; +import type { MigrationExecutionRequest, MigrationExecutionResult, StartExecutionResponse, ExecutionMode } from '@betterdb/shared'; +import { ConnectionRegistry } from '../connections/connection-registry.service'; +import type { ExecutionJob } from './execution/execution-job'; +import { findRedisShakeBinary } from './execution/redisshake-runner'; +import { buildScanReaderToml } from './execution/toml-builder'; +import { parseLogLine } from './execution/log-parser'; +import { runCommandMigration } from './execution/command-migration-worker'; + +@Injectable() +export class MigrationExecutionService { + private readonly logger = new Logger(MigrationExecutionService.name); + private jobs = new Map(); + private readonly MAX_JOBS = 10; + private readonly MAX_LOG_LINES = 500; + + constructor( + private readonly connectionRegistry: ConnectionRegistry, + ) {} + + async startExecution(req: MigrationExecutionRequest): Promise { + const mode: ExecutionMode = req.mode ?? 'redis_shake'; + + // 1. Resolve both connections (throws NotFoundException if missing) + const sourceAdapter = this.connectionRegistry.get(req.sourceConnectionId); + const sourceConfig = this.connectionRegistry.getConfig(req.sourceConnectionId); + const targetAdapter = this.connectionRegistry.get(req.targetConnectionId); + const targetConfig = this.connectionRegistry.getConfig(req.targetConnectionId); + + if (!sourceConfig || !targetConfig) { + throw new NotFoundException('Connection config not found'); + } + + // 2. Validate different connections + if (req.sourceConnectionId === req.targetConnectionId) { + throw new BadRequestException('Source and target must be different connections'); + } + + // 3. Detect if source/target is cluster + const sourceInfo = await sourceAdapter.getInfo(['cluster']); + const sourceClusterSection = (sourceInfo as Record>).cluster ?? {}; + const clusterEnabled = String(sourceClusterSection['cluster_enabled'] ?? '0') === '1'; + + const targetInfo = await targetAdapter.getInfo(['cluster']); + const targetClusterSection = (targetInfo as Record>).cluster ?? {}; + const targetIsCluster = String(targetClusterSection['cluster_enabled'] ?? '0') === '1'; + + // 4. For redis_shake mode, locate the binary upfront + let binaryPath: string | undefined; + if (mode === 'redis_shake') { + try { + binaryPath = findRedisShakeBinary(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + throw new ServiceUnavailableException(message); + } + } + + // 5. Create the job + const id = randomUUID(); + const job: ExecutionJob = { + id, + mode, + status: 'pending', + startedAt: Date.now(), + keysTransferred: 0, + bytesTransferred: 0, + keysSkipped: 0, + totalKeys: 0, + logs: [], + progress: null, + process: null, + tomlPath: null, + pidPath: null, + }; + // 6. Evict old jobs before inserting the new one + this.evictOldJobs(); + + this.jobs.set(id, job); + + // 7. Fire and forget based on mode + if (mode === 'redis_shake') { + const tomlContent = buildScanReaderToml(sourceConfig, targetConfig, clusterEnabled); + const tomlPath = join(os.tmpdir(), `${id}.toml`); + writeFileSync(tomlPath, tomlContent, { encoding: 'utf-8', mode: 0o600 }); + job.tomlPath = tomlPath; + + this.runRedisShake(job, binaryPath!).catch(err => { + this.logger.error(`Execution ${id} failed: ${err.message}`); + }); + } else { + this.runCommandMode(job, sourceConfig, targetConfig, clusterEnabled, targetIsCluster).catch(err => { + this.logger.error(`Execution ${id} failed: ${err.message}`); + }); + } + + return { id, status: 'pending' }; + } + + // ── RedisShake mode ── + + private async runRedisShake(job: ExecutionJob, binaryPath: string): Promise { + try { + const proc = spawn(binaryPath, [job.tomlPath!], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + job.process = proc; + job.status = 'running'; + + // Write PID file for orphan detection on server restart + const pidPath = join(os.tmpdir(), `${job.id}.pid`); + try { + writeFileSync(pidPath, String(proc.pid), { encoding: 'utf-8', mode: 0o600 }); + job.pidPath = pidPath; + } catch { /* non-fatal — orphan detection is best-effort */ } + + const handleData = (chunk: Buffer) => { + const lines = chunk.toString().split('\n'); + for (const line of lines) { + if (!line) continue; + job.logs.push(sanitizeLogLine(line)); + if (job.logs.length > this.MAX_LOG_LINES) { + job.logs.shift(); + } + const parsed = parseLogLine(line); + if (parsed.keysTransferred !== null) job.keysTransferred = parsed.keysTransferred; + if (parsed.bytesTransferred !== null) job.bytesTransferred = parsed.bytesTransferred; + if (parsed.progress !== null) job.progress = parsed.progress; + } + }; + + proc.stdout.on('data', handleData); + proc.stderr.on('data', handleData); + + const code = await new Promise((resolve, reject) => { + proc.on('exit', (exitCode) => resolve(exitCode ?? 1)); + proc.on('error', reject); + }); + + // Status may have been set to 'cancelled' by stopExecution() while the process was running + const statusAfterExit = job.status as string; + if (code === 0) { + if (statusAfterExit !== 'cancelled') { + job.status = 'completed'; + job.progress = 100; + } + } else if (statusAfterExit !== 'cancelled') { + job.status = 'failed'; + job.error = `RedisShake exited with code ${code}`; + } + } catch (err: unknown) { + if ((job.status as string) !== 'cancelled') { + const message = err instanceof Error ? err.message : String(err); + job.status = 'failed'; + job.error = message; + this.logger.error(`Execution ${job.id} error: ${message}`); + } + } finally { + if (!job.completedAt) { + job.completedAt = Date.now(); + } + for (const path of [job.tomlPath, job.pidPath]) { + if (path) { + try { + if (existsSync(path)) unlinkSync(path); + } catch { /* ignore cleanup errors */ } + } + } + job.process = null; + job.tomlPath = null; + job.pidPath = null; + } + } + + // ── Command-based mode ── + + private async runCommandMode( + job: ExecutionJob, + sourceConfig: Parameters[0]['sourceConfig'], + targetConfig: Parameters[0]['targetConfig'], + sourceIsCluster: boolean, + targetIsCluster: boolean, + ): Promise { + job.status = 'running'; + try { + await runCommandMigration({ + sourceConfig, + targetConfig, + sourceIsCluster, + targetIsCluster, + job, + maxLogLines: this.MAX_LOG_LINES, + }); + + if ((job.status as string) !== 'cancelled') { + job.status = 'completed'; + } + } catch (err: unknown) { + if ((job.status as string) !== 'cancelled') { + const message = err instanceof Error ? err.message : String(err); + job.status = 'failed'; + job.error = message; + this.logger.error(`Execution ${job.id} error: ${message}`); + } + } finally { + if (!job.completedAt) { + job.completedAt = Date.now(); + } + } + } + + // ── Shared methods ── + + stopExecution(id: string): boolean { + const job = this.jobs.get(id); + if (!job) return false; + + // Idempotent for terminal states + if (job.status === 'completed' || job.status === 'failed' || job.status === 'cancelled') { + return true; + } + + job.status = 'cancelled'; + + // For redis_shake mode, kill the subprocess + if (job.process) { + const proc = job.process; + try { + proc.kill('SIGTERM'); + } catch { /* process may already be dead */ } + + setTimeout(() => { + if (job.process) { + try { + proc.kill('SIGKILL'); + } catch { /* ignore */ } + } + }, 3000); + } + // For command mode, the worker checks job.status === 'cancelled' between batches + + return true; + } + + getExecution(id: string): MigrationExecutionResult | undefined { + const job = this.jobs.get(id); + if (!job) return undefined; + + return { + id: job.id, + status: job.status, + mode: job.mode, + startedAt: job.startedAt, + completedAt: job.completedAt, + error: job.error, + keysTransferred: job.keysTransferred, + bytesTransferred: job.bytesTransferred, + keysSkipped: job.keysSkipped, + totalKeys: job.totalKeys ?? undefined, + logs: [...job.logs], + progress: job.progress, + }; + } + + private evictOldJobs(): void { + if (this.jobs.size < this.MAX_JOBS) return; + + const terminal = Array.from(this.jobs.entries()) + .filter(([, j]) => j.status === 'completed' || j.status === 'failed' || j.status === 'cancelled') + .sort((a, b) => a[1].startedAt - b[1].startedAt); + + for (const [id] of terminal) { + if (this.jobs.size < this.MAX_JOBS) break; + this.jobs.delete(id); + } + + if (this.jobs.size >= this.MAX_JOBS) { + throw new ServiceUnavailableException( + `Execution job limit reached (${this.MAX_JOBS}). All slots occupied by running jobs — try again later.`, + ); + } + } +} + +// Redact credentials from RedisShake log lines before serving to the frontend +const SENSITIVE_KEYS = /(?:password|username|auth|requirepass|masterauth|token)/i; + +function sanitizeLogLine(line: string): string { + let sanitized = line; + // 1. Quoted sensitive fields: password = "secret" or username:"admin" + sanitized = sanitized.replace( + new RegExp(`(${SENSITIVE_KEYS.source})\\s*[=:]\\s*"(?:[^"\\\\]|\\\\.)*"`, 'gi'), + (match) => { + const eqIdx = match.search(/[=:]/); + return match.slice(0, eqIdx + 1) + ' "***"'; + }, + ); + // 2. Unquoted sensitive fields (skip already-redacted quoted ones) + sanitized = sanitized.replace( + new RegExp(`(${SENSITIVE_KEYS.source})\\s*[=:]\\s*(?!["*])\\S+`, 'gi'), + (match) => { + const eqIdx = match.search(/[=:]/); + return match.slice(0, eqIdx + 1) + ' ***'; + }, + ); + // 3. URL credentials: redis://user:pass@host + sanitized = sanitized.replace(/\/\/[^:]+:[^@]+@/g, '//***:***@'); + return sanitized; +} diff --git a/apps/api/src/migration/migration-validation.service.ts b/apps/api/src/migration/migration-validation.service.ts new file mode 100644 index 00000000..e3bbca60 --- /dev/null +++ b/apps/api/src/migration/migration-validation.service.ts @@ -0,0 +1,269 @@ +import { Injectable, Inject, Logger, NotFoundException, BadRequestException, ServiceUnavailableException } from '@nestjs/common'; +import { randomUUID } from 'crypto'; +import Valkey from 'iovalkey'; +import type { + MigrationValidationRequest, + MigrationValidationResult, + StartValidationResponse, + MigrationAnalysisResult, + DatabaseConnectionConfig, +} from '@betterdb/shared'; +import { ConnectionRegistry } from '../connections/connection-registry.service'; +import type { StoragePort } from '../common/interfaces/storage-port.interface'; +import type { ValidationJob } from './validation/validation-job'; +import { compareKeyCounts } from './validation/key-count-comparator'; +import { validateSample } from './validation/sample-validator'; +import { compareBaseline } from './validation/baseline-comparator'; +import { MigrationService } from './migration.service'; +import { createClient, createTargetClient } from './execution/client-factory'; + +@Injectable() +export class MigrationValidationService { + private readonly logger = new Logger(MigrationValidationService.name); + private jobs = new Map(); + private readonly MAX_JOBS = 10; + + constructor( + private readonly connectionRegistry: ConnectionRegistry, + @Inject('STORAGE_CLIENT') private readonly storage: StoragePort, + private readonly migrationService: MigrationService, + ) {} + + async startValidation(req: MigrationValidationRequest): Promise { + // 1. Resolve both connections (throws NotFoundException if missing) + this.connectionRegistry.get(req.sourceConnectionId); + this.connectionRegistry.get(req.targetConnectionId); + const sourceConfig = this.connectionRegistry.getConfig(req.sourceConnectionId); + const targetConfig = this.connectionRegistry.getConfig(req.targetConnectionId); + + if (!sourceConfig || !targetConfig) { + throw new NotFoundException('Connection config not found'); + } + + // 2. Validate different connections + if (req.sourceConnectionId === req.targetConnectionId) { + throw new BadRequestException('Source and target must be different connections'); + } + + // 3. Optionally retrieve Phase 1 analysis result + let analysisResult: MigrationAnalysisResult | undefined; + if (req.analysisId) { + const job = this.migrationService.getJob(req.analysisId); + if (job && job.status === 'completed') { + // Verify the analysis belongs to the same source/target pair + if ( + (job.sourceConnectionId && job.sourceConnectionId !== req.sourceConnectionId) || + (job.targetConnectionId && job.targetConnectionId !== req.targetConnectionId) + ) { + throw new BadRequestException('Analysis does not match the provided source/target connections'); + } + analysisResult = job; + } + } + + // 4. Create job + const id = randomUUID(); + const job: ValidationJob = { + id, + status: 'pending', + progress: 0, + createdAt: Date.now(), + result: { + sourceConnectionId: req.sourceConnectionId, + targetConnectionId: req.targetConnectionId, + }, + cancelled: false, + }; + // 5. Evict old jobs before inserting the new one + this.evictOldJobs(); + + this.jobs.set(id, job); + + // 6. Fire and forget + const targetAdapter = this.connectionRegistry.get(req.targetConnectionId); + this.runValidation(job, sourceConfig, targetConfig, targetAdapter, req.migrationStartedAt, analysisResult).catch(err => { + this.logger.error(`Validation ${id} failed: ${err.message}`); + }); + + return { id, status: 'pending' }; + } + + private async runValidation( + job: ValidationJob, + sourceConfig: DatabaseConnectionConfig, + targetConfig: DatabaseConnectionConfig, + targetAdapter: import('../common/interfaces/database-port.interface').DatabasePort, + migrationStartedAt?: number, + analysisResult?: MigrationAnalysisResult, + ): Promise { + let sourceClient: Valkey | null = null; + let targetClient: Valkey | null = null; + + try { + job.status = 'running'; + + // Detect if target is a cluster + const targetInfo = await targetAdapter.getInfo(['cluster']); + const targetClusterSection = (targetInfo as Record>).cluster ?? {}; + const targetIsCluster = String(targetClusterSection['cluster_enabled'] ?? '0') === '1'; + + // Create temporary iovalkey clients — same pattern as command-migration-worker.ts + sourceClient = createClient(sourceConfig, 'BetterDB-Validation-Source'); + targetClient = createTargetClient(targetConfig, 'BetterDB-Validation-Target', targetIsCluster); + + // Step 1: Connect check (5%) + try { + await sourceClient.connect(); + } catch (err: unknown) { + // Risk #2: Source may no longer be reachable + const message = err instanceof Error ? err.message : String(err); + job.status = 'failed'; + job.error = `Source instance is not reachable. Ensure it is still running before validating. (${message})`; + return; + } + + try { + await targetClient.connect(); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + job.status = 'failed'; + job.error = `Target instance is not reachable. (${message})`; + return; + } + + // PING both + try { + await Promise.all([sourceClient.ping(), targetClient.ping()]); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + job.status = 'failed'; + job.error = `Source instance is not reachable. Ensure it is still running before validating. (${message})`; + return; + } + + job.progress = 5; + + if (job.cancelled) return; + + // Step 2: Key count comparison (20%) + const keyCount = await compareKeyCounts(sourceClient, targetClient, analysisResult); + job.result.keyCount = keyCount; + job.progress = 20; + + if (job.cancelled) return; + + // Step 3: Sample validation (20–70%) + const sampleValidation = await validateSample(sourceClient, targetClient, 500); + job.result.sampleValidation = sampleValidation; + job.progress = 70; + + if (job.cancelled) return; + + // Step 4: Baseline comparison (80%) + if (migrationStartedAt) { + const baseline = await compareBaseline( + this.storage, + sourceConfig.id, + targetAdapter, + migrationStartedAt, + ); + job.result.baseline = baseline; + } else { + job.result.baseline = { + available: false, + unavailableReason: 'Migration start time not provided — cannot determine baseline window.', + snapshotCount: 0, + baselineWindowMs: 0, + metrics: [], + }; + } + job.progress = 80; + + if (job.cancelled) return; + + // Step 5: Compute summary (100%) + const baselineIssues = (job.result.baseline?.metrics ?? []) + .filter(m => m.status !== 'normal' && m.status !== 'unavailable').length; + + const issueCount = + (sampleValidation.missing ?? 0) + + (sampleValidation.typeMismatches ?? 0) + + (sampleValidation.valueMismatches ?? 0) + + baselineIssues; + + const passed = issueCount === 0 && Math.abs(keyCount.discrepancyPercent) < 1; + + job.result.issueCount = issueCount; + job.result.passed = passed; + job.progress = 100; + job.status = 'completed'; + } catch (err: unknown) { + if (!job.cancelled) { + const message = err instanceof Error ? err.message : String(err); + job.status = 'failed'; + job.error = message; + this.logger.error(`Validation ${job.id} error: ${message}`); + } + } finally { + // Ensure cancelled jobs get a terminal status + if (job.cancelled && job.status === 'running') { + job.status = 'cancelled'; + job.error = job.error ?? 'Cancelled by user'; + } + job.completedAt = Date.now(); + // Graceful cleanup — never Promise.all, never disconnect() + const clients = [sourceClient, targetClient].filter((c): c is Valkey => c !== null); + await Promise.allSettled(clients.map(c => c.quit())); + } + } + + cancelValidation(id: string): boolean { + const job = this.jobs.get(id); + if (!job) return false; + + if (job.status === 'completed' || job.status === 'failed' || job.status === 'cancelled') { + return true; // Already terminal + } + + job.cancelled = true; + job.status = 'cancelled'; + job.error = 'Cancelled by user'; + job.completedAt = Date.now(); + return true; + } + + getValidation(id: string): MigrationValidationResult | undefined { + const job = this.jobs.get(id); + if (!job) return undefined; + + return { + id: job.id, + status: job.status, + progress: job.progress, + createdAt: job.createdAt, + completedAt: job.completedAt, + error: job.error, + ...job.result, + }; + } + + private evictOldJobs(): void { + if (this.jobs.size < this.MAX_JOBS) return; + + const terminal = Array.from(this.jobs.entries()) + .filter(([, j]) => j.status === 'completed' || j.status === 'failed' || j.status === 'cancelled') + .sort((a, b) => a[1].createdAt - b[1].createdAt); + + for (const [id] of terminal) { + if (this.jobs.size < this.MAX_JOBS) break; + this.jobs.delete(id); + } + + if (this.jobs.size >= this.MAX_JOBS) { + throw new ServiceUnavailableException( + `Validation job limit reached (${this.MAX_JOBS}). All slots occupied by running jobs — try again later.`, + ); + } + } +} + diff --git a/apps/api/src/migration/migration.controller.ts b/apps/api/src/migration/migration.controller.ts new file mode 100644 index 00000000..747b7590 --- /dev/null +++ b/apps/api/src/migration/migration.controller.ts @@ -0,0 +1,141 @@ +import { Controller, Get, Post, Delete, Param, Body, UseGuards, NotFoundException, BadRequestException } from '@nestjs/common'; +import type { MigrationAnalysisRequest, StartAnalysisResponse, MigrationAnalysisResult, MigrationExecutionRequest, StartExecutionResponse, MigrationExecutionResult, MigrationValidationRequest, StartValidationResponse, MigrationValidationResult } from '@betterdb/shared'; +import { Feature } from '@betterdb/shared'; +import { LicenseGuard } from '@proprietary/licenses'; +import { RequiresFeature } from '@proprietary/licenses/requires-feature.decorator'; +import { MigrationService } from './migration.service'; +import { MigrationExecutionService } from './migration-execution.service'; +import { MigrationValidationService } from './migration-validation.service'; + +// Migration analysis is intentionally community-tier (no license guard). +// MIGRATION_EXECUTION gating applies to the execution phase only. +@Controller('migration') +export class MigrationController { + constructor( + private readonly migrationService: MigrationService, + private readonly executionService: MigrationExecutionService, + private readonly validationService: MigrationValidationService, + ) {} + + // ── Analysis endpoints (community-tier) ── + + @Post('analysis') + async startAnalysis(@Body() body: MigrationAnalysisRequest): Promise { + if (!body.sourceConnectionId) { + throw new BadRequestException('sourceConnectionId is required'); + } + if (!body.targetConnectionId) { + throw new BadRequestException('targetConnectionId is required'); + } + if (body.sourceConnectionId === body.targetConnectionId) { + throw new BadRequestException('Source and target must be different connections'); + } + if (body.scanSampleSize !== undefined) { + if (body.scanSampleSize < 1000 || body.scanSampleSize > 50000) { + throw new BadRequestException('scanSampleSize must be between 1000 and 50000'); + } + } + return this.migrationService.startAnalysis(body); + } + + @Get('analysis/:id') + getJob(@Param('id') id: string): MigrationAnalysisResult { + const job = this.migrationService.getJob(id); + if (!job) { + throw new NotFoundException(`Analysis job '${id}' not found`); + } + return job; + } + + @Delete('analysis/:id') + cancelJob(@Param('id') id: string): { cancelled: boolean } { + const success = this.migrationService.cancelJob(id); + if (!success) { + throw new NotFoundException(`Analysis job '${id}' not found`); + } + return { cancelled: true }; + } + + // ── Execution endpoints (Pro-tier) ── + + @Post('execution') + @UseGuards(LicenseGuard) + @RequiresFeature(Feature.MIGRATION_EXECUTION) + async startExecution(@Body() body: MigrationExecutionRequest): Promise { + if (!body.sourceConnectionId) { + throw new BadRequestException('sourceConnectionId is required'); + } + if (!body.targetConnectionId) { + throw new BadRequestException('targetConnectionId is required'); + } + if (body.sourceConnectionId === body.targetConnectionId) { + throw new BadRequestException('Source and target must be different connections'); + } + if (body.mode && body.mode !== 'redis_shake' && body.mode !== 'command') { + throw new BadRequestException('mode must be "redis_shake" or "command"'); + } + return this.executionService.startExecution(body); + } + + @Get('execution/:id') + @UseGuards(LicenseGuard) + @RequiresFeature(Feature.MIGRATION_EXECUTION) + getExecution(@Param('id') id: string): MigrationExecutionResult { + const result = this.executionService.getExecution(id); + if (!result) { + throw new NotFoundException(`Execution job '${id}' not found`); + } + return result; + } + + @Delete('execution/:id') + @UseGuards(LicenseGuard) + @RequiresFeature(Feature.MIGRATION_EXECUTION) + stopExecution(@Param('id') id: string): { stopped: true } { + const found = this.executionService.stopExecution(id); + if (!found) { + throw new NotFoundException(`Execution job '${id}' not found`); + } + return { stopped: true }; + } + + // ── Validation endpoints (Pro-tier) ── + + @Post('validation') + @UseGuards(LicenseGuard) + @RequiresFeature(Feature.MIGRATION_EXECUTION) + async startValidation(@Body() body: MigrationValidationRequest): Promise { + if (!body.sourceConnectionId) { + throw new BadRequestException('sourceConnectionId is required'); + } + if (!body.targetConnectionId) { + throw new BadRequestException('targetConnectionId is required'); + } + if (body.sourceConnectionId === body.targetConnectionId) { + throw new BadRequestException('Source and target must be different connections'); + } + return this.validationService.startValidation(body); + } + + @Get('validation/:id') + @UseGuards(LicenseGuard) + @RequiresFeature(Feature.MIGRATION_EXECUTION) + getValidation(@Param('id') id: string): MigrationValidationResult { + const result = this.validationService.getValidation(id); + if (!result) { + throw new NotFoundException(`Validation job '${id}' not found`); + } + return result; + } + + @Delete('validation/:id') + @UseGuards(LicenseGuard) + @RequiresFeature(Feature.MIGRATION_EXECUTION) + cancelValidation(@Param('id') id: string): { cancelled: true } { + const found = this.validationService.cancelValidation(id); + if (!found) { + throw new NotFoundException(`Validation job '${id}' not found`); + } + return { cancelled: true }; + } +} diff --git a/apps/api/src/migration/migration.module.ts b/apps/api/src/migration/migration.module.ts new file mode 100644 index 00000000..40a0127e --- /dev/null +++ b/apps/api/src/migration/migration.module.ts @@ -0,0 +1,15 @@ +import { Module } from '@nestjs/common'; +import { ConnectionsModule } from '../connections/connections.module'; +import { StorageModule } from '../storage/storage.module'; +import { MigrationController } from './migration.controller'; +import { MigrationService } from './migration.service'; +import { MigrationExecutionService } from './migration-execution.service'; +import { MigrationValidationService } from './migration-validation.service'; + +@Module({ + imports: [ConnectionsModule, StorageModule], + controllers: [MigrationController], + providers: [MigrationService, MigrationExecutionService, MigrationValidationService], + exports: [MigrationService, MigrationExecutionService, MigrationValidationService], +}) +export class MigrationModule {} diff --git a/apps/api/src/migration/migration.service.ts b/apps/api/src/migration/migration.service.ts new file mode 100644 index 00000000..1dbbde01 --- /dev/null +++ b/apps/api/src/migration/migration.service.ts @@ -0,0 +1,520 @@ +import { Injectable, Logger, NotFoundException } from '@nestjs/common'; +import { randomUUID } from 'crypto'; +import Valkey from 'iovalkey'; +import type { MigrationAnalysisRequest, MigrationAnalysisResult, StartAnalysisResponse, DataTypeBreakdown, DataTypeCount, TtlDistribution } from '@betterdb/shared'; +import { ConnectionRegistry } from '../connections/connection-registry.service'; +import type { AnalysisJob } from './analysis/analysis-job'; +import { sampleKeyTypes } from './analysis/type-sampler'; +import { sampleTtls } from './analysis/ttl-sampler'; +import { detectHfe } from './analysis/hfe-detector'; +import { analyzeCommands } from './analysis/commandlog-analyzer'; +import { buildInstanceMeta, checkCompatibility } from './analysis/compatibility-checker'; + +@Injectable() +export class MigrationService { + private readonly logger = new Logger(MigrationService.name); + private jobs = new Map(); + private readonly MAX_JOBS = 20; + private readonly STUCK_JOB_TTL_MS = 2 * 60 * 60 * 1000; // 2 hours + + constructor( + private readonly connectionRegistry: ConnectionRegistry, + ) {} + + async startAnalysis(req: MigrationAnalysisRequest): Promise { + // Verify both connections exist before creating job (get() throws NotFoundException if not found) + this.connectionRegistry.get(req.sourceConnectionId); + this.connectionRegistry.get(req.targetConnectionId); + + this.evictOldJobs(); + + const id = randomUUID(); + const job: AnalysisJob = { + id, + status: 'pending', + progress: 0, + createdAt: Date.now(), + result: { id, status: 'pending', progress: 0, createdAt: Date.now() }, + cancelled: false, + nodeClients: [], + }; + + this.jobs.set(id, job); + + // Fire and forget — do not await + this.runAnalysis(job, req).catch(err => { + this.logger.error(`Analysis ${id} failed: ${err.message}`); + }); + + return { id, status: 'pending' }; + } + + getJob(id: string): MigrationAnalysisResult | undefined { + const job = this.jobs.get(id); + if (!job) return undefined; + if (this.isJobStuck(job)) { + this.logger.warn(`Analysis ${id} exceeded stuck-job TTL — cancelling`); + this.cancelJob(id); + // Return the job with its cancelled/failed status rather than 404 + } + return { + ...job.result, + id: job.id, + status: job.status, + progress: job.progress, + createdAt: job.createdAt, + completedAt: job.completedAt, + error: job.error ?? (job.status === 'cancelled' ? 'Analysis timed out' : undefined), + } as MigrationAnalysisResult; + } + + cancelJob(id: string): boolean { + const job = this.jobs.get(id); + if (!job) return false; + job.cancelled = true; + job.status = 'cancelled'; + // Immediately quit all temporary node clients + for (const client of job.nodeClients) { + client.quit().catch(() => {}); + } + job.nodeClients = []; + return true; + } + + private async runAnalysis(job: AnalysisJob, req: MigrationAnalysisRequest): Promise { + const scanSampleSize = req.scanSampleSize ?? 10_000; + const tempClients: Valkey[] = []; + + try { + job.status = 'running'; + job.progress = 5; + + // Step 1: Resolve source connection + const adapter = this.connectionRegistry.get(req.sourceConnectionId); + const config = this.connectionRegistry.getConfig(req.sourceConnectionId); + const capabilities = adapter.getCapabilities(); + + job.result.sourceConnectionId = req.sourceConnectionId; + job.result.sourceConnectionName = config?.name; + job.result.sourceDbType = capabilities.dbType; + job.result.sourceDbVersion = capabilities.version; + + if (job.cancelled) return; + job.progress = 10; + + // Step 2: Get source server info (keyspace for total key count, memory) + const info = await adapter.getInfo(['keyspace', 'memory', 'cluster', 'server', 'persistence']) as Record>; + const keyspaceSection = info.keyspace ?? {}; + const memorySection = info.memory ?? {}; + const clusterSection = info.cluster ?? {}; + + // Parse total keys from keyspace section + let totalKeys = 0; + for (const [key, val] of Object.entries(keyspaceSection)) { + if (key.startsWith('db') && typeof val === 'string') { + const match = val.match(/keys=(\d+)/); + if (match) totalKeys += parseInt(match[1], 10); + } + } + job.result.totalKeys = totalKeys; + + // Parse used_memory + const usedMemory = Number(memorySection['used_memory']) || 0; + job.result.totalMemoryBytes = usedMemory; + + if (job.cancelled) return; + job.progress = 12; + + // Step 2b: Read target info + if (job.cancelled) return; + + const targetAdapter = this.connectionRegistry.get(req.targetConnectionId); + const targetConfig = this.connectionRegistry.getConfig(req.targetConnectionId); + const targetInfo = await targetAdapter.getInfo(['server', 'keyspace', 'cluster', 'memory', 'persistence']); + const targetCapabilities = targetAdapter.getCapabilities(); + + let targetAclUsers: string[] = []; + try { + const client = targetAdapter.getClient(); + const result = await client.call('ACL', 'USERS') as string[]; + targetAclUsers = result ?? []; + } catch { /* ignore - ACL not supported or no permission */ } + + job.result.targetConnectionId = req.targetConnectionId; + job.result.targetConnectionName = targetConfig?.name; + job.result.targetDbType = targetCapabilities.dbType; + job.result.targetDbVersion = targetCapabilities.version; + const targetClusterSection = (targetInfo as Record>).cluster ?? {}; + job.result.targetIsCluster = String(targetClusterSection['cluster_enabled'] ?? '0') === '1'; + + if (job.cancelled) return; + job.progress = 13; + + // Step 3: Cluster check (source) + let isCluster = false; + let clusterMasterCount = 0; + const scanClients: Valkey[] = []; + + const clusterEnabled = String(clusterSection['cluster_enabled'] ?? '0'); + if (clusterEnabled === '1') { + isCluster = true; + const nodes = await adapter.getClusterNodes(); + const masters = nodes.filter(n => n.flags.includes('master')); + clusterMasterCount = masters.length; + + for (const master of masters) { + // Parse address: 'host:port@clusterport' (host may be IPv6) + const addrPart = master.address?.split('@')[0] ?? ''; + const lastColon = addrPart.lastIndexOf(':'); + let host = lastColon > 0 ? addrPart.substring(0, lastColon) : ''; + const port = lastColon > 0 ? parseInt(addrPart.substring(lastColon + 1), 10) : NaN; + // Strip IPv6 brackets — iovalkey expects bare addresses + if (host.startsWith('[') && host.endsWith(']')) { + host = host.slice(1, -1); + } + if (!host || isNaN(port)) continue; + + const client = new Valkey({ + host, + port, + username: config?.username || undefined, + password: config?.password || undefined, + tls: config?.tls ? {} : undefined, + lazyConnect: true, + connectionName: 'BetterDB-Migration-Analysis', + }); + await client.connect(); + tempClients.push(client); + job.nodeClients.push(client); + scanClients.push(client); + } + } else { + scanClients.push(adapter.getClient()); + } + + job.result.isCluster = isCluster; + job.result.clusterMasterCount = clusterMasterCount; + job.result.sampledPerNode = scanSampleSize; + + if (job.cancelled) return; + job.progress = 15; + + // Step 4: Type sampling (SCAN + TYPE) + const sampledKeys = await sampleKeyTypes( + scanClients, + scanSampleSize, + (scanned) => { + const progressRange = 50 - 15; // 15-50% + const totalExpected = scanSampleSize * scanClients.length; + const fraction = Math.min(scanned / totalExpected, 1); + job.progress = Math.round(15 + fraction * progressRange); + }, + ); + + job.result.sampledKeys = sampledKeys.length; + + if (job.cancelled) return; + job.progress = 50; + + // Step 5: Memory sampling (per-node to avoid cross-slot errors in cluster mode) + const keysByClientIndex = new Map(); + for (const sk of sampledKeys) { + const group = keysByClientIndex.get(sk.clientIndex) ?? []; + group.push(sk); + keysByClientIndex.set(sk.clientIndex, group); + } + + const memoryByType = new Map(); + let memoryProcessed = 0; + + for (const [clientIndex, clientKeys] of keysByClientIndex) { + const client = scanClients[clientIndex]; + for (let i = 0; i < clientKeys.length; i += 1000) { + if (job.cancelled) return; + const batch = clientKeys.slice(i, i + 1000); + const pipeline = client.pipeline(); + for (const { key } of batch) { + pipeline.call('MEMORY', 'USAGE', key, 'SAMPLES', '0'); + } + const results = await pipeline.exec(); + if (results) { + for (let j = 0; j < batch.length; j++) { + const [err, mem] = results[j] ?? []; + const bytes = err ? 0 : Number(mem) || 0; + const t = batch[j].type; + const entry = memoryByType.get(t) ?? { count: 0, bytes: 0 }; + entry.count++; + entry.bytes += bytes; + memoryByType.set(t, entry); + } + } + memoryProcessed += batch.length; + job.progress = Math.round(50 + (memoryProcessed / sampledKeys.length) * 15); + } + } + + // Build DataTypeBreakdown + const knownTypes = new Set(['string', 'hash', 'list', 'set', 'zset', 'stream']); + let otherCount = 0; + let otherBytes = 0; + + for (const [typeName, data] of memoryByType) { + if (!knownTypes.has(typeName)) { + otherCount += data.count; + otherBytes += data.bytes; + } + } + + const buildDtc = (typeName: string): DataTypeCount => { + const data = memoryByType.get(typeName); + if (!data) return { count: 0, sampledMemoryBytes: 0, estimatedTotalMemoryBytes: 0 }; + return { + count: data.count, + sampledMemoryBytes: data.bytes, + estimatedTotalMemoryBytes: sampledKeys.length > 0 + ? Math.round((data.bytes / sampledKeys.length) * totalKeys) + : 0, + }; + }; + + const breakdown: DataTypeBreakdown = { + string: buildDtc('string'), + hash: buildDtc('hash'), + list: buildDtc('list'), + set: buildDtc('set'), + zset: buildDtc('zset'), + stream: buildDtc('stream'), + other: { + count: otherCount, + sampledMemoryBytes: otherBytes, + estimatedTotalMemoryBytes: sampledKeys.length > 0 + ? Math.round((otherBytes / sampledKeys.length) * totalKeys) + : 0, + }, + }; + + job.result.dataTypeBreakdown = breakdown; + + // Compute estimated total memory + const totalSampledBytes = Array.from(memoryByType.values()).reduce((s, d) => s + d.bytes, 0); + job.result.estimatedTotalMemoryBytes = sampledKeys.length > 0 + ? Math.round((totalSampledBytes / sampledKeys.length) * totalKeys) + : 0; + + if (job.cancelled) return; + job.progress = 65; + + // Step 6: TTL distribution (per-node) + const mergedTtl: TtlDistribution = { + noExpiry: 0, expiresWithin1h: 0, expiresWithin24h: 0, + expiresWithin7d: 0, expiresAfter7d: 0, sampledKeyCount: sampledKeys.length, + }; + for (const [clientIndex, clientKeys] of keysByClientIndex) { + const nodeTtl = await sampleTtls(scanClients[clientIndex], clientKeys.map(k => k.key)); + mergedTtl.noExpiry += nodeTtl.noExpiry; + mergedTtl.expiresWithin1h += nodeTtl.expiresWithin1h; + mergedTtl.expiresWithin24h += nodeTtl.expiresWithin24h; + mergedTtl.expiresWithin7d += nodeTtl.expiresWithin7d; + mergedTtl.expiresAfter7d += nodeTtl.expiresAfter7d; + } + job.result.ttlDistribution = mergedTtl; + + if (job.cancelled) return; + job.progress = 75; + + // Step 7: HFE detection (per-node) + if (capabilities.dbType === 'valkey') { + const hashKeys = sampledKeys.filter(k => k.type === 'hash'); + const totalEstimatedHashKeys = totalKeys > 0 && sampledKeys.length > 0 + ? Math.round((hashKeys.length / sampledKeys.length) * totalKeys) + : hashKeys.length; + + // Group hash keys by originating client + const hashByClient = new Map(); + for (const hk of hashKeys) { + const group = hashByClient.get(hk.clientIndex) ?? []; + group.push(hk.key); + hashByClient.set(hk.clientIndex, group); + } + + let hfeDetected = false; + let hfeSupported = true; + let hfeKeyCount = 0; + let hfeOversizedHashesSkipped = 0; + + for (const [clientIndex, nodeHashKeys] of hashByClient) { + // Each node's estimated share of total hash keys + const nodeEstimatedTotal = hashKeys.length > 0 + ? Math.round((nodeHashKeys.length / hashKeys.length) * totalEstimatedHashKeys) + : 0; + const hfeResult = await detectHfe(scanClients[clientIndex], nodeHashKeys, nodeEstimatedTotal); + if (!hfeResult.hfeSupported) hfeSupported = false; + if (hfeResult.hfeDetected) hfeDetected = true; + hfeKeyCount += hfeResult.hfeKeyCount; + hfeOversizedHashesSkipped += hfeResult.hfeOversizedHashesSkipped; + } + + job.result.hfeDetected = hfeDetected; + job.result.hfeSupported = hfeSupported; + job.result.hfeKeyCount = hfeKeyCount; + job.result.hfeOversizedHashesSkipped = hfeOversizedHashesSkipped; + } else { + job.result.hfeSupported = false; + job.result.hfeDetected = false; + } + + if (job.cancelled) return; + job.progress = 85; + + // Step 8: Command analysis + job.result.commandAnalysis = await analyzeCommands(adapter); + + if (job.cancelled) return; + job.progress = 90; + + // Step 9: Compatibility checking + // Fetch source ACL users + let sourceAclUsers: string[] = []; + try { + const sourceClient = adapter.getClient(); + const result = await sourceClient.call('ACL', 'USERS') as string[]; + sourceAclUsers = result ?? []; + } catch { /* ignore - ACL not supported or no permission */ } + + // Fetch RDB save config from both instances for reliable persistence detection + let sourceRdbSaveConfig: string | undefined; + let targetRdbSaveConfig: string | undefined; + try { + const sourceClient = adapter.getClient(); + const result = await sourceClient.call('CONFIG', 'GET', 'save') as string[]; + if (result && result.length >= 2) sourceRdbSaveConfig = result[1]; + } catch { /* ignore - CONFIG not permitted */ } + try { + const targetClient = targetAdapter.getClient(); + const result = await targetClient.call('CONFIG', 'GET', 'save') as string[]; + if (result && result.length >= 2) targetRdbSaveConfig = result[1]; + } catch { /* ignore - CONFIG not permitted */ } + + // Build source meta (buildInstanceMeta expects a flat key-value object) + const flatSourceInfo = flattenInfo(info); + const sourceMeta = buildInstanceMeta(flatSourceInfo, capabilities, sourceAclUsers, sourceRdbSaveConfig); + + // Fetch source modules + try { + const sourceClient = adapter.getClient(); + const moduleResult = await sourceClient.call('MODULE', 'LIST') as unknown[]; + sourceMeta.modules = parseModuleList(moduleResult); + } catch { /* ignore */ } + + // Build target meta + const flatTargetInfo = flattenInfo(targetInfo); + const targetMeta = buildInstanceMeta(flatTargetInfo, targetCapabilities, targetAclUsers, targetRdbSaveConfig); + + // Fetch target modules + try { + const targetClient = targetAdapter.getClient(); + const moduleResult = await targetClient.call('MODULE', 'LIST') as unknown[]; + targetMeta.modules = parseModuleList(moduleResult); + } catch { /* ignore */ } + + const incompatibilities = checkCompatibility(sourceMeta, targetMeta, job.result.hfeDetected ?? false); + job.result.incompatibilities = incompatibilities; + job.result.blockingCount = incompatibilities.filter(i => i.severity === 'blocking').length; + job.result.warningCount = incompatibilities.filter(i => i.severity === 'warning').length; + + if (job.cancelled) return; + job.progress = 95; + + // Done + job.progress = 100; + job.status = 'completed'; + job.completedAt = Date.now(); + job.result.status = 'completed'; + job.result.completedAt = job.completedAt; + + this.logger.log(`Analysis ${job.id} completed: blocking=${job.result.blockingCount}, warnings=${job.result.warningCount}, sampledKeys=${sampledKeys.length}, totalKeys=${totalKeys}`); + + } catch (err: unknown) { + if (!job.cancelled) { + const message = err instanceof Error ? err.message : String(err); + job.status = 'failed'; + job.error = message; + job.result.status = 'failed'; + job.result.error = job.error; + job.completedAt = Date.now(); + this.logger.error(`Analysis ${job.id} failed: ${job.error}`); + } + } finally { + // Only quit temporary per-node clients, never the adapter's client + await Promise.allSettled(tempClients.map(c => c.quit())); + job.nodeClients = []; + } + } + + private evictOldJobs(): void { + if (this.jobs.size < this.MAX_JOBS) return; + + // First: cancel and evict stuck running jobs + for (const [id, job] of this.jobs) { + if (this.isJobStuck(job)) { + this.logger.warn(`Evicting stuck analysis ${id}`); + this.cancelJob(id); + this.jobs.delete(id); + } + } + + // Then: evict oldest completed/failed/cancelled + if (this.jobs.size >= this.MAX_JOBS) { + const terminal = Array.from(this.jobs.entries()) + .filter(([, j]) => j.status === 'completed' || j.status === 'failed' || j.status === 'cancelled') + .sort((a, b) => a[1].createdAt - b[1].createdAt); + + for (const [id] of terminal) { + if (this.jobs.size < this.MAX_JOBS) break; + this.jobs.delete(id); + } + } + } + + private isJobStuck(job: AnalysisJob): boolean { + return job.status === 'running' && Date.now() - job.createdAt > this.STUCK_JOB_TTL_MS; + } +} + +/** + * Flatten a nested INFO object (e.g. { keyspace: { db0: '...' }, memory: { used_memory: '...' } }) + * into a flat key-value map (e.g. { db0: '...', used_memory: '...' }). + * The adapter's getInfo() returns section-grouped output; buildInstanceMeta expects flat keys. + */ +function flattenInfo(info: Record): Record { + const flat: Record = {}; + for (const [, value] of Object.entries(info)) { + if (value && typeof value === 'object' && !Array.isArray(value)) { + Object.assign(flat, value); + } + } + return flat; +} + +/** + * Parse the result of MODULE LIST command. + * The result is typically an array of arrays, where each inner element + * contains name/value pairs like: [['name', 'modulename', 'ver', 1, ...], ...] + * or in newer versions: [[name, modulename, ver, 1], ...] + */ +function parseModuleList(result: unknown[]): string[] { + if (!Array.isArray(result)) return []; + const modules: string[] = []; + for (const entry of result) { + if (Array.isArray(entry)) { + // Find the 'name' key and take the next element as the value + for (let i = 0; i < entry.length - 1; i++) { + if (String(entry[i]).toLowerCase() === 'name') { + modules.push(String(entry[i + 1])); + break; + } + } + } + } + return modules; +} diff --git a/apps/api/src/migration/validation/baseline-comparator.ts b/apps/api/src/migration/validation/baseline-comparator.ts new file mode 100644 index 00000000..bf2e2787 --- /dev/null +++ b/apps/api/src/migration/validation/baseline-comparator.ts @@ -0,0 +1,124 @@ +import type { BaselineComparison, BaselineMetric, BaselineMetricStatus } from '@betterdb/shared'; +import type { StoragePort } from '../../common/interfaces/storage-port.interface'; +import type { DatabasePort } from '../../common/interfaces/database-port.interface'; + +const MIN_SNAPSHOTS = 5; + +/** + * Compare target's current metrics against source's pre-migration baseline. + * Uses memory snapshots stored by BetterDB before migration started. + */ +export async function compareBaseline( + storage: StoragePort, + sourceConnectionId: string, + targetAdapter: DatabasePort, + migrationStartedAt: number, +): Promise { + // 1. Get pre-migration source snapshots + const snapshots = await storage.getMemorySnapshots({ + connectionId: sourceConnectionId, + endTime: migrationStartedAt, + limit: 100, + }); + + // 2. Insufficient data check + if (snapshots.length < MIN_SNAPSHOTS) { + return { + available: false, + unavailableReason: + `Insufficient pre-migration data — fewer than ${MIN_SNAPSHOTS} memory snapshots were collected ` + + 'for the source instance before migration started. Connect BetterDB to the source instance ' + + 'earlier next time to establish a baseline.', + snapshotCount: snapshots.length, + baselineWindowMs: 0, + metrics: [], + }; + } + + // 3. Compute averages from source snapshots + let sumOps = 0; + let sumMem = 0; + let sumFrag = 0; + let sumCpu = 0; + + for (const snap of snapshots) { + sumOps += snap.opsPerSec; + sumMem += snap.usedMemory; + sumFrag += snap.memFragmentationRatio; + sumCpu += snap.cpuSys; + } + + const count = snapshots.length; + const avgOps = sumOps / count; + const avgMem = sumMem / count; + const avgFrag = sumFrag / count; + const avgCpu = sumCpu / count; + + // 4. Get current target metrics + const info = await targetAdapter.getInfoParsed(['stats', 'memory', 'cpu']); + + const targetOps = parseFloat(info.stats?.instantaneous_ops_per_sec ?? '0'); + const targetMem = parseFloat(info.memory?.used_memory ?? '0'); + const targetFrag = parseFloat(info.memory?.mem_fragmentation_ratio ?? '0'); + const targetCpu = parseFloat(info.cpu?.used_cpu_sys ?? '0'); + + // 5–6. Build metrics with delta and status + const metrics: BaselineMetric[] = [ + buildMetric('opsPerSec', avgOps, targetOps, (delta) => { + if (delta > 50) return 'elevated'; + if (delta < -30) return 'degraded'; + return 'normal'; + }), + buildMetric('usedMemory', avgMem, targetMem, (delta) => { + if (delta > 20) return 'elevated'; + if (delta < -20) return 'degraded'; + return 'normal'; + }), + buildMetric('memFragmentationRatio', avgFrag, targetFrag, (_delta, targetValue) => { + if (targetValue > 1.5) return 'elevated'; + return 'normal'; + }), + buildMetric('cpuSys', avgCpu, targetCpu, (delta) => { + if (delta > 50) return 'elevated'; + return 'normal'; + }), + ]; + + // 7. Compute baseline window + const oldestTimestamp = snapshots[snapshots.length - 1].timestamp; + const baselineWindowMs = migrationStartedAt - oldestTimestamp; + + return { + available: true, + snapshotCount: count, + baselineWindowMs, + metrics, + }; +} + +function buildMetric( + name: string, + sourceBaseline: number, + targetCurrent: number, + evaluateStatus: (percentDelta: number, targetValue: number) => BaselineMetricStatus, +): BaselineMetric { + if (sourceBaseline === 0) { + return { + name, + sourceBaseline, + targetCurrent, + percentDelta: null, + status: 'unavailable', + }; + } + + const percentDelta = ((targetCurrent - sourceBaseline) / sourceBaseline) * 100; + + return { + name, + sourceBaseline: Math.round(sourceBaseline * 100) / 100, + targetCurrent: Math.round(targetCurrent * 100) / 100, + percentDelta: Math.round(percentDelta * 100) / 100, + status: evaluateStatus(percentDelta, targetCurrent), + }; +} diff --git a/apps/api/src/migration/validation/key-count-comparator.ts b/apps/api/src/migration/validation/key-count-comparator.ts new file mode 100644 index 00000000..23bd4fea --- /dev/null +++ b/apps/api/src/migration/validation/key-count-comparator.ts @@ -0,0 +1,59 @@ +import type Valkey from 'iovalkey'; +import type { KeyCountComparison, MigrationAnalysisResult } from '@betterdb/shared'; + +/** + * Compare key counts between source and target using DBSIZE. + * Optionally enrich with per-type breakdown from Phase 1 analysis. + */ +export async function compareKeyCounts( + sourceClient: Valkey, + targetClient: Valkey, + analysisResult?: Partial, +): Promise { + const [sourceKeys, targetKeys] = await Promise.all([ + sourceClient.dbsize(), + targetClient.dbsize(), + ]); + + const discrepancy = targetKeys - sourceKeys; + const discrepancyPercent = sourceKeys === 0 + ? (targetKeys > 0 ? 100 : 0) + : Math.abs(discrepancy / sourceKeys) * 100; + + const result: KeyCountComparison = { + sourceKeys, + targetKeys, + discrepancy, + discrepancyPercent: Math.round(discrepancyPercent * 100) / 100, + }; + + // Flag when source is empty but target has stale keys + if (sourceKeys === 0 && targetKeys > 0) { + result.warning = + 'Source has 0 keys but target has data. Target may contain stale keys from a previous migration or other writes.'; + } + + // Risk #4: DBSIZE counts all databases, SCAN only covers db0 by default. + // If source is standalone but target is cluster, key count may be misleading. + if (analysisResult?.isCluster === false && analysisResult?.targetIsCluster === true) { + result.warning = + 'Source uses multiple databases; target is cluster-mode (db0 only). Key count may not be directly comparable.'; + } + + // Build per-type breakdown if Phase 1 analysis data is available + if (analysisResult?.dataTypeBreakdown && sourceKeys > 0) { + const ratio = targetKeys / sourceKeys; + const breakdown = analysisResult.dataTypeBreakdown; + const types: Array = ['string', 'hash', 'list', 'set', 'zset', 'stream', 'other']; + + result.typeBreakdown = types + .filter(t => breakdown[t].count > 0) + .map(t => ({ + type: t, + sourceEstimate: breakdown[t].count, + targetEstimate: Math.round(breakdown[t].count * ratio), + })); + } + + return result; +} diff --git a/apps/api/src/migration/validation/sample-validator.ts b/apps/api/src/migration/validation/sample-validator.ts new file mode 100644 index 00000000..1feb241e --- /dev/null +++ b/apps/api/src/migration/validation/sample-validator.ts @@ -0,0 +1,382 @@ +import type Valkey from 'iovalkey'; +import type { SampleValidationResult, SampleKeyResult, SampleKeyStatus } from '@betterdb/shared'; + +const TYPE_BATCH_SIZE = 100; +const MAX_ISSUES = 50; +const LARGE_KEY_THRESHOLD = 100; + +/** + * Spot-check a random sample of keys: type match + value comparison on target. + * Never throws — per-key errors are captured as 'missing' with detail. + */ +export async function validateSample( + sourceClient: Valkey, + targetClient: Valkey, + sampleSize: number = 500, +): Promise { + // 1. Collect sample keys from source via SCAN with random starting cursor + const keys = await collectSampleKeys(sourceClient, sampleSize); + + if (keys.length === 0) { + return { sampledKeys: 0, matched: 0, missing: 0, typeMismatches: 0, valueMismatches: 0, issues: [] }; + } + + // 2. Batch TYPE lookup on source + const sourceTypes = await batchType(sourceClient, keys); + + // 3. Validate each key against target + let matched = 0; + let missing = 0; + let typeMismatches = 0; + let valueMismatches = 0; + const issues: SampleKeyResult[] = []; + + for (let i = 0; i < keys.length; i++) { + const key = keys[i]; + const sourceType = sourceTypes[i]; + + if (sourceType === 'none') { + // Key expired between SCAN and TYPE — skip, don't count toward any outcome + continue; + } + + try { + const result = await validateKey(sourceClient, targetClient, key, sourceType); + + switch (result.status) { + case 'match': + matched++; + break; + case 'missing': + missing++; + if (issues.length < MAX_ISSUES) issues.push(result); + break; + case 'type_mismatch': + typeMismatches++; + if (issues.length < MAX_ISSUES) issues.push(result); + break; + case 'value_mismatch': + valueMismatches++; + if (issues.length < MAX_ISSUES) issues.push(result); + break; + } + } catch { + // Risk mitigation: never throw — count as missing on error + missing++; + if (issues.length < MAX_ISSUES) { + issues.push({ key, type: sourceType, status: 'missing', detail: 'error checking key' }); + } + } + } + + return { + sampledKeys: matched + missing + typeMismatches + valueMismatches, + matched, + missing, + typeMismatches, + valueMismatches, + issues, + }; +} + +// ── Helpers ── + +async function collectSampleKeys(client: Valkey, sampleSize: number): Promise { + const keys: string[] = []; + const seen = new Set(); + // Skip a random number of initial SCAN iterations to avoid always sampling the same keys + const skipIterations = Math.floor(Math.random() * 10); + let cursor = '0'; + let skipped = 0; + + do { + const [nextCursor, batch] = await client.scan(cursor, 'COUNT', 100); + cursor = nextCursor; + + if (skipped < skipIterations) { + skipped++; + if (cursor === '0') break; // keyspace smaller than skip window + continue; + } + + for (const key of batch) { + if (!seen.has(key)) { + seen.add(key); + keys.push(key); + if (keys.length >= sampleSize) return keys; + } + } + } while (cursor !== '0'); + + // If skipping caused us to collect fewer keys than needed, do a full pass + if (keys.length < sampleSize && skipped > 0) { + cursor = '0'; + do { + const [nextCursor, batch] = await client.scan(cursor, 'COUNT', 100); + cursor = nextCursor; + for (const key of batch) { + if (!seen.has(key)) { + seen.add(key); + keys.push(key); + if (keys.length >= sampleSize) return keys; + } + } + } while (cursor !== '0'); + } + + return keys; +} + +async function batchType(client: Valkey, keys: string[]): Promise { + const results: string[] = []; + + for (let i = 0; i < keys.length; i += TYPE_BATCH_SIZE) { + const batch = keys.slice(i, i + TYPE_BATCH_SIZE); + const pipeline = client.pipeline(); + for (const key of batch) { + pipeline.type(key); + } + const pipelineResults = await pipeline.exec(); + if (!pipelineResults) { + for (let j = 0; j < batch.length; j++) results.push('none'); + continue; + } + for (const [err, val] of pipelineResults) { + results.push(err ? 'none' : String(val)); + } + } + + return results; +} + +async function validateKey( + source: Valkey, + target: Valkey, + key: string, + sourceType: string, +): Promise { + // Check target type + const targetType = await target.type(key); + + if (targetType === 'none') { + return { key, type: sourceType, status: 'missing' }; + } + + if (targetType !== sourceType) { + return { + key, + type: sourceType, + status: 'type_mismatch', + detail: `source: ${sourceType}, target: ${targetType}`, + }; + } + + // Types match — compare values based on type + const mismatch = await compareValues(source, target, key, sourceType); + if (mismatch) { + return { key, type: sourceType, status: 'value_mismatch', detail: mismatch }; + } + + return { key, type: sourceType, status: 'match' }; +} + +async function compareValues( + source: Valkey, + target: Valkey, + key: string, + type: string, +): Promise { + switch (type) { + case 'string': + return compareString(source, target, key); + case 'hash': + return compareHash(source, target, key); + case 'list': + return compareList(source, target, key); + case 'set': + return compareSet(source, target, key); + case 'zset': + return compareZset(source, target, key); + case 'stream': + return compareStream(source, target, key); + default: + // Unknown type — types match, skip value comparison + return null; + } +} + +async function compareString(source: Valkey, target: Valkey, key: string): Promise { + const [sourceVal, targetVal] = await Promise.all([ + source.getBuffer(key), + target.getBuffer(key), + ]); + if (sourceVal === null && targetVal === null) return null; + if (sourceVal === null || targetVal === null) return 'value is null on one side'; + if (!sourceVal.equals(targetVal)) return 'string value differs'; + return null; +} + +async function compareHash(source: Valkey, target: Valkey, key: string): Promise { + const [sourceLen, targetLen] = await Promise.all([ + source.hlen(key), + target.hlen(key), + ]); + + if (sourceLen > LARGE_KEY_THRESHOLD || targetLen > LARGE_KEY_THRESHOLD) { + // Risk #3: Large key — count-only comparison + if (Math.abs(sourceLen - targetLen) / Math.max(sourceLen, 1) > 0.05) { + return `field count differs (source: ${sourceLen}, target: ${targetLen}). Large key — value comparison skipped (compared element count only).`; + } + return null; + } + + // Use HSCAN to preserve binary field names as raw Buffers + // (hgetallBuffer returns Record which coerces field names to UTF-8 strings) + const sourceEntries = await scanAllHashFields(source, key); + const targetEntries = await scanAllHashFields(target, key); + + if (sourceEntries.length !== targetEntries.length) { + return `field count differs (source: ${sourceEntries.length}, target: ${targetEntries.length})`; + } + + // Sort by field name bytes for deterministic comparison + sourceEntries.sort((a, b) => a.field.compare(b.field)); + targetEntries.sort((a, b) => a.field.compare(b.field)); + + // Compare all sorted fields (fully binary-safe) + for (let i = 0; i < sourceEntries.length; i++) { + if (!sourceEntries[i].field.equals(targetEntries[i].field)) { + return `field names differ at index ${i}`; + } + if (!sourceEntries[i].value.equals(targetEntries[i].value)) { + return `field "${sourceEntries[i].field.toString()}" value differs`; + } + } + + return null; +} + +async function compareList(source: Valkey, target: Valkey, key: string): Promise { + const [sourceLen, targetLen] = await Promise.all([ + source.llen(key), + target.llen(key), + ]); + + if (sourceLen > LARGE_KEY_THRESHOLD || targetLen > LARGE_KEY_THRESHOLD) { + if (sourceLen !== targetLen) { + return `list length differs (source: ${sourceLen}, target: ${targetLen}). Large key — value comparison skipped (compared element count only).`; + } + return null; + } + + const [sourceItems, targetItems] = await Promise.all([ + source.lrangeBuffer(key, 0, -1), + target.lrangeBuffer(key, 0, -1), + ]); + + if (sourceItems.length !== targetItems.length) { + return `list length differs (source: ${sourceItems.length}, target: ${targetItems.length})`; + } + + for (let i = 0; i < sourceItems.length; i++) { + if (!sourceItems[i].equals(targetItems[i])) { + return `list element differs at index ${i}`; + } + } + + return null; +} + +async function compareSet(source: Valkey, target: Valkey, key: string): Promise { + const [sourceCard, targetCard] = await Promise.all([ + source.scard(key), + target.scard(key), + ]); + + if (sourceCard > LARGE_KEY_THRESHOLD || targetCard > LARGE_KEY_THRESHOLD) { + if (sourceCard !== targetCard) { + return `set cardinality differs (source: ${sourceCard}, target: ${targetCard}). Large key — value comparison skipped (compared element count only).`; + } + return null; + } + + const [sourceMembers, targetMembers] = await Promise.all([ + source.smembersBuffer(key), + target.smembersBuffer(key), + ]); + + if (sourceMembers.length !== targetMembers.length) { + return `set cardinality differs (source: ${sourceMembers.length}, target: ${targetMembers.length})`; + } + + // Sort by raw bytes for deterministic comparison + sourceMembers.sort((a, b) => a.compare(b)); + targetMembers.sort((a, b) => a.compare(b)); + + for (let i = 0; i < sourceMembers.length; i++) { + if (!sourceMembers[i].equals(targetMembers[i])) { + return 'set members differ'; + } + } + + return null; +} + +async function compareZset(source: Valkey, target: Valkey, key: string): Promise { + const [sourceCard, targetCard] = await Promise.all([ + source.zcard(key), + target.zcard(key), + ]); + + if (sourceCard > LARGE_KEY_THRESHOLD || targetCard > LARGE_KEY_THRESHOLD) { + if (sourceCard !== targetCard) { + return `zset cardinality differs (source: ${sourceCard}, target: ${targetCard}). Large key — value comparison skipped (compared element count only).`; + } + return null; + } + + const [sourceData, targetData] = await Promise.all([ + source.callBuffer('ZRANGE', key, '0', '-1', 'WITHSCORES') as Promise, + target.callBuffer('ZRANGE', key, '0', '-1', 'WITHSCORES') as Promise, + ]); + + if (!sourceData && !targetData) return null; + if (!sourceData || !targetData) return 'zset data missing on one side'; + if (sourceData.length !== targetData.length) { + return `zset element count differs (source: ${sourceData.length / 2}, target: ${targetData.length / 2})`; + } + + for (let i = 0; i < sourceData.length; i++) { + if (!sourceData[i].equals(targetData[i])) { + return 'zset member or score differs'; + } + } + + return null; +} + +async function scanAllHashFields(client: Valkey, key: string): Promise> { + const entries: Array<{ field: Buffer; value: Buffer }> = []; + let cursor = '0'; + do { + const [next, fields] = await client.hscanBuffer(key, cursor, 'COUNT', 100); + cursor = String(next); + for (let i = 0; i < fields.length; i += 2) { + entries.push({ field: fields[i], value: fields[i + 1] }); + } + } while (cursor !== '0'); + return entries; +} + +async function compareStream(source: Valkey, target: Valkey, key: string): Promise { + const [sourceLen, targetLen] = await Promise.all([ + source.xlen(key), + target.xlen(key), + ]); + + if (sourceLen !== targetLen) { + return `stream length differs (source: ${sourceLen}, target: ${targetLen})`; + } + + return null; +} diff --git a/apps/api/src/migration/validation/validation-job.ts b/apps/api/src/migration/validation/validation-job.ts new file mode 100644 index 00000000..b283aa77 --- /dev/null +++ b/apps/api/src/migration/validation/validation-job.ts @@ -0,0 +1,12 @@ +import type { ValidationJobStatus, MigrationValidationResult } from '@betterdb/shared'; + +export interface ValidationJob { + id: string; + status: ValidationJobStatus; + progress: number; + createdAt: number; + completedAt?: number; + error?: string; + result: Partial; + cancelled: boolean; +} diff --git a/apps/api/test/api-migration.e2e-spec.ts b/apps/api/test/api-migration.e2e-spec.ts new file mode 100644 index 00000000..1234474a --- /dev/null +++ b/apps/api/test/api-migration.e2e-spec.ts @@ -0,0 +1,254 @@ +import { NestFastifyApplication } from '@nestjs/platform-fastify'; +import request from 'supertest'; +import Valkey from 'iovalkey'; +import { createTestApp } from './test-utils'; + +/** + * E2E tests for the /migration API endpoints. + * Requires Valkey available on DB_PORT (default 6390 from docker-compose.test.yml). + */ +describe('Migration API (e2e)', () => { + let app: NestFastifyApplication; + let sourceConnectionId: string; + let targetConnectionId: string; + let createdConnectionIds: string[] = []; + + const dbPort = Number(process.env.DB_PORT) || 6390; + const dbPassword = process.env.DB_PASSWORD || 'devpassword'; + + beforeAll(async () => { + app = await createTestApp(); + + const seedClient = new Valkey({ host: 'localhost', port: dbPort, password: dbPassword, lazyConnect: true }); + try { + await seedClient.connect(); + await seedClient.set('migration:test:string', 'hello'); + await seedClient.hset('migration:test:hash', 'f1', 'v1', 'f2', 'v2'); + await seedClient.rpush('migration:test:list', 'a', 'b', 'c'); + await seedClient.sadd('migration:test:set', 'm1', 'm2'); + await seedClient.zadd('migration:test:zset', 1, 'z1', 2, 'z2'); + } finally { + await seedClient.quit(); + } + + // Create two connections both pointing to the test Valkey instance + const res1 = await request(app.getHttpServer()) + .post('/connections') + .send({ name: 'Migration Source', host: 'localhost', port: dbPort, password: dbPassword }); + if (res1.status === 200 || res1.status === 201) { + sourceConnectionId = res1.body.id; + createdConnectionIds.push(sourceConnectionId); + } + + const res2 = await request(app.getHttpServer()) + .post('/connections') + .send({ name: 'Migration Target', host: 'localhost', port: dbPort, password: dbPassword }); + if (res2.status === 200 || res2.status === 201) { + targetConnectionId = res2.body.id; + createdConnectionIds.push(targetConnectionId); + } + }, 30_000); + + afterAll(async () => { + // Clean up test keys + const cleanupClient = new Valkey({ host: 'localhost', port: dbPort, password: dbPassword, lazyConnect: true }); + try { + await cleanupClient.connect(); + await cleanupClient.del( + 'migration:test:string', + 'migration:test:hash', + 'migration:test:list', + 'migration:test:set', + 'migration:test:zset', + ); + } catch { /* ignore */ } finally { + await cleanupClient.quit(); + } + + // Clean up created connections + for (const id of createdConnectionIds) { + await request(app.getHttpServer()) + .delete(`/connections/${id}`) + .catch(() => {}); + } + + await app.close(); + }, 30_000); + + describe('Analysis', () => { + it('should reject analysis when source and target are the same connection', async () => { + if (!sourceConnectionId) return; + + await request(app.getHttpServer()) + .post('/migration/analysis') + .send({ + sourceConnectionId, + targetConnectionId: sourceConnectionId, + }) + .expect(400); + }); + + it('should return 404 for unknown analysis ID', async () => { + await request(app.getHttpServer()) + .get('/migration/analysis/nonexistent-id') + .expect(404); + }); + + it('should complete analysis happy path', async () => { + if (!sourceConnectionId || !targetConnectionId) return; + + // Start analysis + const startRes = await request(app.getHttpServer()) + .post('/migration/analysis') + .send({ + sourceConnectionId, + targetConnectionId, + scanSampleSize: 1000, + }) + .expect((res) => { + expect([200, 201]).toContain(res.status); + }); + + expect(startRes.body).toHaveProperty('id'); + expect(startRes.body.status).toBe('pending'); + + const analysisId = startRes.body.id; + + // Poll until completed or timeout + let result: any; + for (let i = 0; i < 30; i++) { + const pollRes = await request(app.getHttpServer()) + .get(`/migration/analysis/${analysisId}`) + .expect(200); + + result = pollRes.body; + if (result.status === 'completed' || result.status === 'failed') break; + await new Promise(r => setTimeout(r, 500)); + } + + expect(result.status).toBe('completed'); + expect(result).toHaveProperty('dataTypeBreakdown'); + expect(result).toHaveProperty('ttlDistribution'); + expect(result).toHaveProperty('incompatibilities'); + expect(result.totalKeys).toBeGreaterThan(0); + }, 30_000); + + it('should cancel analysis', async () => { + if (!sourceConnectionId || !targetConnectionId) return; + + const startRes = await request(app.getHttpServer()) + .post('/migration/analysis') + .send({ + sourceConnectionId, + targetConnectionId, + scanSampleSize: 1000, + }); + + if (startRes.status !== 200 && startRes.status !== 201) return; + + const analysisId = startRes.body.id; + + // Cancel immediately + await request(app.getHttpServer()) + .delete(`/migration/analysis/${analysisId}`) + .expect(200); + + const pollRes = await request(app.getHttpServer()) + .get(`/migration/analysis/${analysisId}`) + .expect(200); + + expect(pollRes.body.status).toBe('cancelled'); + }); + }); + + describe('Validation', () => { + let analysisId: string; + + beforeAll(async () => { + if (!sourceConnectionId || !targetConnectionId) return; + + // Run an analysis first for the validation to reference + const startRes = await request(app.getHttpServer()) + .post('/migration/analysis') + .send({ + sourceConnectionId, + targetConnectionId, + scanSampleSize: 1000, + }); + + if (startRes.status === 200 || startRes.status === 201) { + analysisId = startRes.body.id; + // Wait for it to complete + for (let i = 0; i < 30; i++) { + const pollRes = await request(app.getHttpServer()) + .get(`/migration/analysis/${analysisId}`); + if (pollRes.body.status === 'completed' || pollRes.body.status === 'failed') break; + await new Promise(r => setTimeout(r, 500)); + } + } + }, 30_000); + + it('should complete validation happy path', async () => { + if (!sourceConnectionId || !targetConnectionId) return; + + const startRes = await request(app.getHttpServer()) + .post('/migration/validation') + .send({ + sourceConnectionId, + targetConnectionId, + analysisId, + }); + + // Validation endpoint may require license (Pro tier) + if (startRes.status === 403) return; // Skip if license guard blocks it + + expect([200, 201]).toContain(startRes.status); + expect(startRes.body).toHaveProperty('id'); + + const validationId = startRes.body.id; + + // Poll until completed + let result: any; + for (let i = 0; i < 30; i++) { + const pollRes = await request(app.getHttpServer()) + .get(`/migration/validation/${validationId}`); + + if (pollRes.status === 403) return; // Skip if license guard blocks + result = pollRes.body; + if (result.status === 'completed' || result.status === 'failed') break; + await new Promise(r => setTimeout(r, 500)); + } + + if (result.status === 'completed') { + expect(result).toHaveProperty('keyCount'); + expect(result).toHaveProperty('sampleValidation'); + expect(result.keyCount.sourceKeys).toBeGreaterThan(0); + expect(result.sampleValidation.matched).toBeGreaterThanOrEqual(0); + } + }, 30_000); + + it('should cancel validation', async () => { + if (!sourceConnectionId || !targetConnectionId) return; + + const startRes = await request(app.getHttpServer()) + .post('/migration/validation') + .send({ + sourceConnectionId, + targetConnectionId, + }); + + if (startRes.status === 403) return; // Skip if license guard blocks + + if (startRes.status !== 200 && startRes.status !== 201) return; + + const validationId = startRes.body.id; + + const deleteRes = await request(app.getHttpServer()) + .delete(`/migration/validation/${validationId}`); + + if (deleteRes.status === 403) return; + + expect(deleteRes.status).toBe(200); + }); + }); +}); diff --git a/apps/api/test/migration-topology.e2e-spec.ts b/apps/api/test/migration-topology.e2e-spec.ts new file mode 100644 index 00000000..46125c1a --- /dev/null +++ b/apps/api/test/migration-topology.e2e-spec.ts @@ -0,0 +1,370 @@ +import { NestFastifyApplication } from '@nestjs/platform-fastify'; +import request from 'supertest'; +import Valkey, { Cluster } from 'iovalkey'; +import { execSync } from 'child_process'; +import { join } from 'path'; +import { createTestApp } from './test-utils'; + +/** + * Migration Topology E2E — verifies command-mode migration across all four + * topology combinations: + * + * standalone → standalone + * standalone → cluster + * cluster → standalone + * cluster → cluster + * + * Requires Docker. Skipped unless RUN_TOPOLOGY_TESTS=true is set. + * Run via: pnpm test:migration-topology + */ + +const RUN = process.env.RUN_TOPOLOGY_TESTS === 'true'; + +const PROJECT_ROOT = join(__dirname, '..', '..', '..'); +const COMPOSE_FILE = join(PROJECT_ROOT, 'docker-compose.migration-e2e.yml'); +const COMPOSE_PROJECT = 'migration-e2e'; + +const SRC_STANDALONE_PORT = 6990; +const TGT_STANDALONE_PORT = 6991; +const SRC_CLUSTER_PORT = 7301; // seed node +const TGT_CLUSTER_PORT = 7401; // seed node + +// ── Docker helpers ────────────────────────────────────────────────── + +function compose(cmd: string): string { + return execSync( + `docker compose -p ${COMPOSE_PROJECT} -f "${COMPOSE_FILE}" ${cmd}`, + { encoding: 'utf-8', timeout: 120_000, stdio: ['pipe', 'pipe', 'pipe'] }, + ); +} + +// ── Connection helpers ────────────────────────────────────────────── + +function sleep(ms: number): Promise { + return new Promise(r => setTimeout(r, ms)); +} + +async function waitForStandalone(port: number, timeoutMs = 30_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + const c = new Valkey({ host: '127.0.0.1', port, lazyConnect: true, connectTimeout: 2_000 }); + await c.connect(); + await c.ping(); + await c.quit(); + return; + } catch { /* retry */ } + await sleep(500); + } + throw new Error(`Standalone on port ${port} not ready after ${timeoutMs}ms`); +} + +async function waitForCluster(port: number, timeoutMs = 60_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + const c = new Valkey({ host: '127.0.0.1', port, lazyConnect: true, connectTimeout: 2_000 }); + await c.connect(); + const info = (await c.call('CLUSTER', 'INFO')) as string; + await c.quit(); + if (info.includes('cluster_state:ok')) return; + } catch { /* retry */ } + await sleep(1_000); + } + throw new Error(`Cluster on port ${port} not ready after ${timeoutMs}ms`); +} + +// ── Client factories ──────────────────────────────────────────────── + +async function openClient(port: number, isCluster: boolean): Promise { + if (isCluster) { + const cluster = new Cluster( + [{ host: '127.0.0.1', port }], + { lazyConnect: true }, + ); + await cluster.connect(); + return cluster as unknown as Valkey; + } + const client = new Valkey({ host: '127.0.0.1', port, lazyConnect: true }); + await client.connect(); + return client; +} + +// ── Key seed / verify helpers ─────────────────────────────────────── + +async function seedKeys(client: Valkey, prefix: string): Promise { + await client.set(`${prefix}:str1`, 'value1'); + await client.set(`${prefix}:str2`, 'value2'); + await client.set(`${prefix}:str3`, 'value3'); + await client.hset(`${prefix}:hash1`, 'f1', 'v1', 'f2', 'v2'); + await client.hset(`${prefix}:hash2`, 'field', 'data'); + await client.rpush(`${prefix}:list1`, 'a', 'b', 'c'); + await client.sadd(`${prefix}:set1`, 'm1', 'm2', 'm3'); + await client.zadd(`${prefix}:zset1`, 1, 'z1', 2, 'z2'); + await client.set(`${prefix}:str4`, 'value4'); + await client.set(`${prefix}:str5`, 'value5'); +} + +async function verifyKeys(client: Valkey, prefix: string): Promise { + // 5 strings + expect(await client.get(`${prefix}:str1`)).toBe('value1'); + expect(await client.get(`${prefix}:str2`)).toBe('value2'); + expect(await client.get(`${prefix}:str3`)).toBe('value3'); + expect(await client.get(`${prefix}:str4`)).toBe('value4'); + expect(await client.get(`${prefix}:str5`)).toBe('value5'); + + // 2 hashes + expect(await client.hgetall(`${prefix}:hash1`)).toEqual({ f1: 'v1', f2: 'v2' }); + expect(await client.hgetall(`${prefix}:hash2`)).toEqual({ field: 'data' }); + + // list + expect(await client.lrange(`${prefix}:list1`, 0, -1)).toEqual(['a', 'b', 'c']); + + // set (order is non-deterministic) + const members = await client.smembers(`${prefix}:set1`); + expect(members.sort()).toEqual(['m1', 'm2', 'm3']); + + // sorted set (ordered by score) + const zset = await client.zrange(`${prefix}:zset1`, '0', '-1'); + expect(zset).toEqual(['z1', 'z2']); +} + +// ── Analysis runner ───────────────────────────────────────────────── + +async function runAnalysis( + app: NestFastifyApplication, + sourceId: string, + targetId: string, +): Promise { + const startRes = await request(app.getHttpServer()) + .post('/migration/analysis') + .send({ sourceConnectionId: sourceId, targetConnectionId: targetId, scanSampleSize: 1000 }); + + expect([200, 201]).toContain(startRes.status); + + const analysisId = startRes.body.id; + + let result: any; + for (let i = 0; i < 60; i++) { + const poll = await request(app.getHttpServer()).get(`/migration/analysis/${analysisId}`); + expect(poll.status).toBe(200); + result = poll.body; + if (result.status === 'completed' || result.status === 'failed') break; + await sleep(500); + } + return result; +} + +// ── Migration runner ──────────────────────────────────────────────── + +async function runMigration( + app: NestFastifyApplication, + sourceId: string, + targetId: string, +): Promise<{ status: string; keysTransferred?: number; error?: string } | 'skipped'> { + const startRes = await request(app.getHttpServer()) + .post('/migration/execution') + .send({ sourceConnectionId: sourceId, targetConnectionId: targetId, mode: 'command' }); + + if (startRes.status === 402 || startRes.status === 403) return 'skipped'; + expect([200, 201]).toContain(startRes.status); + + const execId = startRes.body.id; + + let result: any; + for (let i = 0; i < 120; i++) { + const poll = await request(app.getHttpServer()).get(`/migration/execution/${execId}`); + if (poll.status === 402 || poll.status === 403) return 'skipped'; + result = poll.body; + if (result.status === 'completed' || result.status === 'failed') break; + await sleep(500); + } + return result; +} + +// ── Tests ─────────────────────────────────────────────────────────── + +(RUN ? describe : describe.skip)('Migration Topology E2E', () => { + let app: NestFastifyApplication; + const connIds: Record = {}; + + beforeAll(async () => { + // 1. Start topology containers (clean slate) + try { compose('down --remove-orphans --volumes'); } catch { /* ok */ } + compose('up -d'); + + // 2. Wait for standalone instances + await Promise.all([ + waitForStandalone(SRC_STANDALONE_PORT), + waitForStandalone(TGT_STANDALONE_PORT), + ]); + + // 3. Wait for both clusters to form + await Promise.all([ + waitForCluster(SRC_CLUSTER_PORT), + waitForCluster(TGT_CLUSTER_PORT), + ]); + + // 4. Flush all instances to ensure clean state (no leftover data from prior runs) + for (const { port, isCluster } of [ + { port: SRC_STANDALONE_PORT, isCluster: false }, + { port: TGT_STANDALONE_PORT, isCluster: false }, + { port: SRC_CLUSTER_PORT, isCluster: true }, + { port: TGT_CLUSTER_PORT, isCluster: true }, + ]) { + const c = await openClient(port, isCluster); + await c.flushall(); + await c.quit(); + } + + // 5. Seed source standalone (10 keys, prefix "mig:sa") + const sa = await openClient(SRC_STANDALONE_PORT, false); + await seedKeys(sa, 'mig:sa'); + await sa.quit(); + + // 6. Seed source cluster (10 keys, prefix "mig:cl") + const cl = await openClient(SRC_CLUSTER_PORT, true); + await seedKeys(cl, 'mig:cl'); + await cl.quit(); + + // 7. Boot NestJS app with Pro license so execution endpoints are unlocked + process.env.BETTERDB_LICENSE_KEY = 'test-topology-key'; + jest.spyOn(global, 'fetch').mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ valid: true, tier: 'pro', expiresAt: null }), + } as Response); + + app = await createTestApp(); + + // 8. Register four connections via the API + const defs = [ + { key: 'srcSA', name: 'Topo Source Standalone', port: SRC_STANDALONE_PORT }, + { key: 'tgtSA', name: 'Topo Target Standalone', port: TGT_STANDALONE_PORT }, + { key: 'srcCL', name: 'Topo Source Cluster', port: SRC_CLUSTER_PORT }, + { key: 'tgtCL', name: 'Topo Target Cluster', port: TGT_CLUSTER_PORT }, + ]; + for (const d of defs) { + const res = await request(app.getHttpServer()) + .post('/connections') + .send({ name: d.name, host: '127.0.0.1', port: d.port }); + if (res.status === 200 || res.status === 201) { + connIds[d.key] = res.body.id; + } + } + }, 120_000); + + afterAll(async () => { + // Clean up connections + for (const id of Object.values(connIds)) { + try { await request(app.getHttpServer()).delete(`/connections/${id}`); } catch { /* ok */ } + } + if (app) await app.close(); + + // Restore license env / mocks + delete process.env.BETTERDB_LICENSE_KEY; + jest.restoreAllMocks(); + + // Tear down Docker topology + try { compose('down --remove-orphans --volumes'); } catch { /* ok */ } + }, 60_000); + + // ── Shared scenario runner ── + + async function scenario( + sourceKey: string, + targetKey: string, + sourcePrefix: string, + targetPort: number, + targetIsCluster: boolean, + ): Promise { + const srcId = connIds[sourceKey]; + const tgtId = connIds[targetKey]; + if (!srcId || !tgtId) { + throw new Error(`Connection not registered: ${sourceKey} / ${targetKey}`); + } + + // Flush target before migration + const flushClient = await openClient(targetPort, targetIsCluster); + await flushClient.flushall(); + await flushClient.quit(); + + // Run the migration + const result = await runMigration(app, srcId, tgtId); + expect(result).not.toBe('skipped'); + if (result === 'skipped') return; // type guard + + expect(result.status).toBe('completed'); + expect(result.keysTransferred).toBeGreaterThanOrEqual(10); + + // Verify all 10 keys arrived on the target + const target = await openClient(targetPort, targetIsCluster); + try { + await verifyKeys(target, sourcePrefix); + } finally { + await target.quit(); + } + } + + // ── 4 topology combinations ── + + it('standalone → standalone', async () => { + await scenario('srcSA', 'tgtSA', 'mig:sa', TGT_STANDALONE_PORT, false); + }, 60_000); + + it('standalone → cluster', async () => { + await scenario('srcSA', 'tgtCL', 'mig:sa', TGT_CLUSTER_PORT, true); + }, 60_000); + + it('cluster → standalone', async () => { + await scenario('srcCL', 'tgtSA', 'mig:cl', TGT_STANDALONE_PORT, false); + }, 60_000); + + it('cluster → cluster', async () => { + await scenario('srcCL', 'tgtCL', 'mig:cl', TGT_CLUSTER_PORT, true); + }, 60_000); + + // ── Compatibility analysis ── + + it('analysis: cluster → standalone should report a blocking incompatibility', async () => { + const srcId = connIds['srcCL']; + const tgtId = connIds['tgtSA']; + if (!srcId || !tgtId) throw new Error('Connections not registered'); + + const result = await runAnalysis(app, srcId, tgtId); + + expect(result.status).toBe('completed'); + expect(result.incompatibilities).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + severity: 'blocking', + category: 'cluster_topology', + }), + ]), + ); + expect(result.blockingCount).toBeGreaterThanOrEqual(1); + }, 60_000); + + it('analysis: standalone → cluster should report a warning incompatibility', async () => { + const srcId = connIds['srcSA']; + const tgtId = connIds['tgtCL']; + if (!srcId || !tgtId) throw new Error('Connections not registered'); + + const result = await runAnalysis(app, srcId, tgtId); + + expect(result.status).toBe('completed'); + expect(result.incompatibilities).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + severity: 'warning', + category: 'cluster_topology', + }), + ]), + ); + expect(result.warningCount).toBeGreaterThanOrEqual(1); + // Should NOT be blocking — migration is still possible + const clusterBlocking = (result.incompatibilities ?? []).filter( + (i: any) => i.category === 'cluster_topology' && i.severity === 'blocking', + ); + expect(clusterBlocking).toHaveLength(0); + }, 60_000); +}); diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index fc4d220f..ee294352 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -27,6 +27,7 @@ import { KeyAnalytics } from './pages/KeyAnalytics'; import { ClusterDashboard } from './pages/ClusterDashboard'; import { Settings } from './pages/Settings'; import { Webhooks } from './pages/Webhooks'; +import { MigrationPage } from './pages/MigrationPage'; import { VectorSearch } from './pages/VectorSearch'; import { MetricForecasting } from './pages/MetricForecasting'; import { Members } from './pages/Members'; @@ -78,6 +79,7 @@ function AppContent() { + {upgradePromptState.error && ( Webhooks + + Migration + {!cloudUser && ( @@ -218,6 +223,7 @@ function AppLayout({ cloudUser }: { cloudUser: CloudUser | null }) { } /> } /> } /> + } /> {cloudUser && ( } /> )} @@ -225,6 +231,12 @@ function AppLayout({ cloudUser }: { cloudUser: CloudUser | null }) { + ); } diff --git a/apps/web/src/components/migration/AnalysisForm.tsx b/apps/web/src/components/migration/AnalysisForm.tsx new file mode 100644 index 00000000..bca2b749 --- /dev/null +++ b/apps/web/src/components/migration/AnalysisForm.tsx @@ -0,0 +1,175 @@ +import { useState } from 'react'; +import { useConnection } from '../../hooks/useConnection'; +import type { Connection } from '../../hooks/useConnection'; +import { fetchApi } from '../../api/client'; +import type { StartAnalysisResponse } from '@betterdb/shared'; + +interface Props { + onStart: (analysisId: string) => void; +} + +function connectionLabel(c: Connection): string { + const base = `${c.name} (${c.host}:${c.port})`; + if (c.capabilities?.dbType && c.capabilities?.version) { + const type = c.capabilities.dbType === 'valkey' ? 'Valkey' : 'Redis'; + return `${base} — ${type} ${c.capabilities.version}`; + } + return base; +} + +export function AnalysisForm({ onStart }: Props) { + const { connections, currentConnection } = useConnection(); + const [sourceConnectionId, setSourceConnectionId] = useState(currentConnection?.id ?? ''); + const [targetConnectionId, setTargetConnectionId] = useState(''); + const [scanSampleSize, setScanSampleSize] = useState(10000); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const sameConnection = + sourceConnectionId !== '' && + targetConnectionId !== '' && + sourceConnectionId === targetConnectionId; + + const targetConnection = connections.find(c => c.id === targetConnectionId); + const targetIsOffline = targetConnectionId !== '' && targetConnection && !targetConnection.isConnected; + + // The API returns connectionType on each connection but the Connection + // interface in useConnection doesn't surface it. Cast to access at runtime. + const isAgentConnection = (id: string): boolean => { + if (!id) return false; + const conn = connections.find(c => c.id === id) as + | (typeof connections[number] & { connectionType?: 'direct' | 'agent' }) + | undefined; + return conn?.connectionType === 'agent'; + }; + const hasAgentConnection = + isAgentConnection(sourceConnectionId) || isAgentConnection(targetConnectionId); + + const isCloudMode = import.meta.env.VITE_CLOUD_MODE === 'true'; + + const handleSubmit = async (e: React.FormEvent) => { + e.preventDefault(); + if (!sourceConnectionId || !targetConnectionId || sameConnection) return; + setLoading(true); + setError(null); + try { + const res = await fetchApi('/migration/analysis', { + method: 'POST', + body: JSON.stringify({ sourceConnectionId, targetConnectionId, scanSampleSize }), + }); + onStart(res.id); + } catch (err) { + const raw = err instanceof Error ? err.message : 'Failed to start analysis'; + if (/writeable|enableOfflineQueue|offline/i.test(raw)) { + setError(`Could not connect to ${targetConnection?.name ?? 'target'} — the instance appears to be offline. Check the connection before running analysis.`); + } else { + setError(raw); + } + } finally { + setLoading(false); + } + }; + + return ( +
+
+ + +
+ +
+ + + {sameConnection && ( +

+ Source and target must be different connections +

+ )} + {targetIsOffline && !sameConnection && ( +

+ This instance appears to be offline and may not accept connections. +

+ )} +
+ + {hasAgentConnection && ( +

+ One or more selected instances is connected via agent. Contact us at{' '} + support@betterdb.com to + plan your migration — we'll help you do it safely. +

+ )} + + {isCloudMode && ( +

+ Migration execution is not available in BetterDB Cloud. Contact us at{' '} + support@betterdb.com to + plan your migration. +

+ )} + +
+ + +

+ Higher sample = more accurate estimates, slower analysis. +

+
+ + {error && ( +
+ {error} +
+ )} + + +
+ ); +} diff --git a/apps/web/src/components/migration/AnalysisProgressBar.tsx b/apps/web/src/components/migration/AnalysisProgressBar.tsx new file mode 100644 index 00000000..9f6de9aa --- /dev/null +++ b/apps/web/src/components/migration/AnalysisProgressBar.tsx @@ -0,0 +1,87 @@ +import { useState, useEffect, useRef } from 'react'; +import { fetchApi } from '../../api/client'; +import type { MigrationAnalysisResult } from '@betterdb/shared'; + +interface Props { + analysisId: string; + onComplete: (result: MigrationAnalysisResult) => void; + onError: (msg: string) => void; + onCancel: () => void; +} + +function getStepLabel(progress: number): string { + if (progress <= 12) return 'Connecting and reading server info'; + if (progress <= 14) return 'Detecting cluster topology'; + if (progress <= 50) return 'Scanning keyspace'; + if (progress <= 65) return 'Sampling memory usage'; + if (progress <= 75) return 'Analyzing TTL distribution'; + if (progress <= 85) return 'Checking Hash Field Expiry'; + if (progress <= 95) return 'Analyzing command patterns'; + return 'Computing migration verdict'; +} + +export function AnalysisProgressBar({ analysisId, onComplete, onError, onCancel }: Props) { + const [job, setJob] = useState(null); + const onCompleteRef = useRef(onComplete); + const onErrorRef = useRef(onError); + const onCancelRef = useRef(onCancel); + onCompleteRef.current = onComplete; + onErrorRef.current = onError; + onCancelRef.current = onCancel; + + useEffect(() => { + const interval = setInterval(async () => { + try { + const result = await fetchApi(`/migration/analysis/${analysisId}`); + setJob(result); + if (result.status === 'completed') { + clearInterval(interval); + onCompleteRef.current(result); + } else if (result.status === 'failed') { + clearInterval(interval); + onErrorRef.current(result.error ?? 'Analysis failed'); + } else if (result.status === 'cancelled') { + clearInterval(interval); + onCancelRef.current(); + } + } catch { + clearInterval(interval); + onErrorRef.current('Analysis job not found or server error'); + } + }, 2000); + return () => clearInterval(interval); + }, [analysisId]); + + const handleCancel = async () => { + try { + await fetchApi(`/migration/analysis/${analysisId}`, { method: 'DELETE' }); + } catch { + /* ignore */ + } + onCancel(); + }; + + const currentProgress = job?.progress ?? 0; + + return ( +
+
+ Analyzing... + {currentProgress}% +
+
+
+
+

{getStepLabel(currentProgress)}

+ +
+ ); +} diff --git a/apps/web/src/components/migration/ExecutionLogViewer.tsx b/apps/web/src/components/migration/ExecutionLogViewer.tsx new file mode 100644 index 00000000..a142a833 --- /dev/null +++ b/apps/web/src/components/migration/ExecutionLogViewer.tsx @@ -0,0 +1,43 @@ +import { useEffect, useRef } from 'react'; + +interface Props { + logs: string[]; +} + +export function ExecutionLogViewer({ logs }: Props) { + const containerRef = useRef(null); + const isAtBottomRef = useRef(true); + + const handleScroll = () => { + const el = containerRef.current; + if (!el) return; + isAtBottomRef.current = el.scrollTop + el.clientHeight >= el.scrollHeight - 20; + }; + + useEffect(() => { + const el = containerRef.current; + if (el && isAtBottomRef.current) { + el.scrollTop = el.scrollHeight; + } + }, [logs.length]); + + if (logs.length === 0) { + return ( +
+ Waiting for output... +
+ ); + } + + return ( +
+ {logs.slice(-500).map((line, i) => ( +
{line}
+ ))} +
+ ); +} diff --git a/apps/web/src/components/migration/ExecutionPanel.tsx b/apps/web/src/components/migration/ExecutionPanel.tsx new file mode 100644 index 00000000..4368f265 --- /dev/null +++ b/apps/web/src/components/migration/ExecutionPanel.tsx @@ -0,0 +1,157 @@ +import { useState, useEffect, useRef } from 'react'; +import { fetchApi } from '../../api/client'; +import type { MigrationExecutionResult } from '@betterdb/shared'; +import { ExecutionLogViewer } from './ExecutionLogViewer'; + +interface Props { + executionId: string; + onStopped: () => void; +} + +function formatElapsed(startedAt: number, completedAt?: number): string { + const end = completedAt ?? Date.now(); + const seconds = Math.floor((end - startedAt) / 1000); + const m = Math.floor(seconds / 60); + const s = seconds % 60; + return m > 0 ? `${m}m ${s}s` : `${s}s`; +} + +function StatusBadge({ status }: { status: string }) { + switch (status) { + case 'running': + return Running; + case 'completed': + return Completed; + case 'failed': + return Failed; + case 'cancelled': + return Cancelled; + default: + return {status}; + } +} + +export function ExecutionPanel({ executionId, onStopped }: Props) { + const [execution, setExecution] = useState(null); + const onStoppedRef = useRef(onStopped); + onStoppedRef.current = onStopped; + + useEffect(() => { + let stopped = false; + let errorCount = 0; + const poll = async () => { + try { + const result = await fetchApi(`/migration/execution/${executionId}`); + if (stopped) return; + errorCount = 0; + setExecution(result); + if (result.status === 'completed' || result.status === 'failed' || result.status === 'cancelled') { + onStoppedRef.current(); + return; + } + } catch { + if (stopped) return; + errorCount++; + } + if (!stopped) { + const delay = errorCount > 0 ? Math.min(2000 * 2 ** errorCount, 30000) : 2000; + timer = setTimeout(poll, delay); + } + }; + let timer: ReturnType | undefined; + poll(); + return () => { + stopped = true; + if (timer) clearTimeout(timer); + }; + }, [executionId]); + + const handleStop = async () => { + try { + await fetchApi(`/migration/execution/${executionId}`, { method: 'DELETE' }); + } catch { + /* ignore */ + } + // Optimistic transition + onStoppedRef.current(); + }; + + if (!execution) { + return ( +
+

Starting migration...

+
+ ); + } + + return ( +
+ {/* Status bar */} +
+
+ + + {execution.mode === 'command' ? 'Command' : 'RedisShake'} + + + {(execution.keysTransferred ?? 0).toLocaleString()} + {execution.totalKeys ? ` / ${execution.totalKeys.toLocaleString()}` : ''} keys transferred + + {(execution.keysSkipped ?? 0) > 0 && ( + + {execution.keysSkipped!.toLocaleString()} skipped + + )} + + {formatElapsed(execution.startedAt, execution.completedAt)} + +
+ {execution.status === 'running' && ( + + )} +
+ + {/* Progress bar — shown while running */} + {execution.status === 'running' && execution.progress != null && ( +
+
+ Migration progress + {Math.min(100, execution.progress)}% +
+
+
+
+
+ )} + + {/* Status banners */} + {execution.status === 'failed' && ( +
+ {execution.error ?? 'Migration failed'} +
+ )} + {execution.status === 'completed' && ( +
+ Migration complete — {(execution.keysTransferred ?? 0).toLocaleString()} keys transferred + {(execution.keysSkipped ?? 0) > 0 && `, ${execution.keysSkipped!.toLocaleString()} skipped`}. +
+ )} + {execution.status === 'cancelled' && ( +
+ Migration stopped by user. +
+ )} + + {/* Log viewer */} + +
+ ); +} diff --git a/apps/web/src/components/migration/ExportBar.tsx b/apps/web/src/components/migration/ExportBar.tsx new file mode 100644 index 00000000..917924f0 --- /dev/null +++ b/apps/web/src/components/migration/ExportBar.tsx @@ -0,0 +1,37 @@ +import type { MigrationAnalysisResult } from '@betterdb/shared'; + +interface Props { + job: MigrationAnalysisResult; + phase?: string; +} + +export function ExportBar({ job, phase }: Props) { + if (phase === 'executing') return null; + + const handleExportJson = () => { + const blob = new Blob([JSON.stringify(job, null, 2)], { type: 'application/json' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `betterdb-migration-${job.sourceConnectionName ?? 'unknown'}-${Date.now()}.json`; + a.click(); + URL.revokeObjectURL(url); + }; + + return ( + <> + + + + ); +} diff --git a/apps/web/src/components/migration/MigrationReport.tsx b/apps/web/src/components/migration/MigrationReport.tsx new file mode 100644 index 00000000..d770e212 --- /dev/null +++ b/apps/web/src/components/migration/MigrationReport.tsx @@ -0,0 +1,26 @@ +import type { MigrationAnalysisResult } from '@betterdb/shared'; +import { SummarySection } from './sections/SummarySection'; +import { VerdictSection } from './sections/VerdictSection'; +import { DataTypeSection } from './sections/DataTypeSection'; +import { TtlSection } from './sections/TtlSection'; +import { CommandSection } from './sections/CommandSection'; +import { HfeSection } from './sections/HfeSection'; + +interface Props { + job: MigrationAnalysisResult; +} + +export function MigrationReport({ job }: Props) { + return ( +
+ + +
+ + +
+ + +
+ ); +} diff --git a/apps/web/src/components/migration/ValidationPanel.tsx b/apps/web/src/components/migration/ValidationPanel.tsx new file mode 100644 index 00000000..5ea65677 --- /dev/null +++ b/apps/web/src/components/migration/ValidationPanel.tsx @@ -0,0 +1,128 @@ +import { useState, useEffect, useRef } from 'react'; +import { fetchApi } from '../../api/client'; +import type { MigrationValidationResult } from '@betterdb/shared'; +import { KeyCountSection } from './sections/KeyCountSection'; +import { SampleValidationSection } from './sections/SampleValidationSection'; +import { BaselineSection } from './sections/BaselineSection'; + +interface Props { + validationId: string; + onComplete?: () => void; +} + +function getStepLabel(progress: number): string { + if (progress <= 5) return 'Connecting...'; + if (progress <= 20) return 'Comparing key counts...'; + if (progress <= 70) return 'Validating sample keys...'; + if (progress <= 80) return 'Comparing baseline metrics...'; + if (progress <= 99) return 'Finalising...'; + return ''; +} + +export function ValidationPanel({ validationId, onComplete }: Props) { + const [validation, setValidation] = useState(null); + const onCompleteRef = useRef(onComplete); + onCompleteRef.current = onComplete; + + useEffect(() => { + let stopped = false; + let errorCount = 0; + let timer: ReturnType | undefined; + + const poll = async () => { + try { + const result = await fetchApi(`/migration/validation/${validationId}`); + if (stopped) return; + errorCount = 0; + setValidation(result); + if (result.status === 'completed' || result.status === 'failed') { + onCompleteRef.current?.(); + return; + } + } catch { + if (stopped) return; + errorCount++; + } + if (!stopped) { + const delay = errorCount > 0 ? Math.min(2000 * 2 ** errorCount, 30000) : 2000; + timer = setTimeout(poll, delay); + } + }; + + poll(); + return () => { + stopped = true; + if (timer) clearTimeout(timer); + }; + }, [validationId]); + + if (!validation) { + return ( +
+

Starting validation...

+
+ ); + } + + const inProgress = validation.status === 'pending' || validation.status === 'running'; + const stepLabel = getStepLabel(validation.progress); + + return ( +
+ {/* Progress bar — shown while in progress */} + {inProgress && ( +
+
+ Validating... + {validation.progress}% +
+
+
+
+ {stepLabel &&

{stepLabel}

} +
+ )} + + {/* Result banner */} + {validation.status === 'completed' && validation.passed && ( +
+ Validation passed — no issues found. +
+ )} + {validation.status === 'completed' && !validation.passed && ( +
+ Validation complete — {validation.issueCount ?? 0} issue{(validation.issueCount ?? 0) !== 1 ? 's' : ''} found. +
+ )} + {validation.status === 'failed' && ( +
+ {validation.error ?? 'Validation failed'} +
+ )} + + {/* Sections — rendered as they become available */} + {validation.status !== 'pending' && ( +
+ {validation.keyCount !== undefined && ( + + )} + {validation.sampleValidation !== undefined && ( + <> +
+ + + )} + {validation.baseline !== undefined && ( + <> +
+ + + )} +
+ )} +
+ ); +} diff --git a/apps/web/src/components/migration/sections/BaselineSection.tsx b/apps/web/src/components/migration/sections/BaselineSection.tsx new file mode 100644 index 00000000..3bf1e718 --- /dev/null +++ b/apps/web/src/components/migration/sections/BaselineSection.tsx @@ -0,0 +1,122 @@ +import type { BaselineComparison, BaselineMetricStatus } from '@betterdb/shared'; +import { InfoTip } from './InfoTip'; + +interface Props { + baseline?: BaselineComparison; +} + +function formatDuration(ms: number): string { + const hours = Math.floor(ms / (1000 * 60 * 60)); + const minutes = Math.floor((ms % (1000 * 60 * 60)) / (1000 * 60)); + if (hours > 0) return `${hours}h ${minutes}m`; + return `${minutes}m`; +} + +function formatMetricValue(name: string, value: number | null): string { + if (value === null) return '—'; + if (name === 'usedMemory') { + if (value >= 1073741824) return `${(value / 1073741824).toFixed(2)} GB`; + if (value >= 1048576) return `${(value / 1048576).toFixed(2)} MB`; + return `${(value / 1024).toFixed(2)} KB`; + } + if (name === 'memFragmentationRatio') return value.toFixed(2); + if (name === 'opsPerSec') return value.toLocaleString(); + return value.toFixed(2); +} + +const metricTooltips: Record = { + opsPerSec: "Compares the target's current throughput to the source's pre-migration average. A drop to zero is expected if no application traffic has been directed to the target yet. An increase is expected if writes are still landing on the target.", + usedMemory: "Compares the target's current memory to the source's pre-migration average. Higher usage is expected if the target already held data before migration or received additional writes.", + memFragmentationRatio: 'Ratio of OS-allocated memory to actual data. Values near 1.0 are ideal. High values indicate fragmentation from deletes or expires.', + cpuSys: 'Cumulative CPU seconds since server start — not a rate. A large delta is expected when the target has been running longer or processed heavy migration writes.', +}; + +function MetricLabel({ name }: { name: string }) { + const labels: Record = { + opsPerSec: 'Ops/sec', + usedMemory: 'Used Memory', + memFragmentationRatio: 'Mem Fragmentation', + cpuSys: 'CPU Sys', + }; + const tooltip = metricTooltips[name]; + return ( + <> + {labels[name] ?? name} + {tooltip && } + + ); +} + +function StatusBadge({ status }: { status: BaselineMetricStatus }) { + switch (status) { + case 'normal': + return normal; + case 'elevated': + return elevated; + case 'degraded': + return degraded; + case 'unavailable': + return unavailable; + } +} + +export function BaselineSection({ baseline }: Props) { + if (!baseline) { + return

Not available.

; + } + + if (!baseline.available) { + return ( +
+

Baseline Comparison

+
+ {baseline.unavailableReason} +
+
+ ); + } + + return ( +
+

Baseline Comparison

+ +
+ + + + + + + + + + + + {baseline.metrics.map((metric) => ( + + + + + + + + ))} + +
MetricSource BaselineTarget CurrentDeltaStatus
{formatMetricValue(metric.name, metric.sourceBaseline)}{formatMetricValue(metric.name, metric.targetCurrent)} + {metric.percentDelta !== null + ? `${metric.percentDelta >= 0 ? '+' : ''}${metric.percentDelta.toFixed(1)}%` + : '—'} +
+
+ +

+ Baseline computed from {baseline.snapshotCount} snapshots over {formatDuration(baseline.baselineWindowMs)} before migration. +

+ + {/* Risk #5: Single snapshot caveat */} +

+ Target metrics reflect a single sample taken at validation time. For ongoing monitoring, view the target connection's dashboard. +

+
+ ); +} diff --git a/apps/web/src/components/migration/sections/CommandSection.tsx b/apps/web/src/components/migration/sections/CommandSection.tsx new file mode 100644 index 00000000..6b4bf937 --- /dev/null +++ b/apps/web/src/components/migration/sections/CommandSection.tsx @@ -0,0 +1,50 @@ +import type { MigrationAnalysisResult } from '@betterdb/shared'; + +interface Props { + job: MigrationAnalysisResult; +} + +const SOURCE_LABELS: Record = { + commandlog: 'COMMANDLOG (Valkey 8.1+)', + slowlog: 'SLOWLOG (fallback)', + unavailable: 'Unavailable — command history not accessible on this instance.', +}; + +export function CommandSection({ job }: Props) { + const cmd = job.commandAnalysis; + + if (!cmd || cmd.topCommands.length === 0) { + return null; + } + + return ( +
+

Command Analysis

+ + {cmd.topCommands.length > 0 && ( +
+ + + + + + + + + {cmd.topCommands.map(({ command, count }) => ( + + + + + ))} + +
CommandOccurrences
{command}{count.toLocaleString()}
+
+ )} + +

+ Command data sourced from: {SOURCE_LABELS[cmd.sourceUsed] ?? cmd.sourceUsed} +

+
+ ); +} diff --git a/apps/web/src/components/migration/sections/DataTypeSection.tsx b/apps/web/src/components/migration/sections/DataTypeSection.tsx new file mode 100644 index 00000000..370a7f65 --- /dev/null +++ b/apps/web/src/components/migration/sections/DataTypeSection.tsx @@ -0,0 +1,80 @@ +import type { MigrationAnalysisResult } from '@betterdb/shared'; +import { PieChart, Pie, Cell, Tooltip, ResponsiveContainer, Legend } from 'recharts'; + +const COLORS = ['#3b82f6', '#10b981', '#f59e0b', '#ef4444', '#8b5cf6', '#06b6d4', '#6b7280']; +const TYPE_NAMES = ['string', 'hash', 'list', 'set', 'zset', 'stream', 'other'] as const; + +function formatBytes(bytes: number): string { + if (bytes === 0) return '0 B'; + const units = ['B', 'KB', 'MB', 'GB', 'TB']; + const i = Math.floor(Math.log(bytes) / Math.log(1024)); + return `${(bytes / Math.pow(1024, i)).toFixed(1)} ${units[i]}`; +} + +interface Props { + job: MigrationAnalysisResult; +} + +export function DataTypeSection({ job }: Props) { + const breakdown = job.dataTypeBreakdown; + + if (!breakdown) { + return ( +
+

Data Types

+

Not available for this analysis.

+
+ ); + } + + const chartData = TYPE_NAMES + .map(name => ({ name, count: breakdown[name]?.count ?? 0 })) + .filter(d => d.count > 0); + + return ( +
+

Data Types

+
+
+ + + + {chartData.map((_, i) => )} + + + + + +
+
+ + + + + + + + + + + {TYPE_NAMES.map(name => { + const dt = breakdown[name]; + if (!dt || dt.count === 0) return null; + return ( + + + + + + + ); + })} + +
TypeKey CountSampled Memory + Est. Total Memory +
{name}{dt.count.toLocaleString()}{formatBytes(dt.sampledMemoryBytes)}~{formatBytes(dt.estimatedTotalMemoryBytes)}
+
+
+
+ ); +} diff --git a/apps/web/src/components/migration/sections/HfeSection.tsx b/apps/web/src/components/migration/sections/HfeSection.tsx new file mode 100644 index 00000000..030b217c --- /dev/null +++ b/apps/web/src/components/migration/sections/HfeSection.tsx @@ -0,0 +1,54 @@ +import type { MigrationAnalysisResult } from '@betterdb/shared'; +import { AlertTriangle, CheckCircle } from 'lucide-react'; + +interface Props { + job: MigrationAnalysisResult; +} + +export function HfeSection({ job }: Props) { + if (job.hfeSupported === undefined && job.hfeDetected === undefined) { + return ( +
+

Hash Field Expiry

+

Not available for this analysis.

+
+ ); + } + + return ( +
+

Hash Field Expiry

+ + {job.hfeSupported === false ? ( +

+ HFE check not available — source is Redis (Hash Field Expiry is a Valkey-only feature). +

+ ) : job.hfeDetected ? ( +
+ +
+

+ Hash Field Expiry keys detected (~{(job.hfeKeyCount ?? 0).toLocaleString()} estimated). +

+

+ Hash fields with per-field TTLs will lose their expiry metadata during migration + unless the target instance supports HFE (Valkey 8.1+). Verify your target version + before proceeding. +

+
+
+ ) : ( +
+ + Not detected in sample. +
+ )} + + {(job.hfeOversizedHashesSkipped ?? 0) > 0 && ( +

+ Note: {job.hfeOversizedHashesSkipped} hash key(s) with >10,000 fields were skipped during HFE sampling. +

+ )} +
+ ); +} diff --git a/apps/web/src/components/migration/sections/InfoTip.tsx b/apps/web/src/components/migration/sections/InfoTip.tsx new file mode 100644 index 00000000..af93c3e0 --- /dev/null +++ b/apps/web/src/components/migration/sections/InfoTip.tsx @@ -0,0 +1,11 @@ +import { Info } from 'lucide-react'; + +export function InfoTip({ text }: { text: string }) { + return ( + + ); +} diff --git a/apps/web/src/components/migration/sections/KeyCountSection.tsx b/apps/web/src/components/migration/sections/KeyCountSection.tsx new file mode 100644 index 00000000..fd236ff7 --- /dev/null +++ b/apps/web/src/components/migration/sections/KeyCountSection.tsx @@ -0,0 +1,79 @@ +import type { KeyCountComparison } from '@betterdb/shared'; +import { InfoTip } from './InfoTip'; + +interface Props { + keyCount?: KeyCountComparison; +} + +export function KeyCountSection({ keyCount }: Props) { + if (!keyCount) { + return

Not available.

; + } + + const { sourceKeys, targetKeys, discrepancy, discrepancyPercent, warning, typeBreakdown } = keyCount; + + const discrepancyColor = + discrepancyPercent <= 1 + ? 'text-green-700' + : discrepancyPercent <= 5 + ? 'text-amber-700' + : 'text-red-700'; + + const sign = discrepancy >= 0 ? '+' : ''; + + return ( +
+

Key Count Comparison

+ +
+
+ Source +

{sourceKeys.toLocaleString()}

+
+
+ Target +

{targetKeys.toLocaleString()}

+
+
+ + Discrepancy + + +

+ {sign}{discrepancy.toLocaleString()} ({discrepancyPercent}%) +

+
+
+ + {warning && ( +

{warning}

+ )} + + {typeBreakdown && typeBreakdown.length > 0 && ( +
+ + + + + + + + + + {typeBreakdown.map((row) => ( + + + + + + ))} + +
TypeSource (est.)Target (est.)
{row.type}{row.sourceEstimate.toLocaleString()}{row.targetEstimate.toLocaleString()}
+

+ Type counts are estimated from Phase 1 analysis. +

+
+ )} +
+ ); +} diff --git a/apps/web/src/components/migration/sections/SampleValidationSection.tsx b/apps/web/src/components/migration/sections/SampleValidationSection.tsx new file mode 100644 index 00000000..6113a6c8 --- /dev/null +++ b/apps/web/src/components/migration/sections/SampleValidationSection.tsx @@ -0,0 +1,86 @@ +import type { SampleValidationResult, SampleKeyStatus } from '@betterdb/shared'; +import { InfoTip } from './InfoTip'; + +interface Props { + sample?: SampleValidationResult; +} + +function StatusBadge({ status }: { status: SampleKeyStatus }) { + switch (status) { + case 'missing': + return missing; + case 'type_mismatch': + return type mismatch; + case 'value_mismatch': + return value mismatch; + default: + return match; + } +} + +export function SampleValidationSection({ sample }: Props) { + if (!sample) { + return

Not available.

; + } + + const allMatch = sample.missing === 0 && sample.typeMismatches === 0 && sample.valueMismatches === 0; + + return ( +
+

Sample Validation

+ +
+ + {sample.matched} + + /{sample.sampledKeys} matched + + + + {sample.missing > 0 && ( + {sample.missing} missing + )} + {sample.typeMismatches > 0 && ( + {sample.typeMismatches} type mismatch{sample.typeMismatches !== 1 ? 'es' : ''} + )} + {sample.valueMismatches > 0 && ( + {sample.valueMismatches} value mismatch{sample.valueMismatches !== 1 ? 'es' : ''} + )} +
+ + {allMatch && ( +

All sampled keys validated successfully.

+ )} + + {/* Risk #1: timing gap note */} +

+ Some mismatches may reflect keys written to source after migration scanning began. +

+ + {sample.issues.length > 0 && ( +
+ + + + + + + + + + + {sample.issues.map((issue, idx) => ( + + + + + + + ))} + +
KeyTypeStatusDetail
{issue.key}{issue.type}{issue.detail ?? ''}
+
+ )} +
+ ); +} diff --git a/apps/web/src/components/migration/sections/SummarySection.tsx b/apps/web/src/components/migration/sections/SummarySection.tsx new file mode 100644 index 00000000..9507a06b --- /dev/null +++ b/apps/web/src/components/migration/sections/SummarySection.tsx @@ -0,0 +1,86 @@ +import type { MigrationAnalysisResult } from '@betterdb/shared'; +import { AlertTriangle } from 'lucide-react'; + +function formatBytes(bytes: number): string { + if (bytes === 0) return '0 B'; + const units = ['B', 'KB', 'MB', 'GB', 'TB']; + const i = Math.floor(Math.log(bytes) / Math.log(1024)); + return `${(bytes / Math.pow(1024, i)).toFixed(1)} ${units[i]}`; +} + +function DbBadge({ dbType, dbVersion, connectionName }: { + dbType?: 'valkey' | 'redis'; + dbVersion?: string; + connectionName?: string; +}) { + const label = dbType === 'valkey' ? 'Valkey' : dbType === 'redis' ? 'Redis' : 'Unknown'; + const colorClass = dbType === 'valkey' + ? 'bg-teal-100 text-teal-700' + : dbType === 'redis' + ? 'bg-red-100 text-red-700' + : 'bg-gray-100 text-gray-700'; + + return ( +
+ + {label} + + {dbVersion ?? 'Unknown'} + + {connectionName ?? 'Unknown'} + +
+ ); +} + +interface Props { + job: MigrationAnalysisResult; +} + +export function SummarySection({ job }: Props) { + return ( +
+

Summary

+ +
+ + → + +
+ +
+
+

Total Keys

+

{(job.totalKeys ?? 0).toLocaleString()}

+
+
+

Est. Memory

+

~{formatBytes(job.estimatedTotalMemoryBytes ?? 0)}

+
+
+ + {job.isCluster && ( +
+ +

+ Cluster mode detected — analysis covers {job.clusterMasterCount} master nodes. + Key count and memory are aggregated across all masters. +

+
+ )} + +

+ {(job.sampledKeys ?? 0).toLocaleString()} keys sampled out of {(job.totalKeys ?? 0).toLocaleString()} total + {job.isCluster ? ` (${(job.sampledPerNode ?? 0).toLocaleString()} per node)` : ''} +

+
+ ); +} diff --git a/apps/web/src/components/migration/sections/TtlSection.tsx b/apps/web/src/components/migration/sections/TtlSection.tsx new file mode 100644 index 00000000..a19a5b93 --- /dev/null +++ b/apps/web/src/components/migration/sections/TtlSection.tsx @@ -0,0 +1,50 @@ +import type { MigrationAnalysisResult } from '@betterdb/shared'; +import { BarChart, Bar, XAxis, YAxis, Tooltip, ResponsiveContainer, Cell } from 'recharts'; + +interface Props { + job: MigrationAnalysisResult; +} + +export function TtlSection({ job }: Props) { + const ttl = job.ttlDistribution; + + if (!ttl) { + return ( +
+

TTL Distribution

+

Not available for this analysis.

+
+ ); + } + + const data = [ + { name: 'No Expiry', value: ttl.noExpiry, color: '#6b7280' }, + { name: '< 1 hour', value: ttl.expiresWithin1h, color: '#f59e0b' }, + { name: '< 24 hours', value: ttl.expiresWithin24h, color: '#3b82f6' }, + { name: '< 7 days', value: ttl.expiresWithin7d, color: '#10b981' }, + { name: '> 7 days', value: ttl.expiresAfter7d, color: '#8b5cf6' }, + ]; + + return ( +
+

TTL Distribution

+
+ + + + + [`${Number(value).toLocaleString()} keys`, 'Count']} + /> + + {data.map((entry, i) => )} + + + +
+

+ Sampled from {ttl.sampledKeyCount.toLocaleString()} keys. +

+
+ ); +} diff --git a/apps/web/src/components/migration/sections/VerdictSection.tsx b/apps/web/src/components/migration/sections/VerdictSection.tsx new file mode 100644 index 00000000..3fab6243 --- /dev/null +++ b/apps/web/src/components/migration/sections/VerdictSection.tsx @@ -0,0 +1,96 @@ +import type { MigrationAnalysisResult, Incompatibility } from '@betterdb/shared'; +import { CheckCircle, AlertTriangle, XCircle, Info } from 'lucide-react'; + +const SEVERITY_ORDER: Record = { + blocking: 0, + warning: 1, + info: 2, +}; + +const SEVERITY_ICON_MAP: Record = { + blocking: { icon: XCircle, color: 'text-red-600' }, + warning: { icon: AlertTriangle, color: 'text-amber-600' }, + info: { icon: Info, color: 'text-blue-600' }, +}; + +interface Props { + job: MigrationAnalysisResult; +} + +export function VerdictSection({ job }: Props) { + if (job.incompatibilities === undefined) { + return ( +
+

Compatibility

+

Not available for this analysis.

+
+ ); + } + + const blockingCount = job.blockingCount ?? 0; + const warningCount = job.warningCount ?? 0; + + let bannerBg: string; + let bannerText: string; + let BannerIcon: typeof CheckCircle; + let bannerMessage: string; + + if (blockingCount > 0) { + bannerBg = 'bg-red-50 border-red-200'; + bannerText = 'text-red-800'; + BannerIcon = XCircle; + bannerMessage = `${blockingCount} blocking issue(s) — resolve before migrating.`; + } else if (warningCount > 0) { + bannerBg = 'bg-amber-50 border-amber-200'; + bannerText = 'text-amber-800'; + BannerIcon = AlertTriangle; + bannerMessage = `No blocking issues. ${warningCount} warning(s) to review.`; + } else { + bannerBg = 'bg-green-50 border-green-200'; + bannerText = 'text-green-800'; + BannerIcon = CheckCircle; + bannerMessage = 'No compatibility issues found. Migration appears safe.'; + } + + const sorted = [...job.incompatibilities].sort( + (a, b) => SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity], + ); + + return ( +
+
+

Compatibility

+
+ +

{bannerMessage}

+
+
+ + {sorted.length > 0 && ( +
+ {sorted.map((item, idx) => { + const sev = SEVERITY_ICON_MAP[item.severity]; + const SevIcon = sev.icon; + return ( +
+ +
+
+ {item.title} + + {item.category} + +
+

{item.detail}

+
+
+ ); + })} +
+ )} +
+ ); +} diff --git a/apps/web/src/components/migration/sections/__tests__/sections.test.tsx b/apps/web/src/components/migration/sections/__tests__/sections.test.tsx new file mode 100644 index 00000000..be90dc06 --- /dev/null +++ b/apps/web/src/components/migration/sections/__tests__/sections.test.tsx @@ -0,0 +1,252 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import { render, screen, cleanup } from '@testing-library/react'; +import type { + KeyCountComparison, + SampleValidationResult, + BaselineComparison, +} from '@betterdb/shared'; +import { InfoTip } from '../InfoTip'; +import { KeyCountSection } from '../KeyCountSection'; +import { SampleValidationSection } from '../SampleValidationSection'; +import { BaselineSection } from '../BaselineSection'; + +afterEach(() => { + cleanup(); +}); + +// ── InfoTip ── + +describe('InfoTip', () => { + it('should render an SVG (Info icon)', () => { + const { container } = render(); + expect(container.querySelector('svg')).not.toBeNull(); + }); + + it('should set correct data-tooltip-id', () => { + const { container } = render(); + const svg = container.querySelector('svg'); + expect(svg?.getAttribute('data-tooltip-id')).toBe('info-tip'); + }); + + it('should set correct data-tooltip-content', () => { + const { container } = render(); + const svg = container.querySelector('svg'); + expect(svg?.getAttribute('data-tooltip-content')).toBe('My tooltip text'); + }); +}); + +// ── KeyCountSection ── + +describe('KeyCountSection', () => { + it('should render "Not available." when keyCount is undefined', () => { + const { container } = render(); + expect(container.textContent).toContain('Not available.'); + }); + + it('should render source, target, and discrepancy values', () => { + const keyCount: KeyCountComparison = { + sourceKeys: 10000, + targetKeys: 10050, + discrepancy: 50, + discrepancyPercent: 0.5, + }; + render(); + + expect(screen.getByText('10,000')).toBeDefined(); + expect(screen.getByText('10,050')).toBeDefined(); + expect(screen.getByText('+50 (0.5%)')).toBeDefined(); + }); + + it('should show warning text when warning is set', () => { + const keyCount: KeyCountComparison = { + sourceKeys: 100, + targetKeys: 100, + discrepancy: 0, + discrepancyPercent: 0, + warning: 'Multi-DB source detected', + }; + render(); + + expect(screen.getByText('Multi-DB source detected')).toBeDefined(); + }); + + it('should show type breakdown table when data present', () => { + const keyCount: KeyCountComparison = { + sourceKeys: 1000, + targetKeys: 1000, + discrepancy: 0, + discrepancyPercent: 0, + typeBreakdown: [ + { type: 'string', sourceEstimate: 500, targetEstimate: 500 }, + { type: 'hash', sourceEstimate: 300, targetEstimate: 300 }, + ], + }; + render(); + + expect(screen.getByText('string')).toBeDefined(); + expect(screen.getByText('hash')).toBeDefined(); + }); + + it('should render discrepancy info tooltip', () => { + const keyCount: KeyCountComparison = { + sourceKeys: 100, + targetKeys: 110, + discrepancy: 10, + discrepancyPercent: 10, + }; + const { container } = render(); + + const infoIcons = container.querySelectorAll('svg[data-tooltip-id="info-tip"]'); + expect(infoIcons.length).toBeGreaterThanOrEqual(1); + }); +}); + +// ── SampleValidationSection ── + +describe('SampleValidationSection', () => { + it('should render "Not available." when sample is undefined', () => { + const { container } = render(); + expect(container.textContent).toContain('Not available.'); + }); + + it('should render matched count', () => { + const sample: SampleValidationResult = { + sampledKeys: 500, + matched: 495, + missing: 3, + typeMismatches: 1, + valueMismatches: 1, + issues: [], + }; + render(); + + expect(screen.getByText('495')).toBeDefined(); + expect(screen.getByText('/500 matched')).toBeDefined(); + }); + + it('should show all-match success message when no mismatches', () => { + const sample: SampleValidationResult = { + sampledKeys: 500, + matched: 500, + missing: 0, + typeMismatches: 0, + valueMismatches: 0, + issues: [], + }; + render(); + + expect(screen.getByText('All sampled keys validated successfully.')).toBeDefined(); + }); + + it('should render issues table when issues exist', () => { + const sample: SampleValidationResult = { + sampledKeys: 500, + matched: 498, + missing: 1, + typeMismatches: 1, + valueMismatches: 0, + issues: [ + { key: 'user:123', type: 'string', status: 'missing', detail: 'Key not found on target' }, + { key: 'data:456', type: 'hash', status: 'type_mismatch', detail: 'Expected hash, got string' }, + ], + }; + render(); + + expect(screen.getByText('user:123')).toBeDefined(); + expect(screen.getByText('data:456')).toBeDefined(); + expect(screen.getByText('missing')).toBeDefined(); + expect(screen.getByText('type mismatch')).toBeDefined(); + }); + + it('should render sample info tooltip', () => { + const sample: SampleValidationResult = { + sampledKeys: 500, + matched: 500, + missing: 0, + typeMismatches: 0, + valueMismatches: 0, + issues: [], + }; + const { container } = render(); + + const infoIcons = container.querySelectorAll('svg[data-tooltip-id="info-tip"]'); + expect(infoIcons.length).toBeGreaterThanOrEqual(1); + }); +}); + +// ── BaselineSection ── + +describe('BaselineSection', () => { + it('should render "Not available." when baseline is undefined', () => { + const { container } = render(); + expect(container.textContent).toContain('Not available.'); + }); + + it('should show unavailable reason when available is false', () => { + const baseline: BaselineComparison = { + available: false, + unavailableReason: 'Insufficient snapshots collected before migration.', + snapshotCount: 2, + baselineWindowMs: 0, + metrics: [], + }; + render(); + + expect(screen.getByText('Insufficient snapshots collected before migration.')).toBeDefined(); + }); + + it('should render metrics table with correct labels', () => { + const baseline: BaselineComparison = { + available: true, + snapshotCount: 10, + baselineWindowMs: 3600000, + metrics: [ + { name: 'opsPerSec', sourceBaseline: 1000, targetCurrent: 950, percentDelta: -5, status: 'normal' }, + { name: 'usedMemory', sourceBaseline: 1073741824, targetCurrent: 1073741824, percentDelta: 0, status: 'normal' }, + { name: 'memFragmentationRatio', sourceBaseline: 1.05, targetCurrent: 1.1, percentDelta: 4.8, status: 'normal' }, + { name: 'cpuSys', sourceBaseline: 120.5, targetCurrent: 125.3, percentDelta: 4.0, status: 'normal' }, + ], + }; + render(); + + expect(screen.getByText('Ops/sec')).toBeDefined(); + expect(screen.getByText('Used Memory')).toBeDefined(); + expect(screen.getByText('Mem Fragmentation')).toBeDefined(); + expect(screen.getByText('CPU Sys')).toBeDefined(); + }); + + it('should render info tooltip icons for each metric', () => { + const baseline: BaselineComparison = { + available: true, + snapshotCount: 10, + baselineWindowMs: 3600000, + metrics: [ + { name: 'opsPerSec', sourceBaseline: 1000, targetCurrent: 950, percentDelta: -5, status: 'normal' }, + { name: 'usedMemory', sourceBaseline: 1073741824, targetCurrent: 1073741824, percentDelta: 0, status: 'normal' }, + { name: 'memFragmentationRatio', sourceBaseline: 1.05, targetCurrent: 1.1, percentDelta: 4.8, status: 'normal' }, + { name: 'cpuSys', sourceBaseline: 120.5, targetCurrent: 125.3, percentDelta: 4.0, status: 'normal' }, + ], + }; + const { container } = render(); + + const infoIcons = container.querySelectorAll('svg[data-tooltip-id="info-tip"]'); + expect(infoIcons.length).toBe(4); + }); + + it('should format memory values correctly', () => { + const baseline: BaselineComparison = { + available: true, + snapshotCount: 10, + baselineWindowMs: 3600000, + metrics: [ + { name: 'usedMemory', sourceBaseline: 1073741824, targetCurrent: 1073741824, percentDelta: 0, status: 'normal' }, + ], + }; + const { container } = render(); + + // 1073741824 bytes = 1.00 GB — appears in source baseline and target current columns + const gbTexts = container.querySelectorAll('td.font-mono'); + const gbValues = Array.from(gbTexts).filter(td => td.textContent === '1.00 GB'); + expect(gbValues.length).toBe(2); + }); +}); diff --git a/apps/web/src/pages/MigrationPage.tsx b/apps/web/src/pages/MigrationPage.tsx new file mode 100644 index 00000000..1c3985bc --- /dev/null +++ b/apps/web/src/pages/MigrationPage.tsx @@ -0,0 +1,501 @@ +import { useState, useRef, useEffect } from 'react'; +import type { MigrationAnalysisResult, MigrationExecutionResult, ExecutionMode } from '@betterdb/shared'; +import { Feature } from '@betterdb/shared'; +import { fetchApi } from '../api/client'; +import { useLicense } from '../hooks/useLicense'; +import { AnalysisForm } from '../components/migration/AnalysisForm'; +import { AnalysisProgressBar } from '../components/migration/AnalysisProgressBar'; +import { MigrationReport } from '../components/migration/MigrationReport'; +import { ExportBar } from '../components/migration/ExportBar'; +import { ExecutionPanel } from '../components/migration/ExecutionPanel'; +import { ValidationPanel } from '../components/migration/ValidationPanel'; + +type Phase = 'idle' | 'analyzing' | 'analyzed' | 'executing' | 'executed' | 'validating' | 'validated'; + +// ── Helpers ── + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; +} + +function stepIndex(phase: Phase): number { + if (phase === 'idle') return 0; + if (phase === 'analyzing' || phase === 'analyzed') return 1; + return 2; +} + +// ── Small shared components ── + +function LockIcon() { + return ( + + + + ); +} + +const STEPS = ['Configure', 'Analyse', 'Migrate'] as const; + +function StepIndicator({ phase, onBack }: { phase: Phase; onBack?: () => void }) { + const current = stepIndex(phase); + return ( + + ); +} + +// ── Main page ── + +export function MigrationPage() { + const [phase, setPhase] = useState('idle'); + const [analysisId, setAnalysisId] = useState(null); + const [executionId, setExecutionId] = useState(null); + const [validationId, setValidationId] = useState(null); + const [job, setJob] = useState(null); + const [executionResult, setExecutionResult] = useState(null); + const [error, setError] = useState(null); + const { hasFeature } = useLicense(); + + const canExecute = hasFeature(Feature.MIGRATION_EXECUTION); + const blockingCount = job?.blockingCount ?? 0; + const [executionMode, setExecutionMode] = useState('redis_shake'); + + // Issue 1 + 4: confirmation dialog state + const [showConfirmDialog, setShowConfirmDialog] = useState(false); + const [migrationStarting, setMigrationStarting] = useState(false); + + // Scroll target for validation section + const validationRef = useRef(null); + + // Issue 15: history + const [history, setHistory] = useState([]); + const [expandedHistoryId, setExpandedHistoryId] = useState(null); + + // Scroll to validation section when it appears + useEffect(() => { + if (phase === 'validating') { + validationRef.current?.scrollIntoView({ behavior: 'smooth', block: 'start' }); + } + }, [phase]); + + // General cleanup: centralized reset + const resetToIdle = (saveToHistory = true) => { + if (saveToHistory && job) { + setHistory(prev => [job, ...prev].slice(0, 5)); + } + setPhase('idle'); + setJob(null); + setAnalysisId(null); + setExecutionId(null); + setValidationId(null); + setExecutionResult(null); + }; + + // Issue 1: open dialog instead of window.confirm + const handleStartMigration = () => { + if (!job?.sourceConnectionId || !job?.targetConnectionId) return; + setShowConfirmDialog(true); + }; + + // Issue 4: actual API call after user confirms + const handleConfirmMigration = async () => { + if (!job?.sourceConnectionId || !job?.targetConnectionId) return; + setMigrationStarting(true); + try { + const result = await fetchApi<{ id: string }>('/migration/execution', { + method: 'POST', + body: JSON.stringify({ + sourceConnectionId: job.sourceConnectionId, + targetConnectionId: job.targetConnectionId, + mode: executionMode, + }), + }); + setShowConfirmDialog(false); + setExecutionId(result.id); + setPhase('executing'); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + setError(message); + setShowConfirmDialog(false); + } finally { + setMigrationStarting(false); + } + }; + + const handleStartValidation = async () => { + if (!job?.sourceConnectionId || !job?.targetConnectionId) return; + + try { + const result = await fetchApi<{ id: string }>('/migration/validation', { + method: 'POST', + body: JSON.stringify({ + sourceConnectionId: job.sourceConnectionId, + targetConnectionId: job.targetConnectionId, + analysisId: analysisId ?? undefined, + migrationStartedAt: executionResult?.startedAt, + }), + }); + setValidationId(result.id); + setPhase('validating'); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + setError(message); + } + }; + + return ( +
+
+

Migration

+

+ Analyze your source instance to assess migration readiness. +

+
+ + {/* Issue 3: Step indicator */} + resetToIdle() : undefined} + /> + + {error && ( +
+ {error} + +
+ )} + + {phase === 'idle' && ( + { + setAnalysisId(id); + setPhase('analyzing'); + setError(null); + }} + /> + )} + + {phase === 'analyzing' && analysisId && ( + { + setJob(result); + setPhase('analyzed'); + }} + onError={(msg) => { + setError(msg); + setPhase('idle'); + }} + onCancel={() => { + setPhase('idle'); + }} + /> + )} + + {phase === 'analyzed' && job && ( + <> + + + {/* Mode selector + Start Migration button */} +
+ {canExecute && ( +
+ + +
+ )} + + {/* Issue 8: prominent blocking warning */} + {blockingCount > 0 && ( +
+ + + +
+

{blockingCount} blocking issue{blockingCount !== 1 ? 's' : ''} detected

+

Proceeding may cause data loss or incompatibility on the target instance.

+
+
+ )} + + {!canExecute && ( +

+ Migration execution requires a Pro license. Upgrade at betterdb.com/pricing +

+ )} + +
+ {!canExecute ? ( + + ) : ( + + )} + + +
+
+ + )} + + {phase === 'executing' && job && executionId && ( + <> + + { + try { + const result = await fetchApi(`/migration/execution/${executionId}`); + setExecutionResult(result); + } catch { /* ignore */ } + setPhase('executed'); + }} + /> + + )} + + {phase === 'executed' && job && executionId && ( + <> + + {/* already stopped */}} + /> + + {/* Run Validation + actions */} +
+ {!canExecute && ( +

+ Post-migration validation requires a Pro license. Upgrade at betterdb.com/pricing +

+ )} + +
+ {!canExecute ? ( + + ) : ( + + )} + + +
+
+ + )} + + {phase === 'validating' && job && validationId && ( + <> + + {executionId && ( + {/* already stopped */}} + /> + )} +
+ setPhase('validated')} + /> +
+ + )} + + {phase === 'validated' && job && validationId && ( + <> + + {executionId && ( + {/* already stopped */}} + /> + )} +
+ +
+ +
+ + +
+ + )} + + {/* Issue 4: Confirmation dialog */} + {showConfirmDialog && job && ( +
{ if (!migrationStarting) setShowConfirmDialog(false); }} + > +
e.stopPropagation()} + > +

Confirm Migration

+
+
+
Source
+
{job.sourceConnectionName ?? 'Unknown'}
+
+
+
Target
+
{job.targetConnectionName ?? 'Unknown'}
+
+
+
Total keys
+
{(job.totalKeys ?? 0).toLocaleString()}
+
+
+
Estimated memory
+
{formatBytes(job.estimatedTotalMemoryBytes ?? job.totalMemoryBytes ?? 0)}
+
+
+
Mode
+
{executionMode === 'command' ? 'Command-based' : 'DUMP/RESTORE (RedisShake)'}
+
+
+ + {blockingCount > 0 && ( +
+ Warning: {blockingCount} blocking issue{blockingCount !== 1 ? 's' : ''} detected. + Proceeding may cause data loss or incompatibility. +
+ )} + +
+ + +
+
+
+ )} + + {/* Issue 15: Past analyses history */} + {history.length > 0 && ( +
+

Past Analyses

+
+ {history.map(entry => ( +
+ + {expandedHistoryId === entry.id && ( +
+ +
+ )} +
+ ))} +
+
+ )} +
+ ); +} diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index a62368de..a93b73ac 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -14,4 +14,7 @@ export default defineConfig({ '@betterdb/shared/license': path.resolve(__dirname, '../../packages/shared/src/license/index.ts'), }, }, + test: { + environment: 'happy-dom', + }, }); diff --git a/docker-compose.migration-e2e.yml b/docker-compose.migration-e2e.yml new file mode 100644 index 00000000..182ae31a --- /dev/null +++ b/docker-compose.migration-e2e.yml @@ -0,0 +1,186 @@ +# Self-contained topology for migration e2e tests. +# 2 standalone Valkey instances + 2 clusters (3 masters each, no replicas). +# All services use host networking so cluster nodes advertise 127.0.0.1. +# +# Usage: +# docker compose -p migration-e2e -f docker-compose.migration-e2e.yml up -d +# docker compose -p migration-e2e -f docker-compose.migration-e2e.yml down -v + +services: + # ── Standalone instances ────────────────────────────────────────── + + source-standalone: + image: valkey/valkey:8-alpine + container_name: mig-source-standalone + network_mode: host + command: valkey-server --port 6990 --save "" --appendonly no + healthcheck: + test: ["CMD", "valkey-cli", "-p", "6990", "ping"] + interval: 2s + timeout: 3s + retries: 15 + restart: "no" + + target-standalone: + image: valkey/valkey:8-alpine + container_name: mig-target-standalone + network_mode: host + command: valkey-server --port 6991 --save "" --appendonly no + healthcheck: + test: ["CMD", "valkey-cli", "-p", "6991", "ping"] + interval: 2s + timeout: 3s + retries: 15 + restart: "no" + + # ── Source cluster (3 masters, no replicas) ─────────────────────── + + src-node-1: + image: valkey/valkey:8-alpine + container_name: mig-src-node-1 + network_mode: host + command: > + valkey-server --port 7301 + --cluster-enabled yes + --cluster-config-file /data/nodes.conf + --cluster-node-timeout 5000 + --cluster-announce-ip 127.0.0.1 + --cluster-announce-port 7301 + --cluster-announce-bus-port 17301 + --save "" --appendonly no + healthcheck: + test: ["CMD", "valkey-cli", "-p", "7301", "ping"] + interval: 2s + timeout: 3s + retries: 15 + restart: "no" + + src-node-2: + image: valkey/valkey:8-alpine + container_name: mig-src-node-2 + network_mode: host + command: > + valkey-server --port 7302 + --cluster-enabled yes + --cluster-config-file /data/nodes.conf + --cluster-node-timeout 5000 + --cluster-announce-ip 127.0.0.1 + --cluster-announce-port 7302 + --cluster-announce-bus-port 17302 + --save "" --appendonly no + healthcheck: + test: ["CMD", "valkey-cli", "-p", "7302", "ping"] + interval: 2s + timeout: 3s + retries: 15 + restart: "no" + + src-node-3: + image: valkey/valkey:8-alpine + container_name: mig-src-node-3 + network_mode: host + command: > + valkey-server --port 7303 + --cluster-enabled yes + --cluster-config-file /data/nodes.conf + --cluster-node-timeout 5000 + --cluster-announce-ip 127.0.0.1 + --cluster-announce-port 7303 + --cluster-announce-bus-port 17303 + --save "" --appendonly no + healthcheck: + test: ["CMD", "valkey-cli", "-p", "7303", "ping"] + interval: 2s + timeout: 3s + retries: 15 + restart: "no" + + src-cluster-init: + image: valkey/valkey:8-alpine + container_name: mig-src-cluster-init + network_mode: host + depends_on: + src-node-1: { condition: service_healthy } + src-node-2: { condition: service_healthy } + src-node-3: { condition: service_healthy } + command: > + sh -c "valkey-cli --cluster create + 127.0.0.1:7301 127.0.0.1:7302 127.0.0.1:7303 + --cluster-replicas 0 --cluster-yes" + restart: "no" + + # ── Target cluster (3 masters, no replicas) ─────────────────────── + + tgt-node-1: + image: valkey/valkey:8-alpine + container_name: mig-tgt-node-1 + network_mode: host + command: > + valkey-server --port 7401 + --cluster-enabled yes + --cluster-config-file /data/nodes.conf + --cluster-node-timeout 5000 + --cluster-announce-ip 127.0.0.1 + --cluster-announce-port 7401 + --cluster-announce-bus-port 17401 + --save "" --appendonly no + healthcheck: + test: ["CMD", "valkey-cli", "-p", "7401", "ping"] + interval: 2s + timeout: 3s + retries: 15 + restart: "no" + + tgt-node-2: + image: valkey/valkey:8-alpine + container_name: mig-tgt-node-2 + network_mode: host + command: > + valkey-server --port 7402 + --cluster-enabled yes + --cluster-config-file /data/nodes.conf + --cluster-node-timeout 5000 + --cluster-announce-ip 127.0.0.1 + --cluster-announce-port 7402 + --cluster-announce-bus-port 17402 + --save "" --appendonly no + healthcheck: + test: ["CMD", "valkey-cli", "-p", "7402", "ping"] + interval: 2s + timeout: 3s + retries: 15 + restart: "no" + + tgt-node-3: + image: valkey/valkey:8-alpine + container_name: mig-tgt-node-3 + network_mode: host + command: > + valkey-server --port 7403 + --cluster-enabled yes + --cluster-config-file /data/nodes.conf + --cluster-node-timeout 5000 + --cluster-announce-ip 127.0.0.1 + --cluster-announce-port 7403 + --cluster-announce-bus-port 17403 + --save "" --appendonly no + healthcheck: + test: ["CMD", "valkey-cli", "-p", "7403", "ping"] + interval: 2s + timeout: 3s + retries: 15 + restart: "no" + + tgt-cluster-init: + image: valkey/valkey:8-alpine + container_name: mig-tgt-cluster-init + network_mode: host + depends_on: + tgt-node-1: { condition: service_healthy } + tgt-node-2: { condition: service_healthy } + tgt-node-3: { condition: service_healthy } + command: > + sh -c "valkey-cli --cluster create + 127.0.0.1:7401 127.0.0.1:7402 127.0.0.1:7403 + --cluster-replicas 0 --cluster-yes" + restart: "no" diff --git a/docs/migration.md b/docs/migration.md new file mode 100644 index 00000000..6db49458 --- /dev/null +++ b/docs/migration.md @@ -0,0 +1,238 @@ +# Migration + +BetterDB can migrate data between Valkey and Redis instances using a three-phase +workflow: **analysis**, **execution**, and **validation**. Each phase is +independent — you can run analysis without committing to a migration, and +validation is optional after execution completes. + +## Phases + +### 1. Analysis (Community tier) + +Scans the source instance and compares it against the target to produce a +compatibility report. No data is written. + +What it checks: + +- **Key sampling** — SCAN + TYPE on a configurable sample (1,000–50,000 keys). + In cluster mode each master node is sampled independently. +- **Memory estimation** — `MEMORY USAGE` per sampled key, extrapolated to the + full keyspace. +- **TTL distribution** — Groups keys into buckets (no expiry, <1 h, <24 h, + <7 d, >7 d). +- **Hash Field Expiry (HFE)** — Detects per-field TTLs on Valkey 8.1+ via + `HEXPIRETIME`. Skipped on Redis or older Valkey. +- **Compatibility** — Produces a list of incompatibilities with severity levels + (`blocking`, `warning`, `info`). See [Compatibility checks](#compatibility-checks). +- **Command distribution** — Top commands by frequency from `COMMANDLOG` (Valkey + 8+) or `SLOWLOG`. + +### 2. Execution (Pro tier) + +Transfers keys from source to target. Two modes are available: + +| Mode | Mechanism | Best for | +|------|-----------|----------| +| **redis_shake** (default) | External Go binary ([redis-shake](https://github.com/tair-opensource/RedisShake)) | Large datasets, production workloads | +| **command** | In-process Node.js via iovalkey | Simpler deployments, smaller datasets, easier debugging | + +#### Command mode + +Connects directly to the source and target using the iovalkey library. For each +key it reads the value with a type-specific command, writes it to the target, +and preserves the TTL. + +Supported data types: + +| Type | Read | Write | TTL | +|------|------|-------|-----| +| string | `GET` (binary) | `SET PX` | Atomic — single `SET` with `PX` flag | +| hash | `HSCAN` (binary fields) | `HSET` to temp key, then `RENAME` | Lua `RENAME` + `PEXPIRE` | +| list | `LRANGE` in 1,000-element chunks | `RPUSH` to temp key, then `RENAME` | Lua `RENAME` + `PEXPIRE` | +| set | `SMEMBERS` or `SSCAN` (>10 K) | `SADD` to temp key, then `RENAME` | Lua `RENAME` + `PEXPIRE` | +| sorted set | `ZRANGE` or `ZSCAN` (>10 K) | `ZADD` to temp key, then `RENAME` | Lua `RENAME` + `PEXPIRE` | +| stream | `XRANGE` in 1,000-entry chunks | `XADD` to temp key, then `RENAME` | Lua `RENAME` + `PEXPIRE` | + +Compound types (everything except string) are written to a temporary key first, +then atomically renamed to the final key. This avoids partial writes if the +process crashes mid-transfer. If `EVAL` is blocked by ACL on the target, the +rename and TTL are applied as separate commands with a small race window. + +#### RedisShake mode + +Spawns the redis-shake binary as a child process. BetterDB generates the TOML +configuration, manages the process lifecycle, and streams progress from its +stdout. RedisShake auto-discovers cluster topology on both sides, so no special +handling is needed for cluster targets. + +The binary is found in this order: +1. `$REDIS_SHAKE_PATH` environment variable +2. `/usr/local/bin/redis-shake` (Docker image) +3. `~/.betterdb/bin/redis-shake` (npx install) + +### 3. Validation (Pro tier) + +Spot-checks the target after migration to verify data integrity. + +Steps: + +1. **Key count** — `DBSIZE` on both sides. Computes discrepancy percentage. +2. **Sample validation** — SCAN ~500 random keys and compare type + value. + Large keys (>100 elements) are compared by element count only to avoid + timeouts. +3. **Baseline comparison** (optional) — If a migration start time is provided + and BetterDB has >= 5 pre-migration memory snapshots, compares opsPerSec, + usedMemory, fragmentation ratio, and CPU usage against the pre-migration + baseline. + +A validation **passes** when the issue count is 0 and the key count discrepancy +is below 1%. + +## Topology support + +| Source | Target | Status | Notes | +|--------|--------|--------|-------| +| Standalone | Standalone | Supported | Direct key transfer | +| Standalone | Cluster | Supported | Keys are resharded across target slots. Analysis reports a warning. | +| Cluster | Cluster | Supported | Per-master scanning, slot-aware writes | +| Cluster | Standalone | **Blocked** | Analysis reports a blocking incompatibility. The data is spread across slots and cannot be safely collapsed into a single node. | + +## Compatibility checks + +Analysis detects the following incompatibilities: + +| Category | Severity | Condition | +|----------|----------|-----------| +| `cluster_topology` | blocking | Cluster source, standalone target | +| `cluster_topology` | warning | Standalone source, cluster target (keys will be resharded) | +| `type_direction` | blocking | Valkey source, Redis target (Valkey-specific features may be lost) | +| `hfe` | blocking | Hash Field Expiry detected on source, target does not support it | +| `modules` | blocking | Source uses a module not present on target (one entry per module) | +| `multi_db` | blocking | Source uses multiple databases and target is a cluster (clusters only support db0) | +| `multi_db` | warning | Source uses multiple databases, target is standalone but may not be configured for it | +| `maxmemory_policy` | warning | Eviction policy differs between source and target | +| `acl` | warning | Source has custom ACL users that do not exist on target | +| `persistence` | info | Persistence configuration differs | + +Blocking incompatibilities are advisory — the execution endpoint does not +currently enforce them. A future release will reject execution when blocking +incompatibilities exist. + +## Limitations + +### Keys containing `{` (hash tags in cluster mode) + +In cluster mode, Valkey determines which slot a key belongs to by hashing the +substring between the first `{` and the next `}`. This is called a **hash tag**. + +During command-mode migration, compound types are written to a temporary key and +then renamed to the final key. `RENAME` requires both keys to hash to the same +slot. To satisfy this, the temp key reuses the original key's hash tag: + +``` +Original key: user:{12345}:profile +Temp key: __betterdb_mig_a1b2c3d4:{12345} + +Original key: plain-key-no-braces +Temp key: __betterdb_mig_a1b2c3d4:{plain-key-no-braces} +``` + +**Edge case**: If a key contains `{` but no matching `}`, or the content between +the braces is empty (e.g., `foo{}bar`), Valkey hashes the entire key. BetterDB +handles this correctly — the `tempKey()` function only extracts a hash tag when +`{...}` contains at least one character. Otherwise it wraps the full key name as +the tag. + +**Impact on key names with literal braces**: If your keys use `{` as part of +their name rather than as a hash tag (e.g., `json:{data}`), the migration still +works correctly. The content between the first `{…}` pair is reused as the tag, +which guarantees the temp key lands in the same slot. The key's value and name +are preserved exactly. + +### Binary data + +All migrations use `*Buffer` variants of commands (`getBuffer`, `lrangeBuffer`, +`hscanBuffer`, etc.) so binary values are never coerced to UTF-8. Hash field +names are read via `HSCAN` (not `HGETALL`) specifically because `hgetallBuffer` +coerces field names to strings. + +**RedisShake mode**: The TOML configuration builder rejects values (passwords, +connection strings) containing control characters (`\x00–\x08`, `\x0b`, `\x0c`, +`\x0e–\x1f`, `\x7f`) to prevent TOML injection. + +### TTL precision and race conditions + +- **String keys**: TTL is applied atomically via `SET key value PX pttl` — no + window where the key exists without its TTL. +- **Compound types**: A Lua script performs `RENAME` + `PEXPIRE` in a single + `EVAL` call. If the target blocks `EVAL` via ACL, BetterDB falls back to + separate `RENAME` and `PEXPIRE` commands. In this fallback path there is a + brief window where the key exists with no expiry. +- **Expired between read and TTL fetch**: If `PTTL` returns `-2` (key expired), + the target copy is deleted. + +### Hash Field Expiry (HFE) + +Valkey 8.1+ supports per-field TTLs within a hash. Analysis detects HFE usage +via `HEXPIRETIME`, but **command-mode migration does not transfer per-field +expirations**. Only the overall key-level TTL is preserved. If the target does +not support HFE, analysis flags this as a blocking incompatibility. + +### Large keys + +Keys with more than 10,000 elements use cursor-based reads (`HSCAN`, `SSCAN`, +`ZSCAN`) instead of bulk commands to avoid blocking the server. Lists and +streams are always read in 1,000-element chunks regardless of size. + +During validation, keys with more than 100 elements are compared by **element +count only** — full value comparison is skipped to avoid timeouts. + +### Multi-database + +Command-mode migration and cluster mode only operate on database 0. If the +source uses multiple databases (`db0`, `db1`, etc.) and the target is a cluster, +analysis flags this as a blocking incompatibility. For standalone targets, +analysis issues a warning. + +### ACL users and modules + +ACL rules and loaded modules are **not migrated** — they are analyzed and +reported. If the source has custom ACL users missing from the target, analysis +issues a warning. If the source uses modules not loaded on the target, analysis +flags a blocking incompatibility. + +### DBSIZE accuracy in cluster mode + +`DBSIZE` on a cluster client is sent to a single random node, returning a +partial count. This means the key count comparison in validation may be +inaccurate for cluster targets. This is a known limitation. + +### Concurrent writes on the source + +The migration reads a point-in-time snapshot per key but does not freeze the +source. If keys are modified on the source during migration: + +- **Lists** may have different lengths. A post-migration length check warns if + the list grew or shrank. +- **Keys created after SCAN started** are missed entirely. +- **Keys deleted after SCAN** are skipped with no error (the read returns nil). + +For a consistent migration, quiesce writes to the source before starting. + +## Batching and concurrency + +| Parameter | Value | +|-----------|-------| +| SCAN batch size | 500 keys per iteration | +| TYPE lookup batch | 500 keys per pipeline | +| Migration batch | 50 keys in parallel | +| List/stream chunk | 1,000 elements per read | +| Max concurrent analysis jobs | 20 | +| Max concurrent execution jobs | 10 | +| Stuck job timeout | 2 hours (auto-cancelled) | + +## Credential handling + +RedisShake log output is sanitized before being served to the frontend. Patterns +like `password = "secret"` and `redis://user:pass@host` are redacted. Source +passwords are never included in API responses. diff --git a/package.json b/package.json index 657213c8..58141850 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "test:integration": "pnpm --filter api test:integration", "test:integration:redis": "TEST_DB_PORT=6382 pnpm --filter api test:integration:redis", "test:integration:valkey": "TEST_DB_PORT=6380 pnpm --filter api test:integration:valkey", + "test:migration-topology": "pnpm --filter api test:migration-topology", "lint": "turbo lint", "clean": "turbo clean && rm -rf node_modules", "cli:build": "pnpm --filter @betterdb/monitor build", diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 508c0dc3..11b0385f 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -13,4 +13,5 @@ export * from './utils/key-patterns'; export * from './license/index'; export * from './webhooks/index'; export * from './types/vector-index-snapshots'; +export * from './types/migration'; export * from './types/metric-forecasting.types'; diff --git a/packages/shared/src/license/types.ts b/packages/shared/src/license/types.ts index 7f165c6a..20e11905 100644 --- a/packages/shared/src/license/types.ts +++ b/packages/shared/src/license/types.ts @@ -42,6 +42,7 @@ export enum Feature { AI_CLOUD = 'aiCloud', WEBHOOK_COMPLIANCE_EVENTS = 'webhookComplianceEvents', WEBHOOK_DLQ = 'webhookDlq', + MIGRATION_EXECUTION = 'migrationExecution', } export const TIER_FEATURES: Record = { @@ -56,6 +57,7 @@ export const TIER_FEATURES: Record = { Feature.WEBHOOK_CUSTOM_HEADERS, Feature.WEBHOOK_DELIVERY_PAYLOAD, Feature.WEBHOOK_CONFIGURABLE_RETRY, + Feature.MIGRATION_EXECUTION, ], [Tier.enterprise]: Object.values(Feature), }; diff --git a/packages/shared/src/types/migration.ts b/packages/shared/src/types/migration.ts new file mode 100644 index 00000000..a2b18509 --- /dev/null +++ b/packages/shared/src/types/migration.ts @@ -0,0 +1,234 @@ +export type MigrationJobStatus = + | 'pending' + | 'running' + | 'completed' + | 'failed' + | 'cancelled'; + +export type IncompatibilitySeverity = 'blocking' | 'warning' | 'info'; + +export interface Incompatibility { + severity: IncompatibilitySeverity; + category: string; + title: string; + detail: string; +} + +export interface MigrationAnalysisRequest { + sourceConnectionId: string; + targetConnectionId: string; + scanSampleSize?: number; // default 10000, range 1000-50000 +} + +export interface DataTypeCount { + count: number; + sampledMemoryBytes: number; + estimatedTotalMemoryBytes: number; +} + +export interface DataTypeBreakdown { + string: DataTypeCount; + hash: DataTypeCount; + list: DataTypeCount; + set: DataTypeCount; + zset: DataTypeCount; + stream: DataTypeCount; + other: DataTypeCount; +} + +export interface TtlDistribution { + noExpiry: number; + expiresWithin1h: number; + expiresWithin24h: number; + expiresWithin7d: number; + expiresAfter7d: number; + sampledKeyCount: number; +} + +export interface CommandAnalysis { + sourceUsed: 'commandlog' | 'slowlog' | 'unavailable'; + topCommands: Array<{ command: string; count: number }>; +} + +export interface MigrationAnalysisResult { + id: string; + status: MigrationJobStatus; + progress: number; // 0-100 + createdAt: number; + completedAt?: number; + error?: string; + + // Source metadata + sourceConnectionId?: string; + sourceConnectionName?: string; + sourceDbType?: 'valkey' | 'redis'; + sourceDbVersion?: string; + isCluster?: boolean; + clusterMasterCount?: number; + + // Target metadata + targetConnectionId?: string; + targetConnectionName?: string; + targetDbType?: 'valkey' | 'redis'; + targetDbVersion?: string; + targetIsCluster?: boolean; + + // Key / memory overview + totalKeys?: number; + sampledKeys?: number; + sampledPerNode?: number; // scanSampleSize used + totalMemoryBytes?: number; + estimatedTotalMemoryBytes?: number; + + // Section results + dataTypeBreakdown?: DataTypeBreakdown; + hfeDetected?: boolean; + hfeKeyCount?: number; // estimated from sample ratio + hfeSupported?: boolean; // false on Redis + hfeOversizedHashesSkipped?: number; + ttlDistribution?: TtlDistribution; + commandAnalysis?: CommandAnalysis; + + // Compatibility + incompatibilities?: Incompatibility[]; + blockingCount?: number; + warningCount?: number; +} + +export interface StartAnalysisResponse { + id: string; + status: 'pending'; +} + +// ── Phase 2: Execution types ── + +export type ExecutionJobStatus = + | 'pending' + | 'running' + | 'completed' + | 'failed' + | 'cancelled'; + +export type ExecutionMode = 'redis_shake' | 'command'; + +export interface MigrationExecutionRequest { + sourceConnectionId: string; + targetConnectionId: string; + mode?: ExecutionMode; // default 'redis_shake' +} + +export interface MigrationExecutionResult { + id: string; + status: ExecutionJobStatus; + mode: ExecutionMode; + startedAt: number; + completedAt?: number; + error?: string; + keysTransferred?: number; + bytesTransferred?: number; + keysSkipped?: number; + totalKeys?: number; + // Rolling log buffer — last 500 lines. + logs: string[]; + // Parsed progress 0–100, best-effort. null if unparseable. + progress: number | null; +} + +export interface StartExecutionResponse { + id: string; + status: 'pending'; +} + +// ── Phase 3: Validation types ── + +export type ValidationJobStatus = + | 'pending' + | 'running' + | 'completed' + | 'failed' + | 'cancelled'; + +export interface KeyCountComparison { + sourceKeys: number; + targetKeys: number; + discrepancy: number; // targetKeys - sourceKeys (negative = data loss) + discrepancyPercent: number; // abs(discrepancy / sourceKeys) * 100 + warning?: string; // e.g. multi-db source vs cluster target + // Per-type estimates from Phase 1 analysis, null if analysis unavailable + typeBreakdown?: Array<{ + type: string; + sourceEstimate: number; + targetEstimate: number; + }>; +} + +export type SampleKeyStatus = 'match' | 'missing' | 'type_mismatch' | 'value_mismatch'; + +export interface SampleKeyResult { + key: string; + type: string; + status: SampleKeyStatus; + detail?: string; // human-readable explanation for non-match +} + +export interface SampleValidationResult { + sampledKeys: number; + matched: number; + missing: number; + typeMismatches: number; + valueMismatches: number; + // Only the non-matching keys, capped at 50 entries + issues: SampleKeyResult[]; +} + +export type BaselineMetricStatus = 'normal' | 'elevated' | 'degraded' | 'unavailable'; + +export interface BaselineMetric { + name: string; + sourceBaseline: number | null; // avg from pre-migration snapshots, null if unavailable + targetCurrent: number | null; + percentDelta: number | null; // ((target - source) / source) * 100, null if unavailable + status: BaselineMetricStatus; // 'unavailable' if sourceBaseline is null +} + +export interface BaselineComparison { + available: boolean; + unavailableReason?: string; // set when available is false + snapshotCount: number; // how many source snapshots were used + baselineWindowMs: number; // time window used (e.g. last 24h before migration) + metrics: BaselineMetric[]; // opsPerSec, usedMemory, memFragmentationRatio, cpuSys +} + +export interface MigrationValidationRequest { + sourceConnectionId: string; + targetConnectionId: string; + // Optional: link to the analysis that produced the source type breakdown + analysisId?: string; + // Optional: timestamp when migration started, used to bound the baseline window + migrationStartedAt?: number; +} + +export interface MigrationValidationResult { + id: string; + status: ValidationJobStatus; + progress: number; // 0–100 + createdAt: number; + completedAt?: number; + error?: string; + + sourceConnectionId?: string; + targetConnectionId?: string; + + keyCount?: KeyCountComparison; + sampleValidation?: SampleValidationResult; + baseline?: BaselineComparison; + + // Overall health signal + issueCount?: number; // total: missing + type/value mismatches + baseline flags + passed?: boolean; // true if issueCount === 0 and no blocking discrepancies +} + +export interface StartValidationResponse { + id: string; + status: 'pending'; +}