diff --git a/Dockerfile b/Dockerfile index 1796097a5..406784bcc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,7 @@ RUN apt-get update && \ echo "deb https://apt.stellar.org jammy unstable" >/etc/apt/sources.list.d/SDF-unstable.list COPY --from=api-build /bin/wallet-backend /app/ +COPY --from=api-build /src/wallet-backend/config /app/config EXPOSE 8001 WORKDIR /app diff --git a/cmd/ingest.go b/cmd/ingest.go index f512b30de..7404aa075 100644 --- a/cmd/ingest.go +++ b/cmd/ingest.go @@ -37,21 +37,61 @@ func (c *ingestCmd) Command() *cobra.Command { utils.RedisHostOption(&cfg.RedisHost), utils.RedisPortOption(&cfg.RedisPort), { - Name: "ledger-cursor-name", + Name: "ingestion-mode", + Usage: "What mode to run ingestion in - live or backfill", + OptType: types.String, + ConfigKey: &cfg.IngestionMode, + FlagDefault: string(ingest.IngestionModeLive), + Required: true, + }, + { + Name: "latest-ledger-cursor-name", Usage: "Name of last synced ledger cursor, used to keep track of the last ledger ingested by the service. When starting up, ingestion will resume from the ledger number stored in this record. It should be an unique name per container as different containers would overwrite the cursor value of its peers when using the same cursor name.", OptType: types.String, - ConfigKey: &cfg.LedgerCursorName, - FlagDefault: "live_ingest_cursor", + ConfigKey: &cfg.LatestLedgerCursorName, + FlagDefault: "latest_ingest_ledger", Required: true, }, { - Name: "account-tokens-cursor-name", - Usage: "Name of last synced account tokens ledger cursor, used to keep track of the last ledger ingested by the service.", + Name: "oldest-ledger-cursor-name", + Usage: "Name of the oldest ledger cursor, used to track the earliest ledger ingested by the service. Used for backfill operations to know where historical data begins.", OptType: types.String, - ConfigKey: &cfg.AccountTokensCursorName, - FlagDefault: "live_account_tokens_ingest_cursor", + ConfigKey: &cfg.OldestLedgerCursorName, + FlagDefault: "oldest_ingest_ledger", Required: true, }, + { + Name: "backfill-workers", + Usage: "Maximum concurrent workers for backfill processing. Defaults to number of CPUs. Lower values reduce RAM usage at cost of throughput.", + OptType: types.Int, + ConfigKey: &cfg.BackfillWorkers, + FlagDefault: 0, + Required: false, + }, + { + Name: "backfill-batch-size", + Usage: "Number of ledgers per batch during backfill. Defaults to 250. Lower values reduce RAM usage at cost of more DB transactions.", + OptType: types.Int, + ConfigKey: &cfg.BackfillBatchSize, + FlagDefault: 250, + Required: false, + }, + { + Name: "backfill-db-insert-batch-size", + Usage: "Number of ledgers to process before flushing buffer to DB during backfill. Defaults to 50. Lower values reduce RAM usage at cost of more DB transactions.", + OptType: types.Int, + ConfigKey: &cfg.BackfillDBInsertBatchSize, + FlagDefault: 100, + Required: false, + }, + { + Name: "catchup-threshold", + Usage: "Number of ledgers behind network tip that triggers fast catchup via backfilling. Defaults to 100.", + OptType: types.Int, + ConfigKey: &cfg.CatchupThreshold, + FlagDefault: 100, + Required: false, + }, { Name: "archive-url", Usage: "Archive URL for history archives", @@ -73,7 +113,7 @@ func (c *ingestCmd) Command() *cobra.Command { Usage: "Type of ledger backend to use for fetching ledgers. Options: 'rpc' (default) or 'datastore'", OptType: types.String, ConfigKey: &ledgerBackendType, - FlagDefault: string(ingest.LedgerBackendTypeRPC), + FlagDefault: string(ingest.LedgerBackendTypeDatastore), Required: false, }, { @@ -81,7 +121,31 @@ func (c *ingestCmd) Command() *cobra.Command { Usage: "Path to TOML config file for datastore backend. Required when ledger-backend-type is 'datastore'", OptType: types.String, ConfigKey: &cfg.DatastoreConfigPath, - FlagDefault: "", + FlagDefault: "config/datastore-pubnet.toml", + Required: false, + }, + { + Name: "skip-tx-meta", + Usage: "Skip storing transaction metadata (meta_xdr) to reduce storage space and improve insertion performance.", + OptType: types.Bool, + ConfigKey: &cfg.SkipTxMeta, + FlagDefault: true, + Required: false, + }, + { + Name: "skip-tx-envelope", + Usage: "Skip storing transaction envelope (envelope_xdr) to reduce storage space and improve insertion performance.", + OptType: types.Bool, + ConfigKey: &cfg.SkipTxEnvelope, + FlagDefault: true, + Required: false, + }, + { + Name: "enable-participant-filtering", + Usage: "When enabled, only store transactions, operations, and state changes for pre-registered accounts. When disabled (default), store all data.", + OptType: types.Bool, + ConfigKey: &cfg.EnableParticipantFiltering, + FlagDefault: false, Required: false, }, } diff --git a/docker-compose.yaml b/docker-compose.yaml index e83084a26..81b91f804 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -23,24 +23,9 @@ services: - postgres-db:/data/postgres ports: - 5432:5432 - command: | - bash -c " - # Run default postgres initialization - /usr/local/bin/docker-entrypoint.sh postgres & - - # Wait for startup - sleep 10 - - # Create schemas (only on first run) - if [ ! -f /data/postgres/schemas_created ]; then - psql -U postgres -d wallet-backend -c 'CREATE SCHEMA IF NOT EXISTS wallet_backend_mainnet;' - psql -U postgres -d wallet-backend -c 'CREATE SCHEMA IF NOT EXISTS wallet_backend_testnet;' - touch /data/postgres/schemas_created - fi - - # Keep postgres running - wait - " + configs: + - source: postgres_init + target: /docker-entrypoint-initdb.d/init-schemas.sql redis: container_name: redis @@ -305,3 +290,9 @@ volumes: driver: local redis-data: driver: local + +configs: + postgres_init: + content: | + CREATE SCHEMA IF NOT EXISTS wallet_backend_mainnet; + CREATE SCHEMA IF NOT EXISTS wallet_backend_testnet; diff --git a/go.mod b/go.mod index 1393e283b..01170e2b5 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/go-chi/chi v4.1.2+incompatible github.com/go-playground/validator/v10 v10.27.0 github.com/golang-jwt/jwt/v5 v5.2.3 + github.com/jackc/pgx/v5 v5.7.6 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.28 @@ -122,6 +123,9 @@ require ( github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect diff --git a/go.sum b/go.sum index ae22be05d..3b49b26ba 100644 --- a/go.sum +++ b/go.sum @@ -298,6 +298,14 @@ github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jarcoal/httpmock v0.0.0-20161210151336-4442edb3db31 h1:Aw95BEvxJ3K6o9GGv5ppCd1P8hkeIeEJ30FO+OhOJpM= github.com/jarcoal/httpmock v0.0.0-20161210151336-4442edb3db31/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= diff --git a/internal/data/accounts.go b/internal/data/accounts.go index 26de3d767..2e369507c 100644 --- a/internal/data/accounts.go +++ b/internal/data/accounts.go @@ -150,13 +150,10 @@ func (m *AccountModel) IsAccountFeeBumpEligible(ctx context.Context, address str // BatchGetByTxHashes gets the accounts that are associated with the given transaction hashes. func (m *AccountModel) BatchGetByTxHashes(ctx context.Context, txHashes []string, columns string) ([]*types.AccountWithTxHash, error) { - columns = prepareColumnsWithID(columns, types.Account{}, "accounts", "stellar_address") - query := fmt.Sprintf(` - SELECT %s, transactions_accounts.tx_hash + query := ` + SELECT account_id AS stellar_address, tx_hash FROM transactions_accounts - INNER JOIN accounts - ON transactions_accounts.account_id = accounts.stellar_address - WHERE transactions_accounts.tx_hash = ANY($1)`, columns) + WHERE tx_hash = ANY($1)` var accounts []*types.AccountWithTxHash start := time.Now() err := m.DB.SelectContext(ctx, &accounts, query, pq.Array(txHashes)) @@ -173,13 +170,10 @@ func (m *AccountModel) BatchGetByTxHashes(ctx context.Context, txHashes []string // BatchGetByOperationIDs gets the accounts that are associated with the given operation IDs. func (m *AccountModel) BatchGetByOperationIDs(ctx context.Context, operationIDs []int64, columns string) ([]*types.AccountWithOperationID, error) { - columns = prepareColumnsWithID(columns, types.Account{}, "accounts", "stellar_address") - query := fmt.Sprintf(` - SELECT %s, operations_accounts.operation_id + query := ` + SELECT account_id AS stellar_address, operation_id FROM operations_accounts - INNER JOIN accounts - ON operations_accounts.account_id = accounts.stellar_address - WHERE operations_accounts.operation_id = ANY($1)`, columns) + WHERE operation_id = ANY($1)` var accounts []*types.AccountWithOperationID start := time.Now() err := m.DB.SelectContext(ctx, &accounts, query, pq.Array(operationIDs)) @@ -196,8 +190,6 @@ func (m *AccountModel) BatchGetByOperationIDs(ctx context.Context, operationIDs // BatchGetByStateChangeIDs gets the accounts that are associated with the given state change IDs. func (m *AccountModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs []int64, scOrders []int64, columns string) ([]*types.AccountWithStateChangeID, error) { - columns = prepareColumnsWithID(columns, types.Account{}, "accounts", "stellar_address") - // Build tuples for the IN clause. Since (to_id, state_change_order) is the primary key of state_changes, // it will be faster to search on this tuple. tuples := make([]string, len(scOrders)) @@ -206,12 +198,11 @@ func (m *AccountModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs []i } query := fmt.Sprintf(` - SELECT %s, CONCAT(state_changes.to_id, '-', state_changes.state_change_order) AS state_change_id - FROM accounts - INNER JOIN state_changes ON accounts.stellar_address = state_changes.account_id - WHERE (state_changes.to_id, state_changes.state_change_order) IN (%s) - ORDER BY accounts.created_at DESC - `, columns, strings.Join(tuples, ", ")) + SELECT account_id AS stellar_address, CONCAT(to_id, '-', state_change_order) AS state_change_id + FROM state_changes + WHERE (to_id, state_change_order) IN (%s) + ORDER BY ledger_created_at DESC + `, strings.Join(tuples, ", ")) var accountsWithStateChanges []*types.AccountWithStateChangeID start := time.Now() diff --git a/internal/data/ingest_store.go b/internal/data/ingest_store.go index 21d741b52..c28de9295 100644 --- a/internal/data/ingest_store.go +++ b/internal/data/ingest_store.go @@ -12,6 +12,11 @@ import ( "github.com/stellar/wallet-backend/internal/utils" ) +type LedgerRange struct { + GapStart uint32 `db:"gap_start"` + GapEnd uint32 `db:"gap_end"` +} + type IngestStoreModel struct { DB db.ConnectionPool MetricsService metrics.MetricsService @@ -22,17 +27,17 @@ func (m *IngestStoreModel) Get(ctx context.Context, cursorName string) (uint32, start := time.Now() err := m.DB.GetContext(ctx, &lastSyncedLedger, `SELECT value FROM ingest_store WHERE key = $1`, cursorName) duration := time.Since(start).Seconds() - m.MetricsService.ObserveDBQueryDuration("GetLatestLedgerSynced", "ingest_store", duration) + m.MetricsService.ObserveDBQueryDuration("Get", "ingest_store", duration) // First run, key does not exist yet if errors.Is(err, sql.ErrNoRows) { - m.MetricsService.IncDBQuery("GetLatestLedgerSynced", "ingest_store") + m.MetricsService.IncDBQuery("Get", "ingest_store") return 0, nil } if err != nil { - m.MetricsService.IncDBQueryError("GetLatestLedgerSynced", "ingest_store", utils.GetDBErrorType(err)) + m.MetricsService.IncDBQueryError("Get", "ingest_store", utils.GetDBErrorType(err)) return 0, fmt.Errorf("getting latest ledger synced for cursor %s: %w", cursorName, err) } - m.MetricsService.IncDBQuery("GetLatestLedgerSynced", "ingest_store") + m.MetricsService.IncDBQuery("Get", "ingest_store") return lastSyncedLedger, nil } @@ -45,12 +50,48 @@ func (m *IngestStoreModel) Update(ctx context.Context, dbTx db.Transaction, curs start := time.Now() _, err := dbTx.ExecContext(ctx, query, cursorName, ledger) duration := time.Since(start).Seconds() - m.MetricsService.ObserveDBQueryDuration("UpdateLatestLedgerSynced", "ingest_store", duration) + m.MetricsService.ObserveDBQueryDuration("Update", "ingest_store", duration) if err != nil { - m.MetricsService.IncDBQueryError("UpdateLatestLedgerSynced", "ingest_store", utils.GetDBErrorType(err)) + m.MetricsService.IncDBQueryError("Update", "ingest_store", utils.GetDBErrorType(err)) return fmt.Errorf("updating last synced ledger to %d: %w", ledger, err) } - m.MetricsService.IncDBQuery("UpdateLatestLedgerSynced", "ingest_store") + m.MetricsService.IncDBQuery("Update", "ingest_store") + return nil +} +func (m *IngestStoreModel) UpdateMin(ctx context.Context, dbTx db.Transaction, cursorName string, ledger uint32) error { + const query = ` + UPDATE ingest_store + SET value = LEAST(value::integer, $2)::text + WHERE key = $1 + ` + _, err := dbTx.ExecContext(ctx, query, cursorName, ledger) + if err != nil { + return fmt.Errorf("updating minimum ledger for cursor %s: %w", cursorName, err) + } return nil } + +func (m *IngestStoreModel) GetLedgerGaps(ctx context.Context) ([]LedgerRange, error) { + const query = ` + SELECT gap_start, gap_end FROM ( + SELECT + ledger_number + 1 AS gap_start, + LEAD(ledger_number) OVER (ORDER BY ledger_number) - 1 AS gap_end + FROM (SELECT DISTINCT ledger_number FROM transactions) t + ) gaps + WHERE gap_start <= gap_end + ORDER BY gap_start + ` + start := time.Now() + var ledgerGaps []LedgerRange + err := m.DB.SelectContext(ctx, &ledgerGaps, query) + duration := time.Since(start).Seconds() + m.MetricsService.ObserveDBQueryDuration("GetLedgerGaps", "transactions", duration) + if err != nil { + m.MetricsService.IncDBQueryError("GetLedgerGaps", "transactions", utils.GetDBErrorType(err)) + return nil, fmt.Errorf("getting ledger gaps: %w", err) + } + m.MetricsService.IncDBQuery("GetLedgerGaps", "transactions") + return ledgerGaps, nil +} diff --git a/internal/data/ingest_store_test.go b/internal/data/ingest_store_test.go index 9dfaf9ad6..ef8bd907e 100644 --- a/internal/data/ingest_store_test.go +++ b/internal/data/ingest_store_test.go @@ -2,6 +2,7 @@ package data import ( "context" + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -51,8 +52,8 @@ func Test_IngestStoreModel_GetLatestLedgerSynced(t *testing.T) { mockMetricsService := metrics.NewMockMetricsService() mockMetricsService. - On("ObserveDBQueryDuration", "GetLatestLedgerSynced", "ingest_store", mock.Anything).Return(). - On("IncDBQuery", "GetLatestLedgerSynced", "ingest_store").Return() + On("ObserveDBQueryDuration", "Get", "ingest_store", mock.Anything).Return(). + On("IncDBQuery", "Get", "ingest_store").Return() defer mockMetricsService.AssertExpectations(t) m := &IngestStoreModel{ @@ -108,8 +109,8 @@ func Test_IngestStoreModel_UpdateLatestLedgerSynced(t *testing.T) { mockMetricsService := metrics.NewMockMetricsService() mockMetricsService. - On("ObserveDBQueryDuration", "UpdateLatestLedgerSynced", "ingest_store", mock.Anything).Return().Once(). - On("IncDBQuery", "UpdateLatestLedgerSynced", "ingest_store").Return().Once() + On("ObserveDBQueryDuration", "Update", "ingest_store", mock.Anything).Return().Once(). + On("IncDBQuery", "Update", "ingest_store").Return().Once() defer mockMetricsService.AssertExpectations(t) m := &IngestStoreModel{ @@ -136,3 +137,181 @@ func Test_IngestStoreModel_UpdateLatestLedgerSynced(t *testing.T) { }) } } + +func Test_IngestStoreModel_UpdateMin(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + + testCases := []struct { + name string + key string + initialValue uint32 + newValue uint32 + expectedResult uint32 + }{ + { + name: "updates_to_smaller_value", + key: "oldest_ledger_cursor", + initialValue: 1000, + newValue: 500, + expectedResult: 500, + }, + { + name: "keeps_existing_smaller_value", + key: "oldest_ledger_cursor", + initialValue: 500, + newValue: 1000, + expectedResult: 500, + }, + { + name: "keeps_same_value", + key: "oldest_ledger_cursor", + initialValue: 500, + newValue: 500, + expectedResult: 500, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store") + require.NoError(t, err) + + // Insert initial value + _, err = dbConnectionPool.ExecContext(ctx, `INSERT INTO ingest_store (key, value) VALUES ($1, $2)`, tc.key, tc.initialValue) + require.NoError(t, err) + + mockMetricsService := metrics.NewMockMetricsService() + + m := &IngestStoreModel{ + DB: dbConnectionPool, + MetricsService: mockMetricsService, + } + + err = db.RunInTransaction(ctx, m.DB, nil, func(dbTx db.Transaction) error { + return m.UpdateMin(ctx, dbTx, tc.key, tc.newValue) + }) + require.NoError(t, err) + + var dbStoredLedger uint32 + err = m.DB.GetContext(ctx, &dbStoredLedger, `SELECT value FROM ingest_store WHERE key = $1`, tc.key) + require.NoError(t, err) + assert.Equal(t, tc.expectedResult, dbStoredLedger) + }) + } +} + +func Test_IngestStoreModel_GetLedgerGaps(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + + testCases := []struct { + name string + setupDB func(t *testing.T) + expectedGaps []LedgerRange + }{ + { + name: "returns_empty_when_no_transactions", + expectedGaps: nil, + }, + { + name: "returns_empty_when_no_gaps", + setupDB: func(t *testing.T) { + // Insert consecutive ledgers: 100, 101, 102 + for i, ledger := range []uint32{100, 101, 102} { + _, err := dbConnectionPool.ExecContext(ctx, + `INSERT INTO transactions (hash, to_id, envelope_xdr, result_xdr, meta_xdr, ledger_number, ledger_created_at) + VALUES ($1, $2, 'env', 'res', 'meta', $3, NOW())`, + fmt.Sprintf("hash%d", i), i+1, ledger) + require.NoError(t, err) + } + }, + expectedGaps: nil, + }, + { + name: "returns_single_gap", + setupDB: func(t *testing.T) { + // Insert ledgers 100 and 105, creating gap 101-104 + for i, ledger := range []uint32{100, 105} { + _, err := dbConnectionPool.ExecContext(ctx, + `INSERT INTO transactions (hash, to_id, envelope_xdr, result_xdr, meta_xdr, ledger_number, ledger_created_at) + VALUES ($1, $2, 'env', 'res', 'meta', $3, NOW())`, + fmt.Sprintf("hash%d", i), i+1, ledger) + require.NoError(t, err) + } + }, + expectedGaps: []LedgerRange{ + {GapStart: 101, GapEnd: 104}, + }, + }, + { + name: "returns_multiple_gaps", + setupDB: func(t *testing.T) { + // Insert ledgers 100, 105, 110, creating gaps 101-104 and 106-109 + for i, ledger := range []uint32{100, 105, 110} { + _, err := dbConnectionPool.ExecContext(ctx, + `INSERT INTO transactions (hash, to_id, envelope_xdr, result_xdr, meta_xdr, ledger_number, ledger_created_at) + VALUES ($1, $2, 'env', 'res', 'meta', $3, NOW())`, + fmt.Sprintf("hash%d", i), i+1, ledger) + require.NoError(t, err) + } + }, + expectedGaps: []LedgerRange{ + {GapStart: 101, GapEnd: 104}, + {GapStart: 106, GapEnd: 109}, + }, + }, + { + name: "handles_single_ledger_gap", + setupDB: func(t *testing.T) { + // Insert ledgers 100 and 102, creating gap of just 101 + for i, ledger := range []uint32{100, 102} { + _, err := dbConnectionPool.ExecContext(ctx, + `INSERT INTO transactions (hash, to_id, envelope_xdr, result_xdr, meta_xdr, ledger_number, ledger_created_at) + VALUES ($1, $2, 'env', 'res', 'meta', $3, NOW())`, + fmt.Sprintf("hash%d", i), i+1, ledger) + require.NoError(t, err) + } + }, + expectedGaps: []LedgerRange{ + {GapStart: 101, GapEnd: 101}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := dbConnectionPool.ExecContext(ctx, "DELETE FROM transactions") + require.NoError(t, err) + + mockMetricsService := metrics.NewMockMetricsService() + mockMetricsService. + On("ObserveDBQueryDuration", "GetLedgerGaps", "transactions", mock.Anything).Return(). + On("IncDBQuery", "GetLedgerGaps", "transactions").Return() + defer mockMetricsService.AssertExpectations(t) + + m := &IngestStoreModel{ + DB: dbConnectionPool, + MetricsService: mockMetricsService, + } + + if tc.setupDB != nil { + tc.setupDB(t) + } + + gaps, err := m.GetLedgerGaps(ctx) + require.NoError(t, err) + assert.Equal(t, tc.expectedGaps, gaps) + }) + } +} diff --git a/internal/data/operations.go b/internal/data/operations.go index 5d3990d60..4b37c1729 100644 --- a/internal/data/operations.go +++ b/internal/data/operations.go @@ -7,6 +7,8 @@ import ( "time" set "github.com/deckarep/golang-set/v2" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" "github.com/lib/pq" "github.com/stellar/wallet-backend/internal/db" @@ -241,7 +243,7 @@ func (m *OperationModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs [ func (m *OperationModel) BatchInsert( ctx context.Context, sqlExecuter db.SQLExecuter, - operations []types.Operation, + operations []*types.Operation, stellarAddressesByOpID map[int64]set.Set[string], ) ([]int64, error) { if sqlExecuter == nil { @@ -346,3 +348,75 @@ func (m *OperationModel) BatchInsert( return insertedIDs, nil } + +// BatchCopy inserts operations using pgx's binary COPY protocol. +// Uses pgx.Tx for binary format which is faster than lib/pq's text format. +// Uses native pgtype types for optimal performance (see https://github.com/jackc/pgx/issues/763). +func (m *OperationModel) BatchCopy( + ctx context.Context, + pgxTx pgx.Tx, + operations []*types.Operation, + stellarAddressesByOpID map[int64]set.Set[string], +) (int, error) { + if len(operations) == 0 { + return 0, nil + } + + start := time.Now() + + // COPY operations using pgx binary format with native pgtype types + copyCount, err := pgxTx.CopyFrom( + ctx, + pgx.Identifier{"operations"}, + []string{"id", "tx_hash", "operation_type", "operation_xdr", "ledger_number", "ledger_created_at"}, + pgx.CopyFromSlice(len(operations), func(i int) ([]any, error) { + op := operations[i] + return []any{ + pgtype.Int8{Int64: op.ID, Valid: true}, + pgtype.Text{String: op.TxHash, Valid: true}, + pgtype.Text{String: string(op.OperationType), Valid: true}, + pgtype.Text{String: op.OperationXDR, Valid: true}, + pgtype.Int4{Int32: int32(op.LedgerNumber), Valid: true}, + pgtype.Timestamptz{Time: op.LedgerCreatedAt, Valid: true}, + }, nil + }), + ) + if err != nil { + m.MetricsService.IncDBQueryError("BatchCopy", "operations", utils.GetDBErrorType(err)) + return 0, fmt.Errorf("pgx CopyFrom operations: %w", err) + } + if int(copyCount) != len(operations) { + return 0, fmt.Errorf("expected %d rows copied, got %d", len(operations), copyCount) + } + + // COPY operations_accounts using pgx binary format with native pgtype types + if len(stellarAddressesByOpID) > 0 { + var oaRows [][]any + for opID, addresses := range stellarAddressesByOpID { + opIDPgtype := pgtype.Int8{Int64: opID, Valid: true} + for _, addr := range addresses.ToSlice() { + oaRows = append(oaRows, []any{opIDPgtype, pgtype.Text{String: addr, Valid: true}}) + } + } + + _, err = pgxTx.CopyFrom( + ctx, + pgx.Identifier{"operations_accounts"}, + []string{"operation_id", "account_id"}, + pgx.CopyFromRows(oaRows), + ) + if err != nil { + m.MetricsService.IncDBQueryError("BatchCopy", "operations_accounts", utils.GetDBErrorType(err)) + return 0, fmt.Errorf("pgx CopyFrom operations_accounts: %w", err) + } + + m.MetricsService.IncDBQuery("BatchCopy", "operations_accounts") + } + + duration := time.Since(start).Seconds() + m.MetricsService.ObserveDBQueryDuration("BatchCopy", "operations", duration) + m.MetricsService.ObserveDBBatchSize("BatchCopy", "operations", len(operations)) + m.MetricsService.IncDBQuery("BatchCopy", "operations") + + return len(operations), nil +} diff --git a/internal/data/operations_test.go b/internal/data/operations_test.go index 96d2fbbbc..fe959b531 100644 --- a/internal/data/operations_test.go +++ b/internal/data/operations_test.go @@ -2,10 +2,12 @@ package data import ( "context" + "fmt" "testing" "time" set "github.com/deckarep/golang-set/v2" + "github.com/jackc/pgx/v5" "github.com/stellar/go/keypair" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -17,6 +19,31 @@ import ( "github.com/stellar/wallet-backend/internal/metrics" ) +// generateTestOperations creates n test operations for benchmarking. +// It also returns the transaction hash used and a map of operation IDs to addresses. +func generateTestOperations(n int, txHash string, startID int64) ([]*types.Operation, map[int64]set.Set[string]) { + ops := make([]*types.Operation, n) + addressesByOpID := make(map[int64]set.Set[string]) + now := time.Now() + + for i := 0; i < n; i++ { + opID := startID + int64(i) + address := keypair.MustRandom().Address() + + ops[i] = &types.Operation{ + ID: opID, + TxHash: txHash, + OperationType: types.OperationTypePayment, + OperationXDR: fmt.Sprintf("operation_xdr_%d", i), + LedgerNumber: uint32(i + 1), + LedgerCreatedAt: now, + } + addressesByOpID[opID] = set.NewSet(address) + } + + return ops, addressesByOpID +} + func Test_OperationModel_BatchInsert(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -35,21 +62,23 @@ func Test_OperationModel_BatchInsert(t *testing.T) { require.NoError(t, err) // Create referenced transactions first + meta1, meta2 := "meta1", "meta2" + envelope1, envelope2 := "envelope1", "envelope2" tx1 := types.Transaction{ Hash: "tx1", ToID: 1, - EnvelopeXDR: "envelope1", + EnvelopeXDR: &envelope1, ResultXDR: "result1", - MetaXDR: "meta1", + MetaXDR: &meta1, LedgerNumber: 1, LedgerCreatedAt: now, } tx2 := types.Transaction{ Hash: "tx2", ToID: 2, - EnvelopeXDR: "envelope2", + EnvelopeXDR: &envelope2, ResultXDR: "result2", - MetaXDR: "meta2", + MetaXDR: &meta2, LedgerNumber: 2, LedgerCreatedAt: now, } @@ -58,7 +87,7 @@ func Test_OperationModel_BatchInsert(t *testing.T) { sqlxDB, err := dbConnectionPool.SqlxDB(ctx) require.NoError(t, err) txModel := &TransactionModel{DB: dbConnectionPool, MetricsService: metrics.NewMetricsService(sqlxDB)} - _, err = txModel.BatchInsert(ctx, nil, []types.Transaction{tx1, tx2}, map[string]set.Set[string]{ + _, err = txModel.BatchInsert(ctx, nil, []*types.Transaction{&tx1, &tx2}, map[string]set.Set[string]{ tx1.Hash: set.NewSet(kp1.Address()), tx2.Hash: set.NewSet(kp2.Address()), }) @@ -82,7 +111,7 @@ func Test_OperationModel_BatchInsert(t *testing.T) { testCases := []struct { name string useDBTx bool - operations []types.Operation + operations []*types.Operation stellarAddressesByOpID map[int64]set.Set[string] wantAccountLinks map[int64][]string wantErrContains string @@ -91,7 +120,7 @@ func Test_OperationModel_BatchInsert(t *testing.T) { { name: "🟢successful_insert_without_dbTx", useDBTx: false, - operations: []types.Operation{op1, op2}, + operations: []*types.Operation{&op1, &op2}, stellarAddressesByOpID: map[int64]set.Set[string]{op1.ID: set.NewSet(kp1.Address(), kp1.Address(), kp1.Address(), kp1.Address()), op2.ID: set.NewSet(kp2.Address(), kp2.Address())}, wantAccountLinks: map[int64][]string{op1.ID: {kp1.Address()}, op2.ID: {kp2.Address()}}, wantErrContains: "", @@ -100,7 +129,7 @@ func Test_OperationModel_BatchInsert(t *testing.T) { { name: "🟢successful_insert_with_dbTx", useDBTx: true, - operations: []types.Operation{op1}, + operations: []*types.Operation{&op1}, stellarAddressesByOpID: map[int64]set.Set[string]{op1.ID: set.NewSet(kp1.Address())}, wantAccountLinks: map[int64][]string{op1.ID: {kp1.Address()}}, wantErrContains: "", @@ -109,7 +138,7 @@ func Test_OperationModel_BatchInsert(t *testing.T) { { name: "🟢empty_input", useDBTx: false, - operations: []types.Operation{}, + operations: []*types.Operation{}, stellarAddressesByOpID: map[int64]set.Set[string]{}, wantAccountLinks: map[int64][]string{}, wantErrContains: "", @@ -118,7 +147,7 @@ func Test_OperationModel_BatchInsert(t *testing.T) { { name: "🟡duplicate_operation", useDBTx: false, - operations: []types.Operation{op1, op1}, + operations: []*types.Operation{&op1, &op1}, stellarAddressesByOpID: map[int64]set.Set[string]{op1.ID: set.NewSet(kp1.Address())}, wantAccountLinks: map[int64][]string{op1.ID: {kp1.Address()}}, wantErrContains: "", @@ -198,6 +227,178 @@ func Test_OperationModel_BatchInsert(t *testing.T) { } } +func Test_OperationModel_BatchCopy(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + now := time.Now() + + // Create test accounts + kp1 := keypair.MustRandom() + kp2 := keypair.MustRandom() + const q = "INSERT INTO accounts (stellar_address) SELECT UNNEST(ARRAY[$1, $2])" + _, err = dbConnectionPool.ExecContext(ctx, q, kp1.Address(), kp2.Address()) + require.NoError(t, err) + + // Create referenced transactions first + meta1, meta2 := "meta1", "meta2" + envelope1, envelope2 := "envelope1", "envelope2" + tx1 := types.Transaction{ + Hash: "tx1", + ToID: 1, + EnvelopeXDR: &envelope1, + ResultXDR: "result1", + MetaXDR: &meta1, + LedgerNumber: 1, + LedgerCreatedAt: now, + } + tx2 := types.Transaction{ + Hash: "tx2", + ToID: 2, + EnvelopeXDR: &envelope2, + ResultXDR: "result2", + MetaXDR: &meta2, + LedgerNumber: 2, + LedgerCreatedAt: now, + } + + // Insert transactions + sqlxDB, err := dbConnectionPool.SqlxDB(ctx) + require.NoError(t, err) + txModel := &TransactionModel{DB: dbConnectionPool, MetricsService: metrics.NewMetricsService(sqlxDB)} + _, err = txModel.BatchInsert(ctx, nil, []*types.Transaction{&tx1, &tx2}, map[string]set.Set[string]{ + tx1.Hash: set.NewSet(kp1.Address()), + tx2.Hash: set.NewSet(kp2.Address()), + }) + require.NoError(t, err) + + op1 := types.Operation{ + ID: 1, + TxHash: tx1.Hash, + OperationType: types.OperationTypePayment, + OperationXDR: "operation1", + LedgerCreatedAt: now, + } + op2 := types.Operation{ + ID: 2, + TxHash: tx2.Hash, + OperationType: types.OperationTypeCreateAccount, + OperationXDR: "operation2", + LedgerCreatedAt: now, + } + + testCases := []struct { + name string + operations []*types.Operation + stellarAddressesByOpID map[int64]set.Set[string] + wantCount int + wantErrContains string + }{ + { + name: "🟢successful_insert_multiple", + operations: []*types.Operation{&op1, &op2}, + stellarAddressesByOpID: map[int64]set.Set[string]{op1.ID: set.NewSet(kp1.Address()), op2.ID: set.NewSet(kp2.Address())}, + wantCount: 2, + }, + { + name: "🟢empty_input", + operations: []*types.Operation{}, + stellarAddressesByOpID: map[int64]set.Set[string]{}, + wantCount: 0, + }, + { + name: "🟢no_participants", + operations: []*types.Operation{&op1}, + stellarAddressesByOpID: map[int64]set.Set[string]{}, + wantCount: 1, + }, + } + + // Create pgx connection for BatchCopy (requires pgx.Tx, not sqlx.Tx) + conn, err := pgx.Connect(ctx, dbt.DSN) + require.NoError(t, err) + defer conn.Close(ctx) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Clear the database before each test + _, err = dbConnectionPool.ExecContext(ctx, "TRUNCATE operations, operations_accounts CASCADE") + require.NoError(t, err) + + // Create fresh mock for each test case + mockMetricsService := metrics.NewMockMetricsService() + // Only set up metric expectations if we have operations to insert + if len(tc.operations) > 0 { + mockMetricsService. + On("ObserveDBQueryDuration", "BatchCopy", "operations", mock.Anything).Return().Once() + mockMetricsService. + On("ObserveDBBatchSize", "BatchCopy", "operations", mock.Anything).Return().Once() + mockMetricsService. + On("IncDBQuery", "BatchCopy", "operations").Return().Once() + if len(tc.stellarAddressesByOpID) > 0 { + mockMetricsService. + On("IncDBQuery", "BatchCopy", "operations_accounts").Return().Once() + } + } + defer mockMetricsService.AssertExpectations(t) + + m := &OperationModel{ + DB: dbConnectionPool, + MetricsService: mockMetricsService, + } + + // BatchCopy requires a pgx transaction + pgxTx, err := conn.Begin(ctx) + require.NoError(t, err) + + gotCount, err := m.BatchCopy(ctx, pgxTx, tc.operations, tc.stellarAddressesByOpID) + + if tc.wantErrContains != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErrContains) + pgxTx.Rollback(ctx) + return + } + + require.NoError(t, err) + require.NoError(t, pgxTx.Commit(ctx)) + assert.Equal(t, tc.wantCount, gotCount) + + // Verify from DB + var dbInsertedIDs []int64 + err = dbConnectionPool.SelectContext(ctx, &dbInsertedIDs, "SELECT id FROM operations ORDER BY id") + require.NoError(t, err) + assert.Len(t, dbInsertedIDs, tc.wantCount) + + // Verify account links if expected + if len(tc.stellarAddressesByOpID) > 0 && tc.wantCount > 0 { + var accountLinks []struct { + OperationID int64 `db:"operation_id"` + AccountID string `db:"account_id"` + } + err = dbConnectionPool.SelectContext(ctx, &accountLinks, "SELECT operation_id, account_id FROM operations_accounts ORDER BY operation_id, account_id") + require.NoError(t, err) + + // Create a map of operation_id -> set of account_ids + accountLinksMap := make(map[int64][]string) + for _, link := range accountLinks { + accountLinksMap[link.OperationID] = append(accountLinksMap[link.OperationID], link.AccountID) + } + + // Verify each expected operation has its account links + for opID, expectedAddresses := range tc.stellarAddressesByOpID { + actualAddresses := accountLinksMap[opID] + assert.ElementsMatch(t, expectedAddresses.ToSlice(), actualAddresses, "account links for op %d don't match", opID) + } + } + }) + } +} + func TestOperationModel_GetAll(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -661,3 +862,139 @@ func TestOperationModel_BatchGetByStateChangeIDs(t *testing.T) { assert.Equal(t, int64(2), stateChangeIDsFound["2-1"]) assert.Equal(t, int64(1), stateChangeIDsFound["3-1"]) } + +func BenchmarkOperationModel_BatchInsert(b *testing.B) { + dbt := dbtest.OpenB(b) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + if err != nil { + b.Fatalf("failed to open db connection pool: %v", err) + } + defer dbConnectionPool.Close() + + ctx := context.Background() + sqlxDB, err := dbConnectionPool.SqlxDB(ctx) + if err != nil { + b.Fatalf("failed to get sqlx db: %v", err) + } + metricsService := metrics.NewMetricsService(sqlxDB) + + m := &OperationModel{ + DB: dbConnectionPool, + MetricsService: metricsService, + } + + // Create a parent transaction that operations will reference + const txHash = "benchmark_tx_hash" + now := time.Now() + _, err = dbConnectionPool.ExecContext(ctx, ` + INSERT INTO transactions (hash, to_id, envelope_xdr, result_xdr, meta_xdr, ledger_number, ledger_created_at) + VALUES ($1, 1, 'env', 'res', 'meta', 1, $2) + `, txHash, now) + if err != nil { + b.Fatalf("failed to create parent transaction: %v", err) + } + + batchSizes := []int{1000, 5000, 10000, 50000, 100000} + + for _, size := range batchSizes { + b.Run(fmt.Sprintf("size=%d", size), func(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + b.StopTimer() + // Clean up operations before each iteration (keep the parent transaction) + //nolint:errcheck // truncate is best-effort cleanup in benchmarks + dbConnectionPool.ExecContext(ctx, "TRUNCATE operations, operations_accounts CASCADE") + // Generate fresh test data for each iteration + ops, addressesByOpID := generateTestOperations(size, txHash, int64(i*size)) + b.StartTimer() + + _, err := m.BatchInsert(ctx, nil, ops, addressesByOpID) + if err != nil { + b.Fatalf("BatchInsert failed: %v", err) + } + } + }) + } +} + +// BenchmarkOperationModel_BatchCopy benchmarks bulk insert using pgx's binary COPY protocol. +func BenchmarkOperationModel_BatchCopy(b *testing.B) { + dbt := dbtest.OpenB(b) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + if err != nil { + b.Fatalf("failed to open db connection pool: %v", err) + } + defer dbConnectionPool.Close() + + ctx := context.Background() + sqlxDB, err := dbConnectionPool.SqlxDB(ctx) + if err != nil { + b.Fatalf("failed to get sqlx db: %v", err) + } + metricsService := metrics.NewMetricsService(sqlxDB) + + m := &OperationModel{ + DB: dbConnectionPool, + MetricsService: metricsService, + } + + // Create pgx connection for BatchCopy + conn, err := pgx.Connect(ctx, dbt.DSN) + if err != nil { + b.Fatalf("failed to connect with pgx: %v", err) + } + defer conn.Close(ctx) + + // Create a parent transaction that operations will reference + const txHash = "benchmark_tx_hash" + now := time.Now() + _, err = conn.Exec(ctx, ` + INSERT INTO transactions (hash, to_id, envelope_xdr, result_xdr, meta_xdr, ledger_number, ledger_created_at) + VALUES ($1, 1, 'env', 'res', 'meta', 1, $2) + `, txHash, now) + if err != nil { + b.Fatalf("failed to create parent transaction: %v", err) + } + + batchSizes := []int{1000, 5000, 10000, 50000, 100000} + + for _, size := range batchSizes { + b.Run(fmt.Sprintf("size=%d", size), func(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + b.StopTimer() + // Clean up operations before each iteration (keep the parent transaction) + _, err = conn.Exec(ctx, "TRUNCATE operations, operations_accounts CASCADE") + if err != nil { + b.Fatalf("failed to truncate: %v", err) + } + + // Generate fresh test data for each iteration + ops, addressesByOpID := generateTestOperations(size, txHash, int64(i*size)) + + // Start a pgx transaction + pgxTx, err := conn.Begin(ctx) + if err != nil { + b.Fatalf("failed to begin transaction: %v", err) + } + b.StartTimer() + + _, err = m.BatchCopy(ctx, pgxTx, ops, addressesByOpID) + if err != nil { + pgxTx.Rollback(ctx) + b.Fatalf("BatchCopy failed: %v", err) + } + + b.StopTimer() + if err := pgxTx.Commit(ctx); err != nil { + b.Fatalf("failed to commit transaction: %v", err) + } + b.StartTimer() + } + }) + } +} diff --git a/internal/data/statechanges.go b/internal/data/statechanges.go index 68ecd4698..4e236e621 100644 --- a/internal/data/statechanges.go +++ b/internal/data/statechanges.go @@ -2,10 +2,13 @@ package data import ( "context" + "database/sql" "fmt" "strings" "time" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" "github.com/lib/pq" "github.com/stellar/wallet-backend/internal/db" @@ -341,6 +344,109 @@ func (m *StateChangeModel) BatchInsert( return insertedIDs, nil } +// pgtypeTextFromNullString converts sql.NullString to pgtype.Text for efficient binary COPY. +func pgtypeTextFromNullString(ns sql.NullString) pgtype.Text { + return pgtype.Text{String: ns.String, Valid: ns.Valid} +} + +// pgtypeTextFromReasonPtr converts *types.StateChangeReason to pgtype.Text for efficient binary COPY. +func pgtypeTextFromReasonPtr(r *types.StateChangeReason) pgtype.Text { + if r == nil { + return pgtype.Text{Valid: false} + } + return pgtype.Text{String: string(*r), Valid: true} +} + +// jsonbFromMap converts types.NullableJSONB to any for pgx CopyFrom. +// pgx automatically handles map[string]any → JSONB conversion. +func jsonbFromMap(m types.NullableJSONB) any { + if m == nil { + return nil + } + // Return the map directly; pgx handles JSON marshaling automatically + return map[string]any(m) +} + +// jsonbFromSlice converts types.NullableJSON to any for pgx CopyFrom. +// pgx automatically handles []string → JSONB conversion. +func jsonbFromSlice(s types.NullableJSON) any { + if s == nil { + return nil + } + // Return the slice directly; pgx handles JSON marshaling automatically + return []string(s) +} + +// BatchCopy inserts state changes using pgx's binary COPY protocol. +// Uses pgx.Tx for binary format which is faster than lib/pq's text format. +// Uses native pgtype types for optimal performance (see https://github.com/jackc/pgx/issues/763). +func (m *StateChangeModel) BatchCopy( + ctx context.Context, + pgxTx pgx.Tx, + stateChanges []types.StateChange, +) (int, error) { + if len(stateChanges) == 0 { + return 0, nil + } + + start := time.Now() + + // COPY state_changes using pgx binary format with native pgtype types + copyCount, err := pgxTx.CopyFrom( + ctx, + pgx.Identifier{"state_changes"}, + []string{ + "to_id", "state_change_order", "state_change_category", "state_change_reason", + "ledger_created_at", "ledger_number", "account_id", "operation_id", "tx_hash", + "token_id", "amount", "offer_id", "signer_account_id", "spender_account_id", + "sponsored_account_id", "sponsor_account_id", "deployer_account_id", "funder_account_id", + "signer_weights", "thresholds", "trustline_limit", "flags", "key_value", + }, + pgx.CopyFromSlice(len(stateChanges), func(i int) ([]any, error) { + sc := stateChanges[i] + return []any{ + pgtype.Int8{Int64: sc.ToID, Valid: true}, + pgtype.Int8{Int64: sc.StateChangeOrder, Valid: true}, + pgtype.Text{String: string(sc.StateChangeCategory), Valid: true}, + pgtypeTextFromReasonPtr(sc.StateChangeReason), + pgtype.Timestamptz{Time: sc.LedgerCreatedAt, Valid: true}, + pgtype.Int4{Int32: int32(sc.LedgerNumber), Valid: true}, + pgtype.Text{String: sc.AccountID, Valid: true}, + pgtype.Int8{Int64: sc.OperationID, Valid: true}, + pgtype.Text{String: sc.TxHash, Valid: true}, + pgtypeTextFromNullString(sc.TokenID), + pgtypeTextFromNullString(sc.Amount), + pgtypeTextFromNullString(sc.OfferID), + pgtypeTextFromNullString(sc.SignerAccountID), + pgtypeTextFromNullString(sc.SpenderAccountID), + pgtypeTextFromNullString(sc.SponsoredAccountID), + pgtypeTextFromNullString(sc.SponsorAccountID), + pgtypeTextFromNullString(sc.DeployerAccountID), + pgtypeTextFromNullString(sc.FunderAccountID), + jsonbFromMap(sc.SignerWeights), + jsonbFromMap(sc.Thresholds), + jsonbFromMap(sc.TrustlineLimit), + jsonbFromSlice(sc.Flags), + jsonbFromMap(sc.KeyValue), + }, nil + }), + ) + if err != nil { + m.MetricsService.IncDBQueryError("BatchCopy", "state_changes", utils.GetDBErrorType(err)) + return 0, fmt.Errorf("pgx CopyFrom state_changes: %w", err) + } + if int(copyCount) != len(stateChanges) { + return 0, fmt.Errorf("expected %d rows copied, got %d", len(stateChanges), copyCount) + } + + duration := time.Since(start).Seconds() + m.MetricsService.ObserveDBQueryDuration("BatchCopy", "state_changes", duration) + m.MetricsService.ObserveDBBatchSize("BatchCopy", "state_changes", len(stateChanges)) + m.MetricsService.IncDBQuery("BatchCopy", "state_changes") + + return len(stateChanges), nil +} + // BatchGetByTxHash gets state changes for a single transaction with pagination support. func (m *StateChangeModel) BatchGetByTxHash(ctx context.Context, txHash string, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "state_change_order") diff --git a/internal/data/statechanges_test.go b/internal/data/statechanges_test.go index 37f8bf2e3..4ff375f7e 100644 --- a/internal/data/statechanges_test.go +++ b/internal/data/statechanges_test.go @@ -8,6 +8,7 @@ import ( "time" set "github.com/deckarep/golang-set/v2" + "github.com/jackc/pgx/v5" "github.com/stellar/go/keypair" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -19,6 +20,46 @@ import ( "github.com/stellar/wallet-backend/internal/metrics" ) +// generateTestStateChanges creates n test state changes for benchmarking. +// Populates all fields to provide an upper-bound benchmark. +func generateTestStateChanges(n int, txHash string, accountID string, startToID int64) []types.StateChange { + scs := make([]types.StateChange, n) + now := time.Now() + reason := types.StateChangeReasonCredit + + for i := 0; i < n; i++ { + scs[i] = types.StateChange{ + ToID: startToID + int64(i), + StateChangeOrder: 1, + StateChangeCategory: types.StateChangeCategoryBalance, + StateChangeReason: &reason, + LedgerCreatedAt: now, + LedgerNumber: uint32(i + 1), + AccountID: accountID, + OperationID: int64(i + 1), + TxHash: txHash, + // sql.NullString fields + TokenID: sql.NullString{String: fmt.Sprintf("token_%d", i), Valid: true}, + Amount: sql.NullString{String: fmt.Sprintf("%d", (i+1)*100), Valid: true}, + OfferID: sql.NullString{String: fmt.Sprintf("offer_%d", i), Valid: true}, + SignerAccountID: sql.NullString{String: fmt.Sprintf("GSIGNER%032d", i), Valid: true}, + SpenderAccountID: sql.NullString{String: fmt.Sprintf("GSPENDER%031d", i), Valid: true}, + SponsoredAccountID: sql.NullString{String: fmt.Sprintf("GSPONSORED%028d", i), Valid: true}, + SponsorAccountID: sql.NullString{String: fmt.Sprintf("GSPONSOR%030d", i), Valid: true}, + DeployerAccountID: sql.NullString{String: fmt.Sprintf("GDEPLOYER%029d", i), Valid: true}, + FunderAccountID: sql.NullString{String: fmt.Sprintf("GFUNDER%031d", i), Valid: true}, + // JSONB fields + SignerWeights: types.NullableJSONB{"weight": i + 1, "key": fmt.Sprintf("signer_%d", i)}, + Thresholds: types.NullableJSONB{"low": 1, "med": 2, "high": 3}, + TrustlineLimit: types.NullableJSONB{"limit": fmt.Sprintf("%d", (i+1)*1000)}, + Flags: types.NullableJSON{"auth_required", "auth_revocable"}, + KeyValue: types.NullableJSONB{"key": fmt.Sprintf("data_key_%d", i), "value": fmt.Sprintf("data_value_%d", i)}, + } + } + + return scs +} + func TestStateChangeModel_BatchInsert(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -37,28 +78,30 @@ func TestStateChangeModel_BatchInsert(t *testing.T) { require.NoError(t, err) // Create referenced transactions first + meta1, meta2 := "meta1", "meta2" + envelope1, envelope2 := "envelope1", "envelope2" tx1 := types.Transaction{ Hash: "tx1", ToID: 1, - EnvelopeXDR: "envelope1", + EnvelopeXDR: &envelope1, ResultXDR: "result1", - MetaXDR: "meta1", + MetaXDR: &meta1, LedgerNumber: 1, LedgerCreatedAt: now, } tx2 := types.Transaction{ Hash: "tx2", ToID: 2, - EnvelopeXDR: "envelope2", + EnvelopeXDR: &envelope2, ResultXDR: "result2", - MetaXDR: "meta2", + MetaXDR: &meta2, LedgerNumber: 2, LedgerCreatedAt: now, } sqlxDB, err := dbConnectionPool.SqlxDB(ctx) require.NoError(t, err) txModel := &TransactionModel{DB: dbConnectionPool, MetricsService: metrics.NewMetricsService(sqlxDB)} - _, err = txModel.BatchInsert(ctx, nil, []types.Transaction{tx1, tx2}, map[string]set.Set[string]{ + _, err = txModel.BatchInsert(ctx, nil, []*types.Transaction{&tx1, &tx2}, map[string]set.Set[string]{ tx1.Hash: set.NewSet(kp1.Address()), tx2.Hash: set.NewSet(kp2.Address()), }) @@ -171,6 +214,176 @@ func TestStateChangeModel_BatchInsert(t *testing.T) { } } +func TestStateChangeModel_BatchCopy(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + now := time.Now() + + // Create test accounts + kp1 := keypair.MustRandom() + kp2 := keypair.MustRandom() + const q = "INSERT INTO accounts (stellar_address) SELECT UNNEST(ARRAY[$1, $2])" + _, err = dbConnectionPool.ExecContext(ctx, q, kp1.Address(), kp2.Address()) + require.NoError(t, err) + + // Create referenced transactions first + meta1, meta2 := "meta1", "meta2" + envelope1, envelope2 := "envelope1", "envelope2" + tx1 := types.Transaction{ + Hash: "tx1", + ToID: 1, + EnvelopeXDR: &envelope1, + ResultXDR: "result1", + MetaXDR: &meta1, + LedgerNumber: 1, + LedgerCreatedAt: now, + } + tx2 := types.Transaction{ + Hash: "tx2", + ToID: 2, + EnvelopeXDR: &envelope2, + ResultXDR: "result2", + MetaXDR: &meta2, + LedgerNumber: 2, + LedgerCreatedAt: now, + } + sqlxDB, err := dbConnectionPool.SqlxDB(ctx) + require.NoError(t, err) + txModel := &TransactionModel{DB: dbConnectionPool, MetricsService: metrics.NewMetricsService(sqlxDB)} + _, err = txModel.BatchInsert(ctx, nil, []*types.Transaction{&tx1, &tx2}, map[string]set.Set[string]{ + tx1.Hash: set.NewSet(kp1.Address()), + tx2.Hash: set.NewSet(kp2.Address()), + }) + require.NoError(t, err) + + reason := types.StateChangeReasonAdd + sc1 := types.StateChange{ + ToID: 1, + StateChangeOrder: 1, + StateChangeCategory: types.StateChangeCategoryBalance, + StateChangeReason: &reason, + LedgerCreatedAt: now, + LedgerNumber: 1, + AccountID: kp1.Address(), + OperationID: 123, + TxHash: tx1.Hash, + TokenID: sql.NullString{String: "token1", Valid: true}, + Amount: sql.NullString{String: "100", Valid: true}, + } + sc2 := types.StateChange{ + ToID: 2, + StateChangeOrder: 1, + StateChangeCategory: types.StateChangeCategoryBalance, + StateChangeReason: &reason, + LedgerCreatedAt: now, + LedgerNumber: 2, + AccountID: kp2.Address(), + OperationID: 456, + TxHash: tx2.Hash, + } + // State change with nullable JSONB fields + sc3 := types.StateChange{ + ToID: 3, + StateChangeOrder: 1, + StateChangeCategory: types.StateChangeCategorySigner, + StateChangeReason: nil, + LedgerCreatedAt: now, + LedgerNumber: 3, + AccountID: kp1.Address(), + OperationID: 789, + TxHash: tx1.Hash, + SignerWeights: types.NullableJSONB{"weight": 10}, + Thresholds: types.NullableJSONB{"low": 1, "med": 2, "high": 3}, + } + + testCases := []struct { + name string + stateChanges []types.StateChange + wantCount int + wantErrContains string + }{ + { + name: "🟢successful_insert_multiple", + stateChanges: []types.StateChange{sc1, sc2}, + wantCount: 2, + }, + { + name: "🟢empty_input", + stateChanges: []types.StateChange{}, + wantCount: 0, + }, + { + name: "🟢nullable_fields", + stateChanges: []types.StateChange{sc2}, + wantCount: 1, + }, + { + name: "🟢jsonb_fields", + stateChanges: []types.StateChange{sc3}, + wantCount: 1, + }, + } + + // Create pgx connection for BatchCopy (requires pgx.Tx, not sqlx.Tx) + conn, err := pgx.Connect(ctx, dbt.DSN) + require.NoError(t, err) + defer conn.Close(ctx) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Clear the database before each test + _, err = dbConnectionPool.ExecContext(ctx, "TRUNCATE state_changes CASCADE") + require.NoError(t, err) + + // Create fresh mock for each test case + mockMetricsService := metrics.NewMockMetricsService() + // Only set up metric expectations if we have state changes to insert + if len(tc.stateChanges) > 0 { + mockMetricsService. + On("ObserveDBQueryDuration", "BatchCopy", "state_changes", mock.Anything).Return().Once() + mockMetricsService. + On("ObserveDBBatchSize", "BatchCopy", "state_changes", mock.Anything).Return().Once() + mockMetricsService. + On("IncDBQuery", "BatchCopy", "state_changes").Return().Once() + } + defer mockMetricsService.AssertExpectations(t) + + m := &StateChangeModel{ + DB: dbConnectionPool, + MetricsService: mockMetricsService, + } + + // BatchCopy requires a pgx transaction + pgxTx, err := conn.Begin(ctx) + require.NoError(t, err) + + gotCount, err := m.BatchCopy(ctx, pgxTx, tc.stateChanges) + + if tc.wantErrContains != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErrContains) + pgxTx.Rollback(ctx) + return + } + + require.NoError(t, err) + require.NoError(t, pgxTx.Commit(ctx)) + assert.Equal(t, tc.wantCount, gotCount) + + // Verify from DB + var dbInsertedIDs []string + err = dbConnectionPool.SelectContext(ctx, &dbInsertedIDs, "SELECT CONCAT(to_id, '-', state_change_order) FROM state_changes ORDER BY to_id") + require.NoError(t, err) + assert.Len(t, dbInsertedIDs, tc.wantCount) + }) + } +} + func TestStateChangeModel_BatchGetByAccountAddress(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -885,3 +1098,141 @@ func TestStateChangeModel_BatchGetByTxHash(t *testing.T) { assert.Empty(t, stateChanges) }) } + +func BenchmarkStateChangeModel_BatchInsert(b *testing.B) { + dbt := dbtest.OpenB(b) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + if err != nil { + b.Fatalf("failed to open db connection pool: %v", err) + } + defer dbConnectionPool.Close() + + ctx := context.Background() + sqlxDB, err := dbConnectionPool.SqlxDB(ctx) + if err != nil { + b.Fatalf("failed to get sqlx db: %v", err) + } + metricsService := metrics.NewMetricsService(sqlxDB) + + m := &StateChangeModel{ + DB: dbConnectionPool, + MetricsService: metricsService, + } + + // Create a parent transaction that state changes will reference + const txHash = "benchmark_tx_hash" + accountID := keypair.MustRandom().Address() + now := time.Now() + _, err = dbConnectionPool.ExecContext(ctx, ` + INSERT INTO transactions (hash, to_id, envelope_xdr, result_xdr, meta_xdr, ledger_number, ledger_created_at) + VALUES ($1, 1, 'env', 'res', 'meta', 1, $2) + `, txHash, now) + if err != nil { + b.Fatalf("failed to create parent transaction: %v", err) + } + + batchSizes := []int{1000, 5000, 10000, 50000, 100000} + + for _, size := range batchSizes { + b.Run(fmt.Sprintf("size=%d", size), func(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + b.StopTimer() + // Clean up state changes before each iteration (keep the parent transaction) + //nolint:errcheck // truncate is best-effort cleanup in benchmarks + dbConnectionPool.ExecContext(ctx, "TRUNCATE state_changes CASCADE") + // Generate fresh test data for each iteration + scs := generateTestStateChanges(size, txHash, accountID, int64(i*size)) + b.StartTimer() + + _, err := m.BatchInsert(ctx, nil, scs) + if err != nil { + b.Fatalf("BatchInsert failed: %v", err) + } + } + }) + } +} + +// BenchmarkStateChangeModel_BatchCopy benchmarks bulk insert using pgx's binary COPY protocol. +func BenchmarkStateChangeModel_BatchCopy(b *testing.B) { + dbt := dbtest.OpenB(b) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + if err != nil { + b.Fatalf("failed to open db connection pool: %v", err) + } + defer dbConnectionPool.Close() + + ctx := context.Background() + sqlxDB, err := dbConnectionPool.SqlxDB(ctx) + if err != nil { + b.Fatalf("failed to get sqlx db: %v", err) + } + metricsService := metrics.NewMetricsService(sqlxDB) + + m := &StateChangeModel{ + DB: dbConnectionPool, + MetricsService: metricsService, + } + + // Create pgx connection for BatchCopy + conn, err := pgx.Connect(ctx, dbt.DSN) + if err != nil { + b.Fatalf("failed to connect with pgx: %v", err) + } + defer conn.Close(ctx) + + // Create a parent transaction that state changes will reference + const txHash = "benchmark_tx_hash" + accountID := keypair.MustRandom().Address() + now := time.Now() + _, err = conn.Exec(ctx, ` + INSERT INTO transactions (hash, to_id, envelope_xdr, result_xdr, meta_xdr, ledger_number, ledger_created_at) + VALUES ($1, 1, 'env', 'res', 'meta', 1, $2) + `, txHash, now) + if err != nil { + b.Fatalf("failed to create parent transaction: %v", err) + } + + batchSizes := []int{1000, 5000, 10000, 50000, 100000} + + for _, size := range batchSizes { + b.Run(fmt.Sprintf("size=%d", size), func(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + b.StopTimer() + // Clean up state changes before each iteration (keep the parent transaction) + _, err = conn.Exec(ctx, "TRUNCATE state_changes CASCADE") + if err != nil { + b.Fatalf("failed to truncate: %v", err) + } + + // Generate fresh test data for each iteration + scs := generateTestStateChanges(size, txHash, accountID, int64(i*size)) + + // Start a pgx transaction + pgxTx, err := conn.Begin(ctx) + if err != nil { + b.Fatalf("failed to begin transaction: %v", err) + } + b.StartTimer() + + _, err = m.BatchCopy(ctx, pgxTx, scs) + if err != nil { + pgxTx.Rollback(ctx) + b.Fatalf("BatchCopy failed: %v", err) + } + + b.StopTimer() + if err := pgxTx.Commit(ctx); err != nil { + b.Fatalf("failed to commit transaction: %v", err) + } + b.StartTimer() + } + }) + } +} diff --git a/internal/data/transactions.go b/internal/data/transactions.go index 98ed7f288..036ec313b 100644 --- a/internal/data/transactions.go +++ b/internal/data/transactions.go @@ -7,6 +7,8 @@ import ( "time" set "github.com/deckarep/golang-set/v2" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" "github.com/lib/pq" "github.com/stellar/wallet-backend/internal/db" @@ -167,7 +169,7 @@ func (m *TransactionModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs func (m *TransactionModel) BatchInsert( ctx context.Context, sqlExecuter db.SQLExecuter, - txs []types.Transaction, + txs []*types.Transaction, stellarAddressesByTxHash map[string]set.Set[string], ) ([]string, error) { if sqlExecuter == nil { @@ -177,9 +179,9 @@ func (m *TransactionModel) BatchInsert( // 1. Flatten the transactions into parallel slices hashes := make([]string, len(txs)) toIDs := make([]int64, len(txs)) - envelopeXDRs := make([]string, len(txs)) + envelopeXDRs := make([]*string, len(txs)) resultXDRs := make([]string, len(txs)) - metaXDRs := make([]string, len(txs)) + metaXDRs := make([]*string, len(txs)) ledgerNumbers := make([]int, len(txs)) ledgerCreatedAts := make([]time.Time, len(txs)) @@ -275,3 +277,84 @@ func (m *TransactionModel) BatchInsert( return insertedHashes, nil } + +// BatchCopy inserts transactions using pgx's binary COPY protocol. +// Uses pgx.Tx for binary format which is faster than lib/pq's text format. +// Uses native pgtype types for optimal performance (see https://github.com/jackc/pgx/issues/763). +func (m *TransactionModel) BatchCopy( + ctx context.Context, + pgxTx pgx.Tx, + txs []*types.Transaction, + stellarAddressesByTxHash map[string]set.Set[string], +) (int, error) { + if len(txs) == 0 { + return 0, nil + } + + start := time.Now() + + // COPY transactions using pgx binary format with native pgtype types + copyCount, err := pgxTx.CopyFrom( + ctx, + pgx.Identifier{"transactions"}, + []string{"hash", "to_id", "envelope_xdr", "result_xdr", "meta_xdr", "ledger_number", "ledger_created_at"}, + pgx.CopyFromSlice(len(txs), func(i int) ([]any, error) { + tx := txs[i] + return []any{ + pgtype.Text{String: tx.Hash, Valid: true}, + pgtype.Int8{Int64: tx.ToID, Valid: true}, + pgtypeTextFromPtr(tx.EnvelopeXDR), + pgtype.Text{String: tx.ResultXDR, Valid: true}, + pgtypeTextFromPtr(tx.MetaXDR), + pgtype.Int4{Int32: int32(tx.LedgerNumber), Valid: true}, + pgtype.Timestamptz{Time: tx.LedgerCreatedAt, Valid: true}, + }, nil + }), + ) + if err != nil { + m.MetricsService.IncDBQueryError("BatchCopy", "transactions", utils.GetDBErrorType(err)) + return 0, fmt.Errorf("pgx CopyFrom transactions: %w", err) + } + if int(copyCount) != len(txs) { + return 0, fmt.Errorf("expected %d rows copied, got %d", len(txs), copyCount) + } + + // COPY transactions_accounts using pgx binary format with native pgtype types + if len(stellarAddressesByTxHash) > 0 { + var taRows [][]any + for txHash, addresses := range stellarAddressesByTxHash { + txHashPgtype := pgtype.Text{String: txHash, Valid: true} + for _, addr := range addresses.ToSlice() { + taRows = append(taRows, []any{txHashPgtype, pgtype.Text{String: addr, Valid: true}}) + } + } + + _, err = pgxTx.CopyFrom( + ctx, + pgx.Identifier{"transactions_accounts"}, + []string{"tx_hash", "account_id"}, + pgx.CopyFromRows(taRows), + ) + if err != nil { + m.MetricsService.IncDBQueryError("BatchCopy", "transactions_accounts", utils.GetDBErrorType(err)) + return 0, fmt.Errorf("pgx CopyFrom transactions_accounts: %w", err) + } + + m.MetricsService.IncDBQuery("BatchCopy", "transactions_accounts") + } + + duration := time.Since(start).Seconds() + m.MetricsService.ObserveDBQueryDuration("BatchCopy", "transactions", duration) + m.MetricsService.ObserveDBBatchSize("BatchCopy", "transactions", len(txs)) + m.MetricsService.IncDBQuery("BatchCopy", "transactions") + + return len(txs), nil +} + +// pgtypeTextFromPtr converts a *string to pgtype.Text for efficient binary COPY. +func pgtypeTextFromPtr(s *string) pgtype.Text { + if s == nil { + return pgtype.Text{Valid: false} + } + return pgtype.Text{String: *s, Valid: true} +} diff --git a/internal/data/transactions_test.go b/internal/data/transactions_test.go index 9c285d1f1..81f31fd40 100644 --- a/internal/data/transactions_test.go +++ b/internal/data/transactions_test.go @@ -2,10 +2,12 @@ package data import ( "context" + "fmt" "testing" "time" set "github.com/deckarep/golang-set/v2" + "github.com/jackc/pgx/v5" "github.com/stellar/go/keypair" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -17,6 +19,34 @@ import ( "github.com/stellar/wallet-backend/internal/metrics" ) +// generateTestTransactions creates n test transactions for benchmarking. +func generateTestTransactions(n int, startToID int64) ([]*types.Transaction, map[string]set.Set[string]) { + txs := make([]*types.Transaction, n) + addressesByHash := make(map[string]set.Set[string]) + now := time.Now() + + for i := 0; i < n; i++ { + hash := fmt.Sprintf("e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760-%d", startToID+int64(i)) + envelope := "AAAAAgAAAAB/NpQ+s+cP+ztX7ryuKgXrxowZPHd4qAxhseOye/JeUgAehIAC2NL/AAflugAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAwAAAAFQQUxMAAAAAKHc4IKbcW8HPPgy3zOhuqv851y72nfLGa0HVXxIRNzHAAAAAAAAAAAAQ3FwMxshxQAfwV8AAAAAYTGQ3QAAAAAAAAAMAAAAAAAAAAFQQUxMAAAAAKHc4IKbcW8HPPgy3zOhuqv851y72nfLGa0HVXxIRNzHAAAAAAAGXwFksiHwAEXz8QAAAABhoaQjAAAAAAAAAAF78l5SAAAAQD7LgvZA8Pdvfh5L2b9B9RC7DlacGBJuOchuZDHQdVD1P0bn6nGQJXxDDI4oN76J49JxB7bIgDVim39MU43MOgE=" + meta := "AAAAAwAAAAAAAAAEAAAAAwM6nhwAAAAAAAAAAJjy0MY1CPlZ/co80nzufVmo4gd7NqWMb+RiGiPhiviJAAAAC4SozKUDMWgAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAQM6nhwAAAAAAAAAAJjy0MY1CPlZ/co80nzufVmo4gd7NqWMb+RiGiPhiviJAAAAC4SozKUDMWgAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAwM6LTkAAAAAAAAAAKl6DQcpepRdTbO/Vw4hYBENfE/95GevM7SNA0ftK0gtAAAAA8Kuf0AC+zZCAAAATAAAAAMAAAABAAAAAMRxxkNwYslQaok0LlOKGtpATS9Bzx06JV9DIffG4OF1AAAAAAAAAAlsb2JzdHIuY28AAAABAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAMAAAAAAyZ54QAAAABmrTXCAAAAAAAAAAEDOp4cAAAAAAAAAACpeg0HKXqUXU2zv1cOIWARDXxP/eRnrzO0jQNH7StILQAAAAPCrn9AAvs2QgAAAE0AAAADAAAAAQAAAADEccZDcGLJUGqJNC5TihraQE0vQc8dOiVfQyH3xuDhdQAAAAAAAAAJbG9ic3RyLmNvAAAAAQAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAADAAAAAAM6nhwAAAAAZyGdHwAAAAAAAAABAAAABAAAAAMDOp4cAAAAAAAAAACpeg0HKXqUXU2zv1cOIWARDXxP/eRnrzO0jQNH7StILQAAAAPCrn9AAvs2QgAAAE0AAAADAAAAAQAAAADEccZDcGLJUGqJNC5TihraQE0vQc8dOiVfQyH3xuDhdQAAAAAAAAAJbG9ic3RyLmNvAAAAAQAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAADAAAAAAM6nhwAAAAAZyGdHwAAAAAAAAABAzqeHAAAAAAAAAAAqXoNByl6lF1Ns79XDiFgEQ18T/3kZ68ztI0DR+0rSC0AAAACmKiNQAL7NkIAAABNAAAAAwAAAAEAAAAAxHHGQ3BiyVBqiTQuU4oa2kBNL0HPHTolX0Mh98bg4XUAAAAAAAAACWxvYnN0ci5jbwAAAAEAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAwAAAAADOp4cAAAAAGchnR8AAAAAAAAAAwM6nZoAAAAAAAAAALKxMozkOH3rgpz3/u3+93wsR4p6z4K82HmJ5NTuaZbYAAACZaqAwoIBqycyAABVlQAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAMAAAAAAzqSNgAAAABnIVdaAAAAAAAAAAEDOp4cAAAAAAAAAACysTKM5Dh964Kc9/7t/vd8LEeKes+CvNh5ieTU7mmW2AAAAmbUhrSCAasnMgAAVZUAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAADAAAAAAM6kjYAAAAAZyFXWgAAAAAAAAAAAAAAAA==" + result := "AAAAAAAAAMgAAAAAAAAAAgAAAAAAAAADAAAAAAAAAAAAAAABAAAAAH82lD6z5w/7O1fuvK4qBevGjBk8d3ioDGGx47J78l5SAAAAAGExkN0AAAABUEFMTAAAAACh3OCCm3FvBzz4Mt8zobqr/Odcu9p3yxmtB1V8SETcxwAAAAAAAAAAAENxcDMbIcUAH8FfAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAQAAAAB/NpQ+s+cP+ztX7ryuKgXrxowZPHd4qAxhseOye/JeUgAAAABhoaQjAAAAAAAAAAFQQUxMAAAAAKHc4IKbcW8HPPgy3zOhuqv851y72nfLGa0HVXxIRNzHAAAAAAkrzGAARfPxZLIh8AAAAAAAAAAAAAAAAA==" + address := keypair.MustRandom().Address() + + txs[i] = &types.Transaction{ + Hash: hash, + ToID: startToID + int64(i), + EnvelopeXDR: &envelope, + ResultXDR: result, + MetaXDR: &meta, + LedgerNumber: uint32(i + 1), + LedgerCreatedAt: now, + } + addressesByHash[hash] = set.NewSet(address) + } + + return txs, addressesByHash +} + func Test_TransactionModel_BatchInsert(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -34,21 +64,23 @@ func Test_TransactionModel_BatchInsert(t *testing.T) { _, err = dbConnectionPool.ExecContext(ctx, q, kp1.Address(), kp2.Address()) require.NoError(t, err) + meta1, meta2 := "meta1", "meta2" + envelope1, envelope2 := "envelope1", "envelope2" tx1 := types.Transaction{ Hash: "tx1", ToID: 1, - EnvelopeXDR: "envelope1", + EnvelopeXDR: &envelope1, ResultXDR: "result1", - MetaXDR: "meta1", + MetaXDR: &meta1, LedgerNumber: 1, LedgerCreatedAt: now, } tx2 := types.Transaction{ Hash: "tx2", ToID: 2, - EnvelopeXDR: "envelope2", + EnvelopeXDR: &envelope2, ResultXDR: "result2", - MetaXDR: "meta2", + MetaXDR: &meta2, LedgerNumber: 2, LedgerCreatedAt: now, } @@ -56,7 +88,7 @@ func Test_TransactionModel_BatchInsert(t *testing.T) { testCases := []struct { name string useDBTx bool - txs []types.Transaction + txs []*types.Transaction stellarAddressesByHash map[string]set.Set[string] wantAccountLinks map[string][]string wantErrContains string @@ -65,7 +97,7 @@ func Test_TransactionModel_BatchInsert(t *testing.T) { { name: "🟢successful_insert_without_dbTx", useDBTx: false, - txs: []types.Transaction{tx1, tx2}, + txs: []*types.Transaction{&tx1, &tx2}, stellarAddressesByHash: map[string]set.Set[string]{tx1.Hash: set.NewSet(kp1.Address()), tx2.Hash: set.NewSet(kp2.Address())}, wantAccountLinks: map[string][]string{tx1.Hash: {kp1.Address()}, tx2.Hash: {kp2.Address()}}, wantErrContains: "", @@ -74,7 +106,7 @@ func Test_TransactionModel_BatchInsert(t *testing.T) { { name: "🟢successful_insert_with_dbTx", useDBTx: true, - txs: []types.Transaction{tx1}, + txs: []*types.Transaction{&tx1}, stellarAddressesByHash: map[string]set.Set[string]{tx1.Hash: set.NewSet(kp1.Address())}, wantAccountLinks: map[string][]string{tx1.Hash: {kp1.Address()}}, wantErrContains: "", @@ -83,7 +115,7 @@ func Test_TransactionModel_BatchInsert(t *testing.T) { { name: "🟢empty_input", useDBTx: false, - txs: []types.Transaction{}, + txs: []*types.Transaction{}, stellarAddressesByHash: map[string]set.Set[string]{}, wantAccountLinks: map[string][]string{}, wantErrContains: "", @@ -92,7 +124,7 @@ func Test_TransactionModel_BatchInsert(t *testing.T) { { name: "🟡duplicate_transaction", useDBTx: false, - txs: []types.Transaction{tx1, tx1}, + txs: []*types.Transaction{&tx1, &tx1}, stellarAddressesByHash: map[string]set.Set[string]{tx1.Hash: set.NewSet(kp1.Address())}, wantAccountLinks: map[string][]string{tx1.Hash: {kp1.Address()}}, wantErrContains: "", @@ -176,6 +208,168 @@ func Test_TransactionModel_BatchInsert(t *testing.T) { } } +func Test_TransactionModel_BatchCopy(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + now := time.Now() + + // Create test accounts + kp1 := keypair.MustRandom() + kp2 := keypair.MustRandom() + const q = "INSERT INTO accounts (stellar_address) SELECT UNNEST(ARRAY[$1, $2])" + _, err = dbConnectionPool.ExecContext(ctx, q, kp1.Address(), kp2.Address()) + require.NoError(t, err) + + meta1, meta2 := "meta1", "meta2" + envelope1, envelope2 := "envelope1", "envelope2" + tx1 := types.Transaction{ + Hash: "tx1", + ToID: 1, + EnvelopeXDR: &envelope1, + ResultXDR: "result1", + MetaXDR: &meta1, + LedgerNumber: 1, + LedgerCreatedAt: now, + } + tx2 := types.Transaction{ + Hash: "tx2", + ToID: 2, + EnvelopeXDR: &envelope2, + ResultXDR: "result2", + MetaXDR: &meta2, + LedgerNumber: 2, + LedgerCreatedAt: now, + } + // Transaction with nullable fields (nil envelope and meta) + tx3 := types.Transaction{ + Hash: "tx3", + ToID: 3, + EnvelopeXDR: nil, + ResultXDR: "result3", + MetaXDR: nil, + LedgerNumber: 3, + LedgerCreatedAt: now, + } + + testCases := []struct { + name string + txs []*types.Transaction + stellarAddressesByHash map[string]set.Set[string] + wantCount int + wantErrContains string + }{ + { + name: "🟢successful_insert_multiple", + txs: []*types.Transaction{&tx1, &tx2}, + stellarAddressesByHash: map[string]set.Set[string]{tx1.Hash: set.NewSet(kp1.Address()), tx2.Hash: set.NewSet(kp2.Address())}, + wantCount: 2, + }, + { + name: "🟢empty_input", + txs: []*types.Transaction{}, + stellarAddressesByHash: map[string]set.Set[string]{}, + wantCount: 0, + }, + { + name: "🟢nullable_fields", + txs: []*types.Transaction{&tx3}, + stellarAddressesByHash: map[string]set.Set[string]{tx3.Hash: set.NewSet(kp1.Address())}, + wantCount: 1, + }, + { + name: "🟢no_participants", + txs: []*types.Transaction{&tx1}, + stellarAddressesByHash: map[string]set.Set[string]{}, + wantCount: 1, + }, + } + + // Create pgx connection for BatchCopy (requires pgx.Tx, not sqlx.Tx) + conn, err := pgx.Connect(ctx, dbt.DSN) + require.NoError(t, err) + defer conn.Close(ctx) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Clear the database before each test + _, err = dbConnectionPool.ExecContext(ctx, "TRUNCATE transactions, transactions_accounts CASCADE") + require.NoError(t, err) + + // Create fresh mock for each test case + mockMetricsService := metrics.NewMockMetricsService() + // Only set up metric expectations if we have transactions to insert + if len(tc.txs) > 0 { + mockMetricsService. + On("ObserveDBQueryDuration", "BatchCopy", "transactions", mock.Anything).Return().Once() + mockMetricsService. + On("ObserveDBBatchSize", "BatchCopy", "transactions", mock.Anything).Return().Once() + mockMetricsService. + On("IncDBQuery", "BatchCopy", "transactions").Return().Once() + if len(tc.stellarAddressesByHash) > 0 { + mockMetricsService. + On("IncDBQuery", "BatchCopy", "transactions_accounts").Return().Once() + } + } + defer mockMetricsService.AssertExpectations(t) + + m := &TransactionModel{ + DB: dbConnectionPool, + MetricsService: mockMetricsService, + } + + // BatchCopy requires a pgx transaction + pgxTx, err := conn.Begin(ctx) + require.NoError(t, err) + + gotCount, err := m.BatchCopy(ctx, pgxTx, tc.txs, tc.stellarAddressesByHash) + + if tc.wantErrContains != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErrContains) + pgxTx.Rollback(ctx) + return + } + + require.NoError(t, err) + require.NoError(t, pgxTx.Commit(ctx)) + assert.Equal(t, tc.wantCount, gotCount) + + // Verify from DB + var dbInsertedHashes []string + err = dbConnectionPool.SelectContext(ctx, &dbInsertedHashes, "SELECT hash FROM transactions ORDER BY hash") + require.NoError(t, err) + assert.Len(t, dbInsertedHashes, tc.wantCount) + + // Verify account links if expected + if len(tc.stellarAddressesByHash) > 0 && tc.wantCount > 0 { + var accountLinks []struct { + TxHash string `db:"tx_hash"` + AccountID string `db:"account_id"` + } + err = dbConnectionPool.SelectContext(ctx, &accountLinks, "SELECT tx_hash, account_id FROM transactions_accounts ORDER BY tx_hash, account_id") + require.NoError(t, err) + + // Create a map of tx_hash -> set of account_ids + accountLinksMap := make(map[string][]string) + for _, link := range accountLinks { + accountLinksMap[link.TxHash] = append(accountLinksMap[link.TxHash], link.AccountID) + } + + // Verify each expected transaction has its account links + for txHash, expectedAddresses := range tc.stellarAddressesByHash { + actualAddresses := accountLinksMap[txHash] + assert.ElementsMatch(t, expectedAddresses.ToSlice(), actualAddresses, "account links for tx %s don't match", txHash) + } + } + }) + } +} + func TestTransactionModel_GetByHash(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -209,7 +403,8 @@ func TestTransactionModel_GetByHash(t *testing.T) { require.NoError(t, err) assert.Equal(t, txHash, transaction.Hash) assert.Equal(t, int64(1), transaction.ToID) - assert.Equal(t, "envelope", transaction.EnvelopeXDR) + require.NotNil(t, transaction.EnvelopeXDR) + assert.Equal(t, "envelope", *transaction.EnvelopeXDR) } func TestTransactionModel_GetAll(t *testing.T) { @@ -430,3 +625,117 @@ func TestTransactionModel_BatchGetByStateChangeIDs(t *testing.T) { assert.Equal(t, "tx2", stateChangeIDsFound["2-1"]) assert.Equal(t, "tx1", stateChangeIDsFound["3-1"]) } + +func BenchmarkTransactionModel_BatchInsert(b *testing.B) { + dbt := dbtest.OpenB(b) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + if err != nil { + b.Fatalf("failed to open db connection pool: %v", err) + } + defer dbConnectionPool.Close() + + ctx := context.Background() + sqlxDB, err := dbConnectionPool.SqlxDB(ctx) + if err != nil { + b.Fatalf("failed to get sqlx db: %v", err) + } + metricsService := metrics.NewMetricsService(sqlxDB) + + m := &TransactionModel{ + DB: dbConnectionPool, + MetricsService: metricsService, + } + + batchSizes := []int{1000, 5000, 10000, 50000, 100000} + + for _, size := range batchSizes { + b.Run(fmt.Sprintf("size=%d", size), func(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + b.StopTimer() + // Clean up before each iteration + //nolint:errcheck // truncate is best-effort cleanup in benchmarks + dbConnectionPool.ExecContext(ctx, "TRUNCATE transactions, transactions_accounts CASCADE") + // Generate fresh test data for each iteration + txs, addressesByHash := generateTestTransactions(size, int64(i*size)) + b.StartTimer() + + _, err := m.BatchInsert(ctx, nil, txs, addressesByHash) + if err != nil { + b.Fatalf("BatchInsert failed: %v", err) + } + } + }) + } +} + +// BenchmarkTransactionModel_BatchCopy benchmarks bulk insert using pgx's binary COPY protocol. +func BenchmarkTransactionModel_BatchCopy(b *testing.B) { + dbt := dbtest.OpenB(b) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + if err != nil { + b.Fatalf("failed to open db connection pool: %v", err) + } + defer dbConnectionPool.Close() + + ctx := context.Background() + sqlxDB, err := dbConnectionPool.SqlxDB(ctx) + if err != nil { + b.Fatalf("failed to get sqlx db: %v", err) + } + metricsService := metrics.NewMetricsService(sqlxDB) + + m := &TransactionModel{ + DB: dbConnectionPool, + MetricsService: metricsService, + } + + // Create pgx connection for BatchCopy + conn, err := pgx.Connect(ctx, dbt.DSN) + if err != nil { + b.Fatalf("failed to connect with pgx: %v", err) + } + defer conn.Close(ctx) + + batchSizes := []int{1000, 5000, 10000, 50000, 100000} + + for _, size := range batchSizes { + b.Run(fmt.Sprintf("size=%d", size), func(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + b.StopTimer() + // Clean up before each iteration + _, err = conn.Exec(ctx, "TRUNCATE transactions, transactions_accounts CASCADE") + if err != nil { + b.Fatalf("failed to truncate: %v", err) + } + + // Generate fresh test data for each iteration + txs, addressesByHash := generateTestTransactions(size, int64(i*size)) + + // Start a pgx transaction + pgxTx, err := conn.Begin(ctx) + if err != nil { + b.Fatalf("failed to begin transaction: %v", err) + } + b.StartTimer() + + _, err = m.BatchCopy(ctx, pgxTx, txs, addressesByHash) + if err != nil { + pgxTx.Rollback(ctx) + b.Fatalf("BatchCopy failed: %v", err) + } + + b.StopTimer() + if err := pgxTx.Commit(ctx); err != nil { + b.Fatalf("failed to commit transaction: %v", err) + } + b.StartTimer() + } + }) + } +} diff --git a/internal/db/db.go b/internal/db/db.go index 45c97d147..8874dd3c4 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -4,8 +4,10 @@ import ( "context" "database/sql" "fmt" + "strings" "time" + "github.com/jackc/pgx/v5/pgxpool" "github.com/jmoiron/sqlx" "github.com/stellar/go/support/log" ) @@ -17,6 +19,7 @@ type ConnectionPool interface { Ping(ctx context.Context) error SqlDB(ctx context.Context) (*sql.DB, error) SqlxDB(ctx context.Context) (*sqlx.DB, error) + PgxPool() *pgxpool.Pool } // Make sure *DBConnectionPoolImplementation implements DBConnectionPool: @@ -24,11 +27,14 @@ var _ ConnectionPool = (*ConnectionPoolImplementation)(nil) type ConnectionPoolImplementation struct { *sqlx.DB + pgxPool *pgxpool.Pool } const ( MaxDBConnIdleTime = 10 * time.Second MaxOpenDBConns = 30 + MaxIdleDBConns = 20 // Keep warm connections ready in the pool + MaxDBConnLifetime = 5 * time.Minute // Recycle connections periodically ) func OpenDBConnectionPool(dataSourceName string) (ConnectionPool, error) { @@ -38,13 +44,72 @@ func OpenDBConnectionPool(dataSourceName string) (ConnectionPool, error) { } sqlxDB.SetConnMaxIdleTime(MaxDBConnIdleTime) sqlxDB.SetMaxOpenConns(MaxOpenDBConns) + sqlxDB.SetMaxIdleConns(MaxIdleDBConns) + sqlxDB.SetConnMaxLifetime(MaxDBConnLifetime) err = sqlxDB.Ping() if err != nil { return nil, fmt.Errorf("error pinging app DB connection pool: %w", err) } - return &ConnectionPoolImplementation{DB: sqlxDB}, nil + // Create pgx pool for binary COPY operations + pgxPool, err := pgxpool.New(context.Background(), dataSourceName) + if err != nil { + _ = sqlxDB.Close() //nolint:errcheck // Best effort cleanup; primary error is pgx pool creation + return nil, fmt.Errorf("error creating pgx pool: %w", err) + } + + return &ConnectionPoolImplementation{DB: sqlxDB, pgxPool: pgxPool}, nil +} + +// OpenDBConnectionPoolForBackfill creates a connection pool optimized for bulk insert operations. +// It configures session-level settings (synchronous_commit=off, work_mem=256MB) via the connection +// string, which are applied to every new connection in the pool. +// This should ONLY be used for backfill instances, NOT for live ingestion. +func OpenDBConnectionPoolForBackfill(dataSourceName string) (ConnectionPool, error) { + // Append session parameters to connection string for automatic configuration. + // URL-encoded: -c synchronous_commit=off -c work_mem=256MB + backfillParams := "options=-c%20synchronous_commit%3Doff%20-c%20work_mem%3D256MB" + + separator := "?" + if strings.Contains(dataSourceName, "?") { + separator = "&" + } + backfillDSN := dataSourceName + separator + backfillParams + + sqlxDB, err := sqlx.Open("postgres", backfillDSN) + if err != nil { + return nil, fmt.Errorf("error creating backfill DB connection pool: %w", err) + } + sqlxDB.SetConnMaxIdleTime(MaxDBConnIdleTime) + sqlxDB.SetMaxOpenConns(MaxOpenDBConns) + sqlxDB.SetMaxIdleConns(MaxIdleDBConns) + sqlxDB.SetConnMaxLifetime(MaxDBConnLifetime) + + err = sqlxDB.Ping() + if err != nil { + return nil, fmt.Errorf("error pinging backfill DB connection pool: %w", err) + } + + // Create pgx pool for binary COPY operations with backfill settings + pgxPool, err := pgxpool.New(context.Background(), backfillDSN) + if err != nil { + _ = sqlxDB.Close() //nolint:errcheck // Best effort cleanup; primary error is pgx pool creation + return nil, fmt.Errorf("error creating pgx pool for backfill: %w", err) + } + + return &ConnectionPoolImplementation{DB: sqlxDB, pgxPool: pgxPool}, nil +} + +// ConfigureBackfillSession sets session_replication_role to 'replica' which disables FK constraint +// checking. This cannot be set via connection string and requires elevated privileges (superuser +// or replication role). Call this ONCE at backfill startup after creating the connection pool. +func ConfigureBackfillSession(ctx context.Context, db ConnectionPool) error { + _, err := db.ExecContext(ctx, "SET session_replication_role = 'replica'") + if err != nil { + return fmt.Errorf("setting session_replication_role: %w", err) + } + return nil } //nolint:wrapcheck // this is a thin layer on top of the sqlx.DB.BeginTxx method @@ -65,6 +130,20 @@ func (db *ConnectionPoolImplementation) SqlxDB(ctx context.Context) (*sqlx.DB, e return db.DB, nil } +func (db *ConnectionPoolImplementation) PgxPool() *pgxpool.Pool { + return db.pgxPool +} + +// Close closes both the sqlx and pgx pools. +// +//nolint:wrapcheck // this is a thin layer on top of the sqlx.DB.Close method +func (db *ConnectionPoolImplementation) Close() error { + if db.pgxPool != nil { + db.pgxPool.Close() + } + return db.DB.Close() +} + // Transaction is an interface that wraps the sqlx.Tx structs methods. type Transaction interface { SQLExecuter diff --git a/internal/db/dbtest/dbtest.go b/internal/db/dbtest/dbtest.go index 45650ab83..f6b480cfd 100644 --- a/internal/db/dbtest/dbtest.go +++ b/internal/db/dbtest/dbtest.go @@ -30,3 +30,19 @@ func OpenWithoutMigrations(t *testing.T) *dbtest.DB { db := dbtest.Postgres(t) return db } + +// OpenB opens a test database for benchmarks with migrations applied. +func OpenB(b *testing.B) *dbtest.DB { + db := dbtest.Postgres(b) + conn := db.Open() + defer conn.Close() + + migrateDirection := schema.MigrateUp + m := migrate.HttpFileSystemMigrationSource{FileSystem: http.FS(migrations.FS)} + _, err := schema.Migrate(conn.DB, m, migrateDirection, 0) + if err != nil { + b.Fatal(err) + } + + return db +} diff --git a/internal/db/migrations/2025-06-10.2-create_indexer_table_transactions.sql b/internal/db/migrations/2025-06-10.2-create_indexer_table_transactions.sql index 62c6dfa74..eecf7e0f8 100644 --- a/internal/db/migrations/2025-06-10.2-create_indexer_table_transactions.sql +++ b/internal/db/migrations/2025-06-10.2-create_indexer_table_transactions.sql @@ -17,7 +17,7 @@ CREATE INDEX idx_transactions_ledger_created_at ON transactions(ledger_created_a -- Table: transactions_accounts CREATE TABLE transactions_accounts ( tx_hash TEXT NOT NULL REFERENCES transactions(hash) ON DELETE CASCADE, - account_id TEXT NOT NULL REFERENCES accounts(stellar_address) ON DELETE CASCADE, + account_id TEXT NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), PRIMARY KEY (account_id, tx_hash) ); diff --git a/internal/db/migrations/2025-06-10.3-create_indexer_table_operations.sql b/internal/db/migrations/2025-06-10.3-create_indexer_table_operations.sql index ffd097dd0..3110e17de 100644 --- a/internal/db/migrations/2025-06-10.3-create_indexer_table_operations.sql +++ b/internal/db/migrations/2025-06-10.3-create_indexer_table_operations.sql @@ -18,7 +18,7 @@ CREATE INDEX idx_operations_ledger_created_at ON operations(ledger_created_at); -- Table: operations_accounts CREATE TABLE operations_accounts ( operation_id BIGINT NOT NULL REFERENCES operations(id) ON DELETE CASCADE, - account_id TEXT NOT NULL REFERENCES accounts(stellar_address) ON DELETE CASCADE, + account_id TEXT NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), PRIMARY KEY (account_id, operation_id) ); diff --git a/internal/db/migrations/2025-06-10.4-create_indexer_table_state_changes.sql b/internal/db/migrations/2025-06-10.4-create_indexer_table_state_changes.sql index d5fb68bea..e14b478d2 100644 --- a/internal/db/migrations/2025-06-10.4-create_indexer_table_state_changes.sql +++ b/internal/db/migrations/2025-06-10.4-create_indexer_table_state_changes.sql @@ -9,7 +9,7 @@ CREATE TABLE state_changes ( ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), ledger_created_at TIMESTAMPTZ NOT NULL, ledger_number INTEGER NOT NULL, - account_id TEXT NOT NULL REFERENCES accounts(stellar_address), + account_id TEXT NOT NULL, operation_id BIGINT NOT NULL, tx_hash TEXT NOT NULL REFERENCES transactions(hash), token_id TEXT, diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 5ba2fe8c8..e41c78f36 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -30,16 +30,18 @@ type IndexerBufferInterface interface { PushStateChange(transaction types.Transaction, operation types.Operation, stateChange types.StateChange) GetTransactionsParticipants() map[string]set.Set[string] GetOperationsParticipants() map[int64]set.Set[string] + GetAllParticipants() []string GetNumberOfTransactions() int GetNumberOfOperations() int - GetTransactions() []types.Transaction - GetOperations() []types.Operation + GetTransactions() []*types.Transaction + GetOperations() []*types.Operation GetStateChanges() []types.StateChange GetTrustlineChanges() []types.TrustlineChange GetContractChanges() []types.ContractChange PushContractChange(contractChange types.ContractChange) PushTrustlineChange(trustlineChange types.TrustlineChange) - MergeBuffer(other IndexerBufferInterface) + Merge(other IndexerBufferInterface) + Clear() } type TokenTransferProcessorInterface interface { @@ -56,31 +58,18 @@ type OperationProcessorInterface interface { Name() string } -type AccountModelInterface interface { - BatchGetByIDs(ctx context.Context, accountIDs []string) ([]string, error) -} - -// PrecomputedTransactionData holds all the data needed to process a transaction -// without re-computing participants, operations participants, and state changes. -type PrecomputedTransactionData struct { - Transaction ingest.LedgerTransaction - TxParticipants set.Set[string] - OpsParticipants map[int64]processors.OperationParticipants - StateChanges []types.StateChange - TrustlineChanges []types.TrustlineChange - ContractChanges []types.ContractChange - AllParticipants set.Set[string] // Union of all participants for this transaction -} - type Indexer struct { participantsProcessor ParticipantsProcessorInterface tokenTransferProcessor TokenTransferProcessorInterface processors []OperationProcessorInterface pool pond.Pool metricsService processors.MetricsServiceInterface + skipTxMeta bool + skipTxEnvelope bool + networkPassphrase string } -func NewIndexer(networkPassphrase string, pool pond.Pool, metricsService processors.MetricsServiceInterface) *Indexer { +func NewIndexer(networkPassphrase string, pool pond.Pool, metricsService processors.MetricsServiceInterface, skipTxMeta bool, skipTxEnvelope bool) *Indexer { return &Indexer{ participantsProcessor: processors.NewParticipantsProcessor(networkPassphrase), tokenTransferProcessor: processors.NewTokenTransferProcessor(networkPassphrase, metricsService), @@ -89,17 +78,22 @@ func NewIndexer(networkPassphrase string, pool pond.Pool, metricsService process processors.NewContractDeployProcessor(networkPassphrase, metricsService), contract_processors.NewSACEventsProcessor(networkPassphrase, metricsService), }, - pool: pool, - metricsService: metricsService, + pool: pool, + metricsService: metricsService, + skipTxMeta: skipTxMeta, + skipTxEnvelope: skipTxEnvelope, + networkPassphrase: networkPassphrase, } } -// CollectAllTransactionData collects all transaction data (participants, operations, state changes) from all transactions in a ledger in parallel -func (i *Indexer) CollectAllTransactionData(ctx context.Context, transactions []ingest.LedgerTransaction) ([]PrecomputedTransactionData, set.Set[string], error) { - // Process transaction data collection in parallel +// ProcessLedgerTransactions processes all transactions in a ledger in parallel. +// It collects transaction data (participants, operations, state changes) and populates the buffer in a single pass. +// Returns the total participant count for metrics. +func (i *Indexer) ProcessLedgerTransactions(ctx context.Context, transactions []ingest.LedgerTransaction, ledgerBuffer IndexerBufferInterface) (int, error) { group := i.pool.NewGroupContext(ctx) - precomputedData := make([]PrecomputedTransactionData, len(transactions)) + txnBuffers := make([]*IndexerBuffer, len(transactions)) + participantCounts := make([]int, len(transactions)) var errs []error errMu := sync.Mutex{} @@ -107,200 +101,140 @@ func (i *Indexer) CollectAllTransactionData(ctx context.Context, transactions [] index := idx tx := tx group.Submit(func() { - txData := PrecomputedTransactionData{ - Transaction: tx, - AllParticipants: set.NewSet[string](), - } - - // Get transaction participants - txParticipants, err := i.participantsProcessor.GetTransactionParticipants(tx) - if err != nil { - errMu.Lock() - errs = append(errs, fmt.Errorf("getting transaction participants: %w", err)) - errMu.Unlock() - return - } - txData.TxParticipants = txParticipants - txData.AllParticipants = txData.AllParticipants.Union(txParticipants) - - // Get operations participants - opsParticipants, err := i.participantsProcessor.GetOperationsParticipants(tx) - if err != nil { - errMu.Lock() - errs = append(errs, fmt.Errorf("getting operations participants: %w", err)) - errMu.Unlock() - return - } - txData.OpsParticipants = opsParticipants - for _, opParticipants := range opsParticipants { - txData.AllParticipants = txData.AllParticipants.Union(opParticipants.Participants) - } - - // Get state changes - stateChanges, err := i.getTransactionStateChanges(ctx, tx, opsParticipants) + buffer := NewIndexerBuffer() + count, err := i.processTransaction(ctx, tx, buffer) if err != nil { errMu.Lock() - errs = append(errs, fmt.Errorf("getting transaction state changes: %w", err)) + errs = append(errs, fmt.Errorf("processing transaction at ledger=%d tx=%d: %w", tx.Ledger.LedgerSequence(), tx.Index, err)) errMu.Unlock() return } - txData.StateChanges = stateChanges - for _, stateChange := range stateChanges { - txData.AllParticipants.Add(stateChange.AccountID) - } - - trustlineChanges := []types.TrustlineChange{} - contractChanges := []types.ContractChange{} - for _, stateChange := range stateChanges { - switch stateChange.StateChangeCategory { - case types.StateChangeCategoryTrustline: - trustlineChange := types.TrustlineChange{ - AccountID: stateChange.AccountID, - OperationID: stateChange.OperationID, - Asset: stateChange.TrustlineAsset, - LedgerNumber: tx.Ledger.LedgerSequence(), - } - //exhaustive:ignore - switch *stateChange.StateChangeReason { - case types.StateChangeReasonAdd: - trustlineChange.Operation = types.TrustlineOpAdd - case types.StateChangeReasonRemove: - trustlineChange.Operation = types.TrustlineOpRemove - case types.StateChangeReasonUpdate: - continue - } - trustlineChanges = append(trustlineChanges, trustlineChange) - case types.StateChangeCategoryBalance: - // Only store contract changes when: - // - Account is C-address, OR - // - Account is G-address AND contract is NOT SAC or NATIVE (custom/SEP41 tokens): SAC token balances for G-addresses are stored in trustlines - accountIsContract := isContractAddress(stateChange.AccountID) - tokenIsSACOrNative := stateChange.ContractType == types.ContractTypeSAC || stateChange.ContractType == types.ContractTypeNative - - if accountIsContract || !tokenIsSACOrNative { - contractChange := types.ContractChange{ - AccountID: stateChange.AccountID, - OperationID: stateChange.OperationID, - ContractID: stateChange.TokenID.String, - LedgerNumber: tx.Ledger.LedgerSequence(), - ContractType: stateChange.ContractType, - } - contractChanges = append(contractChanges, contractChange) - } - default: - continue - } - } - txData.TrustlineChanges = trustlineChanges - txData.ContractChanges = contractChanges - - // Add to collection - precomputedData[index] = txData + txnBuffers[index] = buffer + participantCounts[index] = count }) } + if err := group.Wait(); err != nil { - return nil, nil, fmt.Errorf("waiting for transaction data collection: %w", err) + return 0, fmt.Errorf("waiting for transaction processing: %w", err) } if len(errs) > 0 { - return nil, nil, fmt.Errorf("collecting transaction data: %w", errors.Join(errs...)) + return 0, fmt.Errorf("processing transactions: %w", errors.Join(errs...)) } - // Merge all participant sets for the single DB call - allParticipants := set.NewSet[string]() - for _, txData := range precomputedData { - allParticipants = allParticipants.Union(txData.AllParticipants) + // Merge buffers and count participants + totalParticipants := 0 + for idx, buffer := range txnBuffers { + ledgerBuffer.Merge(buffer) + totalParticipants += participantCounts[idx] } - return precomputedData, allParticipants, nil + return totalParticipants, nil } -// ProcessTransactions processes transactions, operations and state changes using precomputed data. It then inserts them into the indexer buffer. -func (i *Indexer) ProcessTransactions(ctx context.Context, precomputedData []PrecomputedTransactionData, existingAccounts set.Set[string], ledgerBuffer IndexerBufferInterface) error { - // Process transactions in parallel using precomputed data - group := i.pool.NewGroupContext(ctx) - var errs []error - errMu := sync.Mutex{} - txnBuffers := make([]*IndexerBuffer, len(precomputedData)) - for idx, txData := range precomputedData { - index := idx - txData := txData - group.Submit(func() { - buffer := NewIndexerBuffer() - if err := i.processPrecomputedTransaction(ctx, txData, existingAccounts, buffer); err != nil { - errMu.Lock() - defer errMu.Unlock() - errs = append(errs, fmt.Errorf("processing precomputed transaction data at ledger=%d tx=%d: %w", txData.Transaction.Ledger.LedgerSequence(), txData.Transaction.Index, err)) - } - txnBuffers[index] = buffer - }) - } - if err := group.Wait(); err != nil { - return fmt.Errorf("waiting for transaction processing: %w", err) - } - if len(errs) > 0 { - return fmt.Errorf("processing transactions: %w", errors.Join(errs...)) +// processTransaction processes a single transaction - collects data and populates buffer. +// Returns participant count for metrics. +func (i *Indexer) processTransaction(ctx context.Context, tx ingest.LedgerTransaction, buffer *IndexerBuffer) (int, error) { + // Get transaction participants + txParticipants, err := i.participantsProcessor.GetTransactionParticipants(tx) + if err != nil { + return 0, fmt.Errorf("getting transaction participants: %w", err) } - for _, buffer := range txnBuffers { - ledgerBuffer.MergeBuffer(buffer) + + // Get operations participants + opsParticipants, err := i.participantsProcessor.GetOperationsParticipants(tx) + if err != nil { + return 0, fmt.Errorf("getting operations participants: %w", err) } - return nil -} + // Get state changes + stateChanges, err := i.getTransactionStateChanges(ctx, tx, opsParticipants) + if err != nil { + return 0, fmt.Errorf("getting transaction state changes: %w", err) + } -// processPrecomputedTransaction processes a transaction using precomputed data without re-computing participants or state changes -func (i *Indexer) processPrecomputedTransaction(ctx context.Context, precomputedData PrecomputedTransactionData, existingAccounts set.Set[string], buffer *IndexerBuffer) error { // Convert transaction data - dataTx, err := processors.ConvertTransaction(&precomputedData.Transaction) + dataTx, err := processors.ConvertTransaction(&tx, i.skipTxMeta, i.skipTxEnvelope, i.networkPassphrase) if err != nil { - return fmt.Errorf("creating data transaction: %w", err) + return 0, fmt.Errorf("creating data transaction: %w", err) + } + + // Count all unique participants for metrics + allParticipants := set.NewSet[string]() + allParticipants = allParticipants.Union(txParticipants) + for _, opParticipants := range opsParticipants { + allParticipants = allParticipants.Union(opParticipants.Participants) + } + for _, stateChange := range stateChanges { + allParticipants.Add(stateChange.AccountID) } // Insert transaction participants - if precomputedData.TxParticipants.Cardinality() != 0 { - for participant := range precomputedData.TxParticipants.Iter() { - if !existingAccounts.Contains(participant) { - continue - } - buffer.PushTransaction(participant, *dataTx) - } + for participant := range txParticipants.Iter() { + buffer.PushTransaction(participant, *dataTx) } // Insert operations participants - var dataOp *types.Operation operationsMap := make(map[int64]*types.Operation) - for opID, opParticipants := range precomputedData.OpsParticipants { - dataOp, err = processors.ConvertOperation(&precomputedData.Transaction, &opParticipants.OpWrapper.Operation, opID) - if err != nil { - return fmt.Errorf("creating data operation: %w", err) + for opID, opParticipants := range opsParticipants { + dataOp, opErr := processors.ConvertOperation(&tx, &opParticipants.OpWrapper.Operation, opID) + if opErr != nil { + return 0, fmt.Errorf("creating data operation: %w", opErr) } operationsMap[opID] = dataOp for participant := range opParticipants.Participants.Iter() { - if !existingAccounts.Contains(participant) { - continue - } buffer.PushOperation(participant, *dataOp, *dataTx) } } - // Insert trustline changes - for _, trustlineChange := range precomputedData.TrustlineChanges { - buffer.PushTrustlineChange(trustlineChange) - } - - // Insert contract changes - for _, contractChange := range precomputedData.ContractChanges { - buffer.PushContractChange(contractChange) + // Process state changes to extract trustline and contract changes + for _, stateChange := range stateChanges { + //exhaustive:ignore + switch stateChange.StateChangeCategory { + case types.StateChangeCategoryTrustline: + trustlineChange := types.TrustlineChange{ + AccountID: stateChange.AccountID, + OperationID: stateChange.OperationID, + Asset: stateChange.TrustlineAsset, + LedgerNumber: tx.Ledger.LedgerSequence(), + } + //exhaustive:ignore + switch *stateChange.StateChangeReason { + case types.StateChangeReasonAdd: + trustlineChange.Operation = types.TrustlineOpAdd + case types.StateChangeReasonRemove: + trustlineChange.Operation = types.TrustlineOpRemove + case types.StateChangeReasonUpdate: + continue + } + buffer.PushTrustlineChange(trustlineChange) + case types.StateChangeCategoryBalance: + // Only store contract changes when: + // - Account is C-address, OR + // - Account is G-address AND contract is NOT SAC or NATIVE (custom/SEP41 tokens): SAC token balances for G-addresses are stored in trustlines + accountIsContract := isContractAddress(stateChange.AccountID) + tokenIsSACOrNative := stateChange.ContractType == types.ContractTypeSAC || stateChange.ContractType == types.ContractTypeNative + + if accountIsContract || !tokenIsSACOrNative { + contractChange := types.ContractChange{ + AccountID: stateChange.AccountID, + OperationID: stateChange.OperationID, + ContractID: stateChange.TokenID.String, + LedgerNumber: tx.Ledger.LedgerSequence(), + ContractType: stateChange.ContractType, + } + buffer.PushContractChange(contractChange) + } + } } // Sort state changes and set order for this transaction - stateChanges := precomputedData.StateChanges sort.Slice(stateChanges, func(i, j int) bool { return stateChanges[i].SortKey < stateChanges[j].SortKey }) perOpIdx := make(map[int64]int) - for i := range stateChanges { - sc := &stateChanges[i] + for idx := range stateChanges { + sc := &stateChanges[idx] // State changes are 1-indexed within an operation/transaction. if sc.OperationID != 0 { @@ -311,9 +245,10 @@ func (i *Indexer) processPrecomputedTransaction(ctx context.Context, precomputed } } - // Process state changes + // Insert state changes for _, stateChange := range stateChanges { - if !existingAccounts.Contains(stateChange.AccountID) { + // Skip empty state changes (no account to associate with) + if stateChange.AccountID == "" { continue } @@ -331,7 +266,7 @@ func (i *Indexer) processPrecomputedTransaction(ctx context.Context, precomputed buffer.PushStateChange(*dataTx, operation, stateChange) } - return nil + return allParticipants.Cardinality(), nil } // getTransactionStateChanges processes operations of a transaction and calculates all state changes diff --git a/internal/indexer/indexer_buffer.go b/internal/indexer/indexer_buffer.go index b98b8244f..caa2a58d2 100644 --- a/internal/indexer/indexer_buffer.go +++ b/internal/indexer/indexer_buffer.go @@ -49,6 +49,7 @@ type IndexerBuffer struct { stateChanges []types.StateChange trustlineChanges []types.TrustlineChange contractChanges []types.ContractChange + allParticipants set.Set[string] } // NewIndexerBuffer creates a new IndexerBuffer with initialized data structures. @@ -62,6 +63,7 @@ func NewIndexerBuffer() *IndexerBuffer { stateChanges: make([]types.StateChange, 0), trustlineChanges: make([]types.TrustlineChange, 0), contractChanges: make([]types.ContractChange, 0), + allParticipants: set.NewSet[string](), } } @@ -98,6 +100,11 @@ func (b *IndexerBuffer) pushTransactionUnsafe(participant string, transaction *t // Add participant - O(1) with automatic deduplication b.participantsByTxHash[txHash].Add(participant) + + // Track participant in global set for batch account insertion + if participant != "" { + b.allParticipants.Add(participant) + } } // GetNumberOfTransactions returns the count of unique transactions in the buffer. @@ -120,13 +127,13 @@ func (b *IndexerBuffer) GetNumberOfOperations() int { // GetTransactions returns all unique transactions. // Thread-safe: uses read lock. -func (b *IndexerBuffer) GetTransactions() []types.Transaction { +func (b *IndexerBuffer) GetTransactions() []*types.Transaction { b.mu.RLock() defer b.mu.RUnlock() - txs := make([]types.Transaction, 0, len(b.txByHash)) + txs := make([]*types.Transaction, 0, len(b.txByHash)) for _, txPtr := range b.txByHash { - txs = append(txs, *txPtr) + txs = append(txs, txPtr) } return txs @@ -190,13 +197,13 @@ func (b *IndexerBuffer) PushOperation(participant string, operation types.Operat // GetOperations returns all unique operations from the canonical storage. // Returns values (not pointers) for API compatibility. // Thread-safe: uses read lock. -func (b *IndexerBuffer) GetOperations() []types.Operation { +func (b *IndexerBuffer) GetOperations() []*types.Operation { b.mu.RLock() defer b.mu.RUnlock() - ops := make([]types.Operation, 0, len(b.opByID)) + ops := make([]*types.Operation, 0, len(b.opByID)) for _, opPtr := range b.opByID { - ops = append(ops, *opPtr) + ops = append(ops, opPtr) } return ops } @@ -223,6 +230,11 @@ func (b *IndexerBuffer) pushOperationUnsafe(participant string, operation *types b.participantsByOpID[opID] = set.NewSet[string]() } b.participantsByOpID[opID].Add(participant) + + // Track participant in global set for batch account insertion + if participant != "" { + b.allParticipants.Add(participant) + } } // PushStateChange adds a state change along with its associated transaction and operation. @@ -248,7 +260,17 @@ func (b *IndexerBuffer) GetStateChanges() []types.StateChange { return b.stateChanges } -// MergeBuffer merges another IndexerBuffer into this buffer. This is used to combine +// GetAllParticipants returns all unique participants (Stellar addresses) that have been +// recorded during transaction, operation, and state change processing. +// Thread-safe: uses read lock. +func (b *IndexerBuffer) GetAllParticipants() []string { + b.mu.RLock() + defer b.mu.RUnlock() + + return b.allParticipants.ToSlice() +} + +// Merge merges another IndexerBuffer into this buffer. This is used to combine // per-ledger or per-transaction buffers into a single buffer for batch DB insertion. // // MERGE STRATEGY: @@ -268,7 +290,7 @@ func (b *IndexerBuffer) GetStateChanges() []types.StateChange { // Zero temporary allocations - uses direct map/set manipulation. // // Thread-safe: acquires write lock on this buffer, read lock on other buffer. -func (b *IndexerBuffer) MergeBuffer(other IndexerBufferInterface) { +func (b *IndexerBuffer) Merge(other IndexerBufferInterface) { b.mu.Lock() defer b.mu.Unlock() @@ -284,24 +306,28 @@ func (b *IndexerBuffer) MergeBuffer(other IndexerBufferInterface) { // Merge transactions (canonical storage) - this establishes our canonical pointers maps.Copy(b.txByHash, otherBuffer.txByHash) for txHash, otherParticipants := range otherBuffer.participantsByTxHash { - if _, exists := b.participantsByTxHash[txHash]; !exists { - b.participantsByTxHash[txHash] = set.NewSet[string]() - } - // Iterate other's set, add participants from OUR txByHash - for participant := range otherParticipants.Iter() { - b.participantsByTxHash[txHash].Add(participant) // O(1) Add + if existing, exists := b.participantsByTxHash[txHash]; exists { + // Merge into existing set - iterate and add (Union creates new set) + for participant := range otherParticipants.Iter() { + existing.Add(participant) + } + } else { + // Clone the set instead of creating empty + iterating + b.participantsByTxHash[txHash] = otherParticipants.Clone() } } // Merge operations (canonical storage) maps.Copy(b.opByID, otherBuffer.opByID) for opID, otherParticipants := range otherBuffer.participantsByOpID { - if _, exists := b.participantsByOpID[opID]; !exists { - b.participantsByOpID[opID] = set.NewSet[string]() - } - // Iterate other's set, add canonical pointers from OUR opByID - for participant := range otherParticipants.Iter() { - b.participantsByOpID[opID].Add(participant) // O(1) Add + if existing, exists := b.participantsByOpID[opID]; exists { + // Merge into existing set - iterate and add (Union creates new set) + for participant := range otherParticipants.Iter() { + existing.Add(participant) + } + } else { + // Clone the set instead of creating empty + iterating + b.participantsByOpID[opID] = otherParticipants.Clone() } } @@ -313,4 +339,31 @@ func (b *IndexerBuffer) MergeBuffer(other IndexerBufferInterface) { // Merge contract changes b.contractChanges = append(b.contractChanges, otherBuffer.contractChanges...) + + // Merge all participants + for participant := range otherBuffer.allParticipants.Iter() { + b.allParticipants.Add(participant) + } +} + +// Clear resets the buffer to its initial empty state while preserving allocated capacity. +// Use this to reuse the buffer after flushing data to the database during backfill. +// Thread-safe: acquires write lock. +func (b *IndexerBuffer) Clear() { + b.mu.Lock() + defer b.mu.Unlock() + + // Clear maps (keep allocated backing arrays) + clear(b.txByHash) + clear(b.participantsByTxHash) + clear(b.opByID) + clear(b.participantsByOpID) + + // Reset slices (reuse underlying arrays by slicing to zero) + b.stateChanges = b.stateChanges[:0] + b.trustlineChanges = b.trustlineChanges[:0] + b.contractChanges = b.contractChanges[:0] + + // Clear all participants set + b.allParticipants.Clear() } diff --git a/internal/indexer/indexer_buffer_test.go b/internal/indexer/indexer_buffer_test.go index 0521c63e8..12991ea57 100644 --- a/internal/indexer/indexer_buffer_test.go +++ b/internal/indexer/indexer_buffer_test.go @@ -45,7 +45,7 @@ func TestIndexerBuffer_PushTransaction(t *testing.T) { assert.Equal(t, 2, indexerBuffer.GetNumberOfTransactions()) // Assert GetAllTransactions - assert.ElementsMatch(t, []types.Transaction{tx1, tx2}, indexerBuffer.GetTransactions()) + assert.ElementsMatch(t, []*types.Transaction{&tx1, &tx2}, indexerBuffer.GetTransactions()) }) t.Run("🟢 concurrent pushes", func(t *testing.T) { @@ -268,7 +268,7 @@ func TestIndexerBuffer_GetAllTransactions(t *testing.T) { allTxs := indexerBuffer.GetTransactions() require.Len(t, allTxs, 2) - assert.ElementsMatch(t, []types.Transaction{tx1, tx2}, allTxs) + assert.ElementsMatch(t, []*types.Transaction{&tx1, &tx2}, allTxs) }) } @@ -303,7 +303,7 @@ func TestIndexerBuffer_GetAllOperations(t *testing.T) { allOps := indexerBuffer.GetOperations() require.Len(t, allOps, 2) - assert.ElementsMatch(t, []types.Operation{op1, op2}, allOps) + assert.ElementsMatch(t, []*types.Operation{&op1, &op2}, allOps) }) } @@ -345,12 +345,95 @@ func TestIndexerBuffer_GetAllStateChanges(t *testing.T) { }) } -func TestIndexerBuffer_MergeBuffer(t *testing.T) { +func TestIndexerBuffer_GetAllParticipants(t *testing.T) { + t.Run("🟢 returns empty for new buffer", func(t *testing.T) { + indexerBuffer := NewIndexerBuffer() + participants := indexerBuffer.GetAllParticipants() + assert.Empty(t, participants) + }) + + t.Run("🟢 collects participants from transactions", func(t *testing.T) { + indexerBuffer := NewIndexerBuffer() + + tx1 := types.Transaction{Hash: "tx_hash_1"} + tx2 := types.Transaction{Hash: "tx_hash_2"} + + indexerBuffer.PushTransaction("alice", tx1) + indexerBuffer.PushTransaction("bob", tx2) + indexerBuffer.PushTransaction("alice", tx2) // duplicate participant + + participants := indexerBuffer.GetAllParticipants() + assert.ElementsMatch(t, []string{"alice", "bob"}, participants) + }) + + t.Run("🟢 collects participants from operations", func(t *testing.T) { + indexerBuffer := NewIndexerBuffer() + + tx := types.Transaction{Hash: "tx_hash_1"} + op1 := types.Operation{ID: 1, TxHash: tx.Hash} + op2 := types.Operation{ID: 2, TxHash: tx.Hash} + + indexerBuffer.PushOperation("alice", op1, tx) + indexerBuffer.PushOperation("bob", op2, tx) + indexerBuffer.PushOperation("charlie", op2, tx) + + participants := indexerBuffer.GetAllParticipants() + assert.ElementsMatch(t, []string{"alice", "bob", "charlie"}, participants) + }) + + t.Run("🟢 collects participants from state changes", func(t *testing.T) { + indexerBuffer := NewIndexerBuffer() + + tx := types.Transaction{Hash: "tx_hash_1", ToID: 1} + op := types.Operation{ID: 1, TxHash: tx.Hash} + + sc1 := types.StateChange{ToID: 1, StateChangeOrder: 1, AccountID: "alice", OperationID: 1} + sc2 := types.StateChange{ToID: 2, StateChangeOrder: 1, AccountID: "bob", OperationID: 1} + sc3 := types.StateChange{ToID: 3, StateChangeOrder: 1, AccountID: "charlie", OperationID: 0} // fee change + + indexerBuffer.PushStateChange(tx, op, sc1) + indexerBuffer.PushStateChange(tx, op, sc2) + indexerBuffer.PushStateChange(tx, types.Operation{}, sc3) + + participants := indexerBuffer.GetAllParticipants() + assert.ElementsMatch(t, []string{"alice", "bob", "charlie"}, participants) + }) + + t.Run("🟢 collects unique participants from all sources", func(t *testing.T) { + indexerBuffer := NewIndexerBuffer() + + tx := types.Transaction{Hash: "tx_hash_1", ToID: 1} + op := types.Operation{ID: 1, TxHash: tx.Hash} + sc := types.StateChange{ToID: 1, StateChangeOrder: 1, AccountID: "dave", OperationID: 1} + + // Add participants from different sources + indexerBuffer.PushTransaction("alice", tx) + indexerBuffer.PushOperation("bob", op, tx) + indexerBuffer.PushStateChange(tx, op, sc) + + participants := indexerBuffer.GetAllParticipants() + // alice (tx), bob (op which also adds to tx), dave (state change which also adds to tx and op) + assert.ElementsMatch(t, []string{"alice", "bob", "dave"}, participants) + }) + + t.Run("🟢 ignores empty participants", func(t *testing.T) { + indexerBuffer := NewIndexerBuffer() + + tx := types.Transaction{Hash: "tx_hash_1"} + indexerBuffer.PushTransaction("", tx) // empty participant + indexerBuffer.PushTransaction("alice", tx) + + participants := indexerBuffer.GetAllParticipants() + assert.ElementsMatch(t, []string{"alice"}, participants) + }) +} + +func TestIndexerBuffer_Merge(t *testing.T) { t.Run("🟢 merge empty buffers", func(t *testing.T) { buffer1 := NewIndexerBuffer() buffer2 := NewIndexerBuffer() - buffer1.MergeBuffer(buffer2) + buffer1.Merge(buffer2) assert.Equal(t, 0, buffer1.GetNumberOfTransactions()) assert.Len(t, buffer1.GetStateChanges(), 0) }) @@ -365,12 +448,12 @@ func TestIndexerBuffer_MergeBuffer(t *testing.T) { buffer1.PushTransaction("alice", tx1) buffer2.PushTransaction("bob", tx2) - buffer1.MergeBuffer(buffer2) + buffer1.Merge(buffer2) // Verify transactions allTxs := buffer1.GetTransactions() assert.Len(t, allTxs, 2) - assert.ElementsMatch(t, []types.Transaction{tx1, tx2}, allTxs) + assert.ElementsMatch(t, []*types.Transaction{&tx1, &tx2}, allTxs) // Verify transaction participants txParticipants := buffer1.GetTransactionsParticipants() @@ -389,12 +472,12 @@ func TestIndexerBuffer_MergeBuffer(t *testing.T) { buffer1.PushOperation("alice", op1, tx1) buffer2.PushOperation("bob", op2, tx1) - buffer1.MergeBuffer(buffer2) + buffer1.Merge(buffer2) // Verify operations allOps := buffer1.GetOperations() assert.Len(t, allOps, 2) - assert.ElementsMatch(t, []types.Operation{op1, op2}, allOps) + assert.ElementsMatch(t, []*types.Operation{&op1, &op2}, allOps) // Verify operation participants opParticipants := buffer1.GetOperationsParticipants() @@ -415,7 +498,7 @@ func TestIndexerBuffer_MergeBuffer(t *testing.T) { buffer1.PushStateChange(tx, op, sc1) buffer2.PushStateChange(tx, op, sc2) - buffer1.MergeBuffer(buffer2) + buffer1.Merge(buffer2) // Verify state changes allStateChanges := buffer1.GetStateChanges() @@ -441,7 +524,7 @@ func TestIndexerBuffer_MergeBuffer(t *testing.T) { buffer2.PushTransaction("charlie", tx2) buffer2.PushOperation("bob", op1, tx1) - buffer1.MergeBuffer(buffer2) + buffer1.Merge(buffer2) // Verify transactions allTxs := buffer1.GetTransactions() @@ -469,7 +552,7 @@ func TestIndexerBuffer_MergeBuffer(t *testing.T) { buffer2.PushOperation("bob", op1, tx1) buffer2.PushStateChange(tx1, op1, sc1) - buffer1.MergeBuffer(buffer2) + buffer1.Merge(buffer2) assert.Equal(t, 1, buffer1.GetNumberOfTransactions()) assert.Len(t, buffer1.GetOperations(), 1) @@ -483,7 +566,7 @@ func TestIndexerBuffer_MergeBuffer(t *testing.T) { tx1 := types.Transaction{Hash: "tx_hash_1"} buffer1.PushTransaction("alice", tx1) - buffer1.MergeBuffer(buffer2) + buffer1.Merge(buffer2) assert.Equal(t, 1, buffer1.GetNumberOfTransactions()) }) @@ -506,12 +589,12 @@ func TestIndexerBuffer_MergeBuffer(t *testing.T) { go func() { defer wg.Done() - buffer1.MergeBuffer(buffer2) + buffer1.Merge(buffer2) }() go func() { defer wg.Done() - buffer1.MergeBuffer(buffer3) + buffer1.Merge(buffer3) }() wg.Wait() @@ -541,7 +624,7 @@ func TestIndexerBuffer_MergeBuffer(t *testing.T) { buffer2.PushOperation("bob", op2, tx2) buffer2.PushStateChange(tx2, op2, sc2) - buffer1.MergeBuffer(buffer2) + buffer1.Merge(buffer2) // Verify transactions allTxs := buffer1.GetTransactions() @@ -566,4 +649,23 @@ func TestIndexerBuffer_MergeBuffer(t *testing.T) { assert.Equal(t, set.NewSet("alice"), opParticipants[int64(1)]) assert.Equal(t, set.NewSet("bob"), opParticipants[int64(2)]) }) + + t.Run("🟢 merge all participants", func(t *testing.T) { + buffer1 := NewIndexerBuffer() + buffer2 := NewIndexerBuffer() + + tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} + tx2 := types.Transaction{Hash: "tx_hash_2", ToID: 2} + + buffer1.PushTransaction("alice", tx1) + buffer1.PushTransaction("bob", tx1) + buffer2.PushTransaction("charlie", tx2) + buffer2.PushTransaction("dave", tx2) + + buffer1.Merge(buffer2) + + // Verify all participants merged + allParticipants := buffer1.GetAllParticipants() + assert.ElementsMatch(t, []string{"alice", "bob", "charlie", "dave"}, allParticipants) + }) } diff --git a/internal/indexer/indexer_test.go b/internal/indexer/indexer_test.go index a6a0c9c13..a96c1dd4d 100644 --- a/internal/indexer/indexer_test.go +++ b/internal/indexer/indexer_test.go @@ -134,7 +134,7 @@ func TestIndexer_NewIndexer(t *testing.T) { networkPassphrase := network.TestNetworkPassphrase pool := pond.NewPool(runtime.NumCPU()) - indexer := NewIndexer(networkPassphrase, pool, nil) + indexer := NewIndexer(networkPassphrase, pool, nil, false, false) require.NotNil(t, indexer) assert.NotNil(t, indexer.participantsProcessor) @@ -142,9 +142,11 @@ func TestIndexer_NewIndexer(t *testing.T) { assert.NotNil(t, indexer.processors) assert.NotNil(t, indexer.pool) assert.Len(t, indexer.processors, 3) // effects, contract deploy, SAC events + assert.False(t, indexer.skipTxMeta) + assert.False(t, indexer.skipTxEnvelope) } -func TestIndexer_CollectAllTransactionData(t *testing.T) { +func TestIndexer_ProcessLedgerTransactions(t *testing.T) { t.Run("🟢 single transaction with participants", func(t *testing.T) { // Create mocks mockParticipants := &MockParticipantsProcessor{} @@ -188,61 +190,34 @@ func TestIndexer_CollectAllTransactionData(t *testing.T) { tokenTransferProcessor: mockTokenTransfer, processors: []OperationProcessorInterface{mockEffects, mockContractDeploy, mockSACEvents}, pool: pond.NewPool(runtime.NumCPU()), + networkPassphrase: network.TestNetworkPassphrase, } - // Test CollectAllTransactionData - precomputedData, allParticipants, err := indexer.CollectAllTransactionData(context.Background(), []ingest.LedgerTransaction{testTx}) + // Test ProcessLedgerTransactions + buffer := NewIndexerBuffer() + participantCount, err := indexer.ProcessLedgerTransactions(context.Background(), []ingest.LedgerTransaction{testTx}, buffer) // Assert results require.NoError(t, err) - assert.Len(t, precomputedData, 1) - assert.Equal(t, set.NewSet("alice", "bob", "charlie", "dave"), allParticipants) - - // Verify PrecomputedTransactionData structure - txData := precomputedData[0] - assert.Equal(t, testTx.Index, txData.Transaction.Index) - assert.Equal(t, testTx.Hash, txData.Transaction.Hash) - assert.NotNil(t, txData.TxParticipants) - assert.NotNil(t, txData.OpsParticipants) - assert.NotNil(t, txData.StateChanges) - assert.NotNil(t, txData.AllParticipants) - - // Verify exact transaction participants - assert.Equal(t, 2, txData.TxParticipants.Cardinality()) - assert.True(t, txData.TxParticipants.Contains("alice")) - assert.True(t, txData.TxParticipants.Contains("bob")) - - // Verify operations participants - assert.Contains(t, txData.OpsParticipants, int64(1)) - opPart := txData.OpsParticipants[int64(1)] - assert.Equal(t, 1, opPart.Participants.Cardinality()) - assert.True(t, opPart.Participants.Contains("alice")) + assert.Equal(t, 4, participantCount) // alice, bob, charlie, dave - // Verify state changes - assert.Len(t, txData.StateChanges, 3) - for _, sc := range txData.StateChanges { - switch sc.AccountID { - case "alice": - assert.Equal(t, int64(1), sc.ToID) - assert.Equal(t, int64(1), sc.OperationID) - assert.Equal(t, "1-1", sc.SortKey) - case "charlie": - assert.Equal(t, int64(2), sc.ToID) - assert.Equal(t, int64(1), sc.OperationID) - assert.Equal(t, "1-2", sc.SortKey) - case "dave": - assert.Equal(t, int64(3), sc.ToID) - assert.Equal(t, int64(0), sc.OperationID) - assert.Equal(t, "0-1", sc.SortKey) - } - } + // Verify transactions + allTxs := buffer.GetTransactions() + require.Len(t, allTxs, 1, "should have 1 transaction") + + // Verify transaction participants + txParticipantsMap := buffer.GetTransactionsParticipants() + txHash := "0102030000000000000000000000000000000000000000000000000000000000" + assert.True(t, txParticipantsMap[txHash].Contains("alice"), "alice should be in tx participants") + assert.True(t, txParticipantsMap[txHash].Contains("bob"), "bob should be in tx participants") + + // Verify operations + allOps := buffer.GetOperations() + require.Len(t, allOps, 1, "should have 1 operation") - // Verify AllParticipants is union of all - assert.Equal(t, 4, txData.AllParticipants.Cardinality()) - assert.True(t, txData.AllParticipants.Contains("alice")) - assert.True(t, txData.AllParticipants.Contains("bob")) - assert.True(t, txData.AllParticipants.Contains("charlie")) - assert.True(t, txData.AllParticipants.Contains("dave")) + // Verify state changes + stateChanges := buffer.GetStateChanges() + assert.Len(t, stateChanges, 3, "should have 3 state changes") // Verify mock expectations mockParticipants.AssertExpectations(t) @@ -305,36 +280,21 @@ func TestIndexer_CollectAllTransactionData(t *testing.T) { tokenTransferProcessor: mockTokenTransfer, processors: []OperationProcessorInterface{mockEffects, mockContractDeploy, mockSACEvents}, pool: pond.NewPool(runtime.NumCPU()), + networkPassphrase: network.TestNetworkPassphrase, } - // Test CollectAllTransactionData - precomputedData, allParticipants, err := indexer.CollectAllTransactionData(context.Background(), []ingest.LedgerTransaction{testTx, testTx2}) + // Test ProcessLedgerTransactions + buffer := NewIndexerBuffer() + participantCount, err := indexer.ProcessLedgerTransactions(context.Background(), []ingest.LedgerTransaction{testTx, testTx2}, buffer) // Assert results require.NoError(t, err) - assert.Len(t, precomputedData, 2) - assert.Equal(t, set.NewSet("alice", "bob", "charlie"), allParticipants) - - // Verify each transaction - for _, txData := range precomputedData { - assert.NotNil(t, txData.TxParticipants) - assert.NotNil(t, txData.OpsParticipants) - assert.NotNil(t, txData.StateChanges) - assert.NotNil(t, txData.AllParticipants) - - if txData.Transaction.Index == 1 { - assert.Equal(t, 2, txData.TxParticipants.Cardinality()) - assert.True(t, txData.TxParticipants.Contains("alice")) - assert.True(t, txData.TxParticipants.Contains("bob")) - assert.Contains(t, txData.OpsParticipants, int64(1)) - } - if txData.Transaction.Index == 2 { - assert.Equal(t, 2, txData.TxParticipants.Cardinality()) - assert.True(t, txData.TxParticipants.Contains("bob")) - assert.True(t, txData.TxParticipants.Contains("charlie")) - assert.Contains(t, txData.OpsParticipants, int64(2)) - } - } + // alice, bob from tx1 + bob, charlie from tx2 = 2+2=4 (bob counted twice since per-tx) + assert.Equal(t, 4, participantCount) + + // Verify transactions + allTxs := buffer.GetTransactions() + require.Len(t, allTxs, 2, "should have 2 transactions") // Verify mock expectations mockParticipants.AssertExpectations(t) @@ -358,15 +318,17 @@ func TestIndexer_CollectAllTransactionData(t *testing.T) { tokenTransferProcessor: mockTokenTransfer, processors: []OperationProcessorInterface{mockEffects, mockContractDeploy, mockSACEvents}, pool: pond.NewPool(runtime.NumCPU()), + networkPassphrase: network.TestNetworkPassphrase, } - // Test CollectAllTransactionData - precomputedData, allParticipants, err := indexer.CollectAllTransactionData(context.Background(), []ingest.LedgerTransaction{}) + // Test ProcessLedgerTransactions + buffer := NewIndexerBuffer() + participantCount, err := indexer.ProcessLedgerTransactions(context.Background(), []ingest.LedgerTransaction{}, buffer) // Assert results require.NoError(t, err) - assert.Len(t, precomputedData, 0) - assert.Equal(t, set.NewSet[string](), allParticipants) + assert.Equal(t, 0, participantCount) + assert.Equal(t, 0, buffer.GetNumberOfTransactions()) // Verify mock expectations mockParticipants.AssertExpectations(t) @@ -395,20 +357,16 @@ func TestIndexer_CollectAllTransactionData(t *testing.T) { tokenTransferProcessor: mockTokenTransfer, processors: []OperationProcessorInterface{mockEffects, mockContractDeploy, mockSACEvents}, pool: pond.NewPool(runtime.NumCPU()), + networkPassphrase: network.TestNetworkPassphrase, } - // Test CollectAllTransactionData - precomputedData, allParticipants, err := indexer.CollectAllTransactionData(context.Background(), []ingest.LedgerTransaction{testTx}) + // Test ProcessLedgerTransactions + buffer := NewIndexerBuffer() + participantCount, err := indexer.ProcessLedgerTransactions(context.Background(), []ingest.LedgerTransaction{testTx}, buffer) // Assert results require.NoError(t, err) - assert.Len(t, precomputedData, 1) - assert.Equal(t, set.NewSet[string](), allParticipants) - - txData := precomputedData[0] - assert.Equal(t, 0, txData.TxParticipants.Cardinality()) - assert.Equal(t, 0, len(txData.OpsParticipants)) - assert.Equal(t, 0, txData.AllParticipants.Cardinality()) + assert.Equal(t, 0, participantCount) // Verify mock expectations mockParticipants.AssertExpectations(t) @@ -435,16 +393,17 @@ func TestIndexer_CollectAllTransactionData(t *testing.T) { tokenTransferProcessor: mockTokenTransfer, processors: []OperationProcessorInterface{mockEffects, mockContractDeploy, mockSACEvents}, pool: pond.NewPool(runtime.NumCPU()), + networkPassphrase: network.TestNetworkPassphrase, } - // Test CollectAllTransactionData - precomputedData, allParticipants, err := indexer.CollectAllTransactionData(context.Background(), []ingest.LedgerTransaction{testTx}) + // Test ProcessLedgerTransactions + buffer := NewIndexerBuffer() + participantCount, err := indexer.ProcessLedgerTransactions(context.Background(), []ingest.LedgerTransaction{testTx}, buffer) // Assert results require.Error(t, err) assert.Contains(t, err.Error(), "getting transaction participants: participant error") - assert.Nil(t, precomputedData) - assert.Nil(t, allParticipants) + assert.Equal(t, 0, participantCount) // Verify mock expectations mockParticipants.AssertExpectations(t) @@ -472,16 +431,17 @@ func TestIndexer_CollectAllTransactionData(t *testing.T) { tokenTransferProcessor: mockTokenTransfer, processors: []OperationProcessorInterface{mockEffects, mockContractDeploy, mockSACEvents}, pool: pond.NewPool(runtime.NumCPU()), + networkPassphrase: network.TestNetworkPassphrase, } - // Test CollectAllTransactionData - precomputedData, allParticipants, err := indexer.CollectAllTransactionData(context.Background(), []ingest.LedgerTransaction{testTx}) + // Test ProcessLedgerTransactions + buffer := NewIndexerBuffer() + participantCount, err := indexer.ProcessLedgerTransactions(context.Background(), []ingest.LedgerTransaction{testTx}, buffer) // Assert results require.Error(t, err) assert.Contains(t, err.Error(), "getting operations participants: operations error") - assert.Nil(t, precomputedData) - assert.Nil(t, allParticipants) + assert.Equal(t, 0, participantCount) // Verify mock expectations mockParticipants.AssertExpectations(t) @@ -510,16 +470,17 @@ func TestIndexer_CollectAllTransactionData(t *testing.T) { tokenTransferProcessor: mockTokenTransfer, processors: []OperationProcessorInterface{mockEffects, mockContractDeploy, mockSACEvents}, pool: pond.NewPool(runtime.NumCPU()), + networkPassphrase: network.TestNetworkPassphrase, } - // Test CollectAllTransactionData - precomputedData, allParticipants, err := indexer.CollectAllTransactionData(context.Background(), []ingest.LedgerTransaction{testTx}) + // Test ProcessLedgerTransactions + buffer := NewIndexerBuffer() + participantCount, err := indexer.ProcessLedgerTransactions(context.Background(), []ingest.LedgerTransaction{testTx}, buffer) // Assert results require.Error(t, err) assert.Contains(t, err.Error(), "processing token transfer state changes: token transfer error") - assert.Nil(t, precomputedData) - assert.Nil(t, allParticipants) + assert.Equal(t, 0, participantCount) // Verify mock expectations mockParticipants.AssertExpectations(t) @@ -560,16 +521,17 @@ func TestIndexer_CollectAllTransactionData(t *testing.T) { tokenTransferProcessor: mockTokenTransfer, processors: []OperationProcessorInterface{mockEffects, mockContractDeploy, mockSACEvents}, pool: pond.NewPool(runtime.NumCPU()), + networkPassphrase: network.TestNetworkPassphrase, } - // Test CollectAllTransactionData - precomputedData, allParticipants, err := indexer.CollectAllTransactionData(context.Background(), []ingest.LedgerTransaction{testTx}) + // Test ProcessLedgerTransactions + buffer := NewIndexerBuffer() + participantCount, err := indexer.ProcessLedgerTransactions(context.Background(), []ingest.LedgerTransaction{testTx}, buffer) // Assert results require.Error(t, err) assert.Contains(t, err.Error(), "processing effects state changes: effects error") - assert.Nil(t, precomputedData) - assert.Nil(t, allParticipants) + assert.Equal(t, 0, participantCount) // Verify mock expectations mockParticipants.AssertExpectations(t) @@ -578,253 +540,61 @@ func TestIndexer_CollectAllTransactionData(t *testing.T) { mockContractDeploy.AssertExpectations(t) mockSACEvents.AssertExpectations(t) }) -} - -func TestIndexer_ProcessTransactions(t *testing.T) { - t.Run("🟢 process with all existing accounts", func(t *testing.T) { - // Setup - precomputedData := []PrecomputedTransactionData{ - { - Transaction: testTx, - TxParticipants: set.NewSet("alice", "bob"), - OpsParticipants: map[int64]processors.OperationParticipants{ - 1: { - OpWrapper: &operation_processor.TransactionOperationWrapper{ - Index: 0, - Operation: createAccountOp, - Network: network.TestNetworkPassphrase, - LedgerSequence: 12345, - }, - Participants: set.NewSet("alice"), - }, - }, - StateChanges: []types.StateChange{ - {ToID: 1, AccountID: "alice", OperationID: 1, SortKey: "1-1"}, - }, - AllParticipants: set.NewSet("alice", "bob"), - }, - } - existingAccounts := set.NewSet("alice", "bob") - realBuffer := NewIndexerBuffer() - - // Create indexer - indexer := &Indexer{ - participantsProcessor: &MockParticipantsProcessor{}, - tokenTransferProcessor: &MockTokenTransferProcessor{}, - processors: []OperationProcessorInterface{}, - pool: pond.NewPool(runtime.NumCPU()), - } - // Test ProcessTransactions - err := indexer.ProcessTransactions(context.Background(), precomputedData, existingAccounts, realBuffer) - - // Assert - require.NoError(t, err) - - // Verify transactions for alice and bob - txParticipants := realBuffer.GetTransactionsParticipants() - txHash := "0102030000000000000000000000000000000000000000000000000000000000" - assert.True(t, txParticipants[txHash].Contains("alice"), "alice should be in tx participants") - assert.True(t, txParticipants[txHash].Contains("bob"), "bob should be in tx participants") - - // Verify transaction exists and has correct data - allTxs := realBuffer.GetTransactions() - require.Len(t, allTxs, 1, "should have 1 transaction") - assert.Equal(t, txHash, allTxs[0].Hash) - assert.Equal(t, uint32(12345), allTxs[0].LedgerNumber) - - // Verify operations for alice (only alice is in operation participants) - opParticipants := realBuffer.GetOperationsParticipants() - assert.True(t, opParticipants[int64(1)].Contains("alice"), "alice should be in operation 1 participants") - assert.False(t, opParticipants[int64(1)].Contains("bob"), "bob should NOT be in operation 1 participants") - - // Verify operation exists and has correct data - allOps := realBuffer.GetOperations() - require.Len(t, allOps, 1, "should have 1 operation") - assert.Equal(t, int64(1), allOps[0].ID) - assert.Equal(t, txHash, allOps[0].TxHash) + t.Run("🟢 multiple state changes per operation verify ordering", func(t *testing.T) { + // Create mocks + mockParticipants := &MockParticipantsProcessor{} + mockTokenTransfer := &MockTokenTransferProcessor{} + mockEffects := &MockOperationProcessor{} + mockContractDeploy := &MockOperationProcessor{} + mockSACEvents := &MockOperationProcessor{} - // Verify state changes - stateChanges := realBuffer.GetStateChanges() - require.Len(t, stateChanges, 1, "should have 1 state change") - assert.Equal(t, "alice", stateChanges[0].AccountID) - assert.Equal(t, int64(1), stateChanges[0].ToID) - assert.Equal(t, int64(1), stateChanges[0].OperationID) - assert.Equal(t, int64(1), stateChanges[0].StateChangeOrder, "first state change should have order 1") - }) + // Setup mock expectations + txParticipants := set.NewSet("alice") + mockParticipants.On("GetTransactionParticipants", testTx).Return(txParticipants, nil) - t.Run("🟢 process with mixed existing/non-existing accounts", func(t *testing.T) { - // Setup - precomputedData := []PrecomputedTransactionData{ - { - Transaction: testTx, - TxParticipants: set.NewSet("alice", "bob", "charlie"), - OpsParticipants: map[int64]processors.OperationParticipants{}, - StateChanges: []types.StateChange{ - {ToID: 1, AccountID: "alice", OperationID: 0, SortKey: "0-1"}, - {ToID: 2, AccountID: "charlie", OperationID: 0, SortKey: "0-2"}, + opParticipants := map[int64]processors.OperationParticipants{ + 1: { + OpWrapper: &operation_processor.TransactionOperationWrapper{ + Index: 0, + Operation: createAccountOp, + Network: network.TestNetworkPassphrase, + LedgerSequence: 12345, }, - AllParticipants: set.NewSet("alice", "bob", "charlie"), - }, - } - existingAccounts := set.NewSet("alice", "bob") // charlie doesn't exist - realBuffer := NewIndexerBuffer() - - // Create indexer - indexer := &Indexer{ - participantsProcessor: &MockParticipantsProcessor{}, - tokenTransferProcessor: &MockTokenTransferProcessor{}, - processors: []OperationProcessorInterface{}, - pool: pond.NewPool(runtime.NumCPU()), - } - - // Test ProcessTransactions - err := indexer.ProcessTransactions(context.Background(), precomputedData, existingAccounts, realBuffer) - - // Assert - require.NoError(t, err) - - // Verify transactions for alice and bob - txParticipants := realBuffer.GetTransactionsParticipants() - txHash := "0102030000000000000000000000000000000000000000000000000000000000" - assert.True(t, txParticipants[txHash].Contains("alice"), "alice should be in tx participants") - assert.True(t, txParticipants[txHash].Contains("bob"), "bob should be in tx participants") - assert.False(t, txParticipants[txHash].Contains("charlie"), "charlie should NOT be in tx participants (doesn't exist)") - - // Verify transaction exists - allTxs := realBuffer.GetTransactions() - require.Len(t, allTxs, 1, "should have 1 transaction") - assert.Equal(t, txHash, allTxs[0].Hash) - - // Verify state changes - only alice should have one (charlie's should be skipped) - stateChanges := realBuffer.GetStateChanges() - require.Len(t, stateChanges, 1, "should have 1 state change (alice only, charlie skipped)") - assert.Equal(t, "alice", stateChanges[0].AccountID, "only alice's state change should be present") - assert.Equal(t, int64(1), stateChanges[0].ToID) - assert.Equal(t, int64(0), stateChanges[0].OperationID, "fee state change (OperationID=0)") - assert.Equal(t, int64(1), stateChanges[0].StateChangeOrder, "fee state changes get order 1") - }) - - t.Run("🟢 process with no existing accounts", func(t *testing.T) { - // Setup - precomputedData := []PrecomputedTransactionData{ - { - Transaction: testTx, - TxParticipants: set.NewSet("alice", "bob"), - OpsParticipants: map[int64]processors.OperationParticipants{}, - StateChanges: []types.StateChange{}, - AllParticipants: set.NewSet("alice", "bob"), + Participants: set.NewSet("alice"), }, } - existingAccounts := set.NewSet[string]() // No existing accounts - realBuffer := NewIndexerBuffer() - - // Create indexer - indexer := &Indexer{ - participantsProcessor: &MockParticipantsProcessor{}, - tokenTransferProcessor: &MockTokenTransferProcessor{}, - processors: []OperationProcessorInterface{}, - pool: pond.NewPool(runtime.NumCPU()), - } - - // Test ProcessTransactions - err := indexer.ProcessTransactions(context.Background(), precomputedData, existingAccounts, realBuffer) - - // Assert - require.NoError(t, err) - - // Verify no transactions - assert.Equal(t, 0, realBuffer.GetNumberOfTransactions(), "should have 0 transactions") - - // Verify no state changes - stateChanges := realBuffer.GetStateChanges() - assert.Len(t, stateChanges, 0, "should have 0 state changes") - }) - - t.Run("🟢 empty precomputed data", func(t *testing.T) { - // Setup - precomputedData := []PrecomputedTransactionData{} - existingAccounts := set.NewSet("alice") - realBuffer := NewIndexerBuffer() - - // Create indexer - indexer := &Indexer{ - participantsProcessor: &MockParticipantsProcessor{}, - tokenTransferProcessor: &MockTokenTransferProcessor{}, - processors: []OperationProcessorInterface{}, - pool: pond.NewPool(runtime.NumCPU()), - } - - // Test ProcessTransactions - err := indexer.ProcessTransactions(context.Background(), precomputedData, existingAccounts, realBuffer) - - // Assert - require.NoError(t, err) + mockParticipants.On("GetOperationsParticipants", testTx).Return(opParticipants, nil) - // Verify buffer is empty with no precomputed data - assert.Equal(t, 0, realBuffer.GetNumberOfTransactions(), "should have 0 transactions") - stateChanges := realBuffer.GetStateChanges() - assert.Len(t, stateChanges, 0, "should have 0 state changes") - }) + mockEffects.On("ProcessOperation", mock.Anything, mock.Anything).Return([]types.StateChange{ + {ToID: 1, AccountID: "alice", OperationID: 1, SortKey: "1-1"}, + {ToID: 2, AccountID: "alice", OperationID: 1, SortKey: "1-2"}, + {ToID: 3, AccountID: "alice", OperationID: 1, SortKey: "1-3"}, + }, nil) + mockContractDeploy.On("ProcessOperation", mock.Anything, mock.Anything).Return([]types.StateChange{}, nil) + mockSACEvents.On("ProcessOperation", mock.Anything, mock.Anything).Return([]types.StateChange{}, nil) - t.Run("🟢 multiple state changes per operation verify ordering", func(t *testing.T) { - // Setup - precomputedData := []PrecomputedTransactionData{ - { - Transaction: testTx, - TxParticipants: set.NewSet("alice"), - OpsParticipants: map[int64]processors.OperationParticipants{ - 1: { - OpWrapper: &operation_processor.TransactionOperationWrapper{ - Index: 0, - Operation: createAccountOp, - Network: network.TestNetworkPassphrase, - LedgerSequence: 12345, - }, - Participants: set.NewSet("alice"), - }, - }, - StateChanges: []types.StateChange{ - {ToID: 1, AccountID: "alice", OperationID: 1, SortKey: "1-1"}, - {ToID: 2, AccountID: "alice", OperationID: 1, SortKey: "1-2"}, - {ToID: 3, AccountID: "alice", OperationID: 1, SortKey: "1-3"}, - }, - AllParticipants: set.NewSet("alice"), - }, - } - existingAccounts := set.NewSet("alice") - realBuffer := NewIndexerBuffer() + mockTokenTransfer.On("ProcessTransaction", mock.Anything, testTx).Return([]types.StateChange{}, nil) // Create indexer indexer := &Indexer{ - participantsProcessor: &MockParticipantsProcessor{}, - tokenTransferProcessor: &MockTokenTransferProcessor{}, - processors: []OperationProcessorInterface{}, + participantsProcessor: mockParticipants, + tokenTransferProcessor: mockTokenTransfer, + processors: []OperationProcessorInterface{mockEffects, mockContractDeploy, mockSACEvents}, pool: pond.NewPool(runtime.NumCPU()), + networkPassphrase: network.TestNetworkPassphrase, } - // Test ProcessTransactions - err := indexer.ProcessTransactions(context.Background(), precomputedData, existingAccounts, realBuffer) + // Test ProcessLedgerTransactions + buffer := NewIndexerBuffer() + participantCount, err := indexer.ProcessLedgerTransactions(context.Background(), []ingest.LedgerTransaction{testTx}, buffer) // Assert require.NoError(t, err) - - // Verify transaction participants - txParticipants := realBuffer.GetTransactionsParticipants() - require.Len(t, txParticipants, 1, "should have 1 transaction with participants") - - // Verify operation participants - opParticipants := realBuffer.GetOperationsParticipants() - require.Len(t, opParticipants, 1, "should have 1 operation with participants") - assert.True(t, opParticipants[int64(1)].Contains("alice"), "alice should be in operation 1 participants") - - // Verify operation exists - allOps := realBuffer.GetOperations() - require.Len(t, allOps, 1, "should have 1 operation") - assert.Equal(t, int64(1), allOps[0].ID) + assert.Equal(t, 1, participantCount) // Verify state changes with correct ordering - stateChanges := realBuffer.GetStateChanges() + stateChanges := buffer.GetStateChanges() require.Len(t, stateChanges, 3, "should have 3 state changes") // Verify first state change @@ -844,6 +614,13 @@ func TestIndexer_ProcessTransactions(t *testing.T) { assert.Equal(t, int64(3), stateChanges[2].ToID) assert.Equal(t, int64(1), stateChanges[2].OperationID) assert.Equal(t, int64(3), stateChanges[2].StateChangeOrder, "third state change should have order 3") + + // Verify mock expectations + mockParticipants.AssertExpectations(t) + mockTokenTransfer.AssertExpectations(t) + mockEffects.AssertExpectations(t) + mockContractDeploy.AssertExpectations(t) + mockSACEvents.AssertExpectations(t) }) } diff --git a/internal/indexer/processors/token_transfer.go b/internal/indexer/processors/token_transfer.go index 22ee07587..b04300317 100644 --- a/internal/indexer/processors/token_transfer.go +++ b/internal/indexer/processors/token_transfer.go @@ -99,7 +99,9 @@ func (p *TokenTransferProcessor) ProcessTransaction(ctx context.Context, tx inge if err != nil { return nil, fmt.Errorf("processing fee events for transaction hash: %s, err: %w", txHash, err) } - stateChanges = append(stateChanges, feeChange) + if feeChange.AccountID != "" { + stateChanges = append(stateChanges, feeChange) + } for _, e := range txEvents.OperationEvents { meta := e.GetMeta() diff --git a/internal/indexer/processors/utils.go b/internal/indexer/processors/utils.go index 7ca2b6540..6764c7691 100644 --- a/internal/indexer/processors/utils.go +++ b/internal/indexer/processors/utils.go @@ -8,6 +8,7 @@ import ( "github.com/stellar/go/ingest" "github.com/stellar/go/strkey" "github.com/stellar/go/toid" + "github.com/stellar/go/txnbuild" "github.com/stellar/go/xdr" "github.com/stellar/wallet-backend/internal/indexer/types" @@ -83,33 +84,63 @@ func safeStringFromDetails(details map[string]any, key string) (string, error) { return "", fmt.Errorf("invalid %s value", key) } -func ConvertTransaction(transaction *ingest.LedgerTransaction) (*types.Transaction, error) { - envelopeXDR, err := xdr.MarshalBase64(transaction.Envelope) +func ConvertTransaction(transaction *ingest.LedgerTransaction, skipTxMeta bool, skipTxEnvelope bool, networkPassphrase string) (*types.Transaction, error) { + var envelopeXDR *string + envelopeXDRStr, err := xdr.MarshalBase64(transaction.Envelope) if err != nil { return nil, fmt.Errorf("marshalling transaction envelope: %w", err) } + if !skipTxEnvelope { + envelopeXDR = &envelopeXDRStr + } + resultXDR, err := xdr.MarshalBase64(transaction.Result) if err != nil { return nil, fmt.Errorf("marshalling transaction result: %w", err) } - metaXDR, err := xdr.MarshalBase64(transaction.UnsafeMeta) + var metaXDR *string + if !skipTxMeta { + metaXDRStr, marshalErr := xdr.MarshalBase64(transaction.UnsafeMeta) + if marshalErr != nil { + return nil, fmt.Errorf("marshalling transaction meta: %w", marshalErr) + } + metaXDR = &metaXDRStr + } + + // Calculate inner transaction hash + genericTx, err := txnbuild.TransactionFromXDR(envelopeXDRStr) if err != nil { - return nil, fmt.Errorf("marshalling transaction meta: %w", err) + return nil, fmt.Errorf("deserializing envelope xdr: %w", err) + } + + var innerTx *txnbuild.Transaction + if feeBumpTx, ok := genericTx.FeeBump(); ok { + innerTx = feeBumpTx.InnerTransaction() + } else if tx, ok := genericTx.Transaction(); ok { + innerTx = tx + } else { + return nil, fmt.Errorf("transaction is neither fee bump nor inner transaction") + } + + innerTxHash, err := innerTx.HashHex(networkPassphrase) + if err != nil { + return nil, fmt.Errorf("generating inner hash hex: %w", err) } ledgerSequence := transaction.Ledger.LedgerSequence() transactionID := toid.New(int32(ledgerSequence), int32(transaction.Index), 0).ToInt64() return &types.Transaction{ - ToID: transactionID, - Hash: transaction.Hash.HexString(), - LedgerCreatedAt: transaction.Ledger.ClosedAt(), - EnvelopeXDR: envelopeXDR, - ResultXDR: resultXDR, - MetaXDR: metaXDR, - LedgerNumber: ledgerSequence, + ToID: transactionID, + Hash: transaction.Hash.HexString(), + LedgerCreatedAt: transaction.Ledger.ClosedAt(), + EnvelopeXDR: envelopeXDR, + ResultXDR: resultXDR, + MetaXDR: metaXDR, + LedgerNumber: ledgerSequence, + InnerTransactionHash: innerTxHash, }, nil } diff --git a/internal/indexer/processors/utils_test.go b/internal/indexer/processors/utils_test.go index ae02b224b..96ddf49e8 100644 --- a/internal/indexer/processors/utils_test.go +++ b/internal/indexer/processors/utils_test.go @@ -32,17 +32,49 @@ func Test_ConvertTransaction(t *testing.T) { ingestTx, err := ledgerTxReader.Read() require.NoError(t, err) - gotDataTx, err := ConvertTransaction(&ingestTx) + gotDataTx, err := ConvertTransaction(&ingestTx, false, false, network.TestNetworkPassphrase) require.NoError(t, err) + metaXDR := unsafeMetaXDRStr + envelopeXDR := envelopeXDRStr wantDataTx := &types.Transaction{ - Hash: "64eb94acc50eefc323cea80387fdceefc31466cc3a69eb8d2b312e0b5c3c62f0", - ToID: 20929375637504, - EnvelopeXDR: envelopeXDRStr, - ResultXDR: txResultPairXDRStr, - MetaXDR: unsafeMetaXDRStr, - LedgerNumber: 4873, - LedgerCreatedAt: time.Date(2025, time.June, 19, 0, 3, 16, 0, time.UTC), + Hash: "64eb94acc50eefc323cea80387fdceefc31466cc3a69eb8d2b312e0b5c3c62f0", + ToID: 20929375637504, + EnvelopeXDR: &envelopeXDR, + ResultXDR: txResultPairXDRStr, + MetaXDR: &metaXDR, + LedgerNumber: 4873, + LedgerCreatedAt: time.Date(2025, time.June, 19, 0, 3, 16, 0, time.UTC), + InnerTransactionHash: "afaef8a1b657ad5d2360cc001eb31b763bfd3430cba20273d49ff44be2a2152e", + } + assert.Equal(t, wantDataTx, gotDataTx) +} + +func Test_ConvertTransaction_SkipTxEnvelope(t *testing.T) { + var lcm xdr.LedgerCloseMeta + err := xdr.SafeUnmarshalBase64(ledgerCloseMetaXDR, &lcm) + require.NoError(t, err) + + ledgerTxReader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(network.TestNetworkPassphrase, lcm) + require.NoError(t, err) + ingestTx, err := ledgerTxReader.Read() + require.NoError(t, err) + + // skipTxEnvelope = true + gotDataTx, err := ConvertTransaction(&ingestTx, false, true, network.TestNetworkPassphrase) + require.NoError(t, err) + + metaXDR := unsafeMetaXDRStr + // envelopeXDR should be nil + wantDataTx := &types.Transaction{ + Hash: "64eb94acc50eefc323cea80387fdceefc31466cc3a69eb8d2b312e0b5c3c62f0", + ToID: 20929375637504, + EnvelopeXDR: nil, + ResultXDR: txResultPairXDRStr, + MetaXDR: &metaXDR, + LedgerNumber: 4873, + LedgerCreatedAt: time.Date(2025, time.June, 19, 0, 3, 16, 0, time.UTC), + InnerTransactionHash: "afaef8a1b657ad5d2360cc001eb31b763bfd3430cba20273d49ff44be2a2152e", } assert.Equal(t, wantDataTx, gotDataTx) } diff --git a/internal/indexer/types/types.go b/internal/indexer/types/types.go index b51e9f8ad..232b6ecc4 100644 --- a/internal/indexer/types/types.go +++ b/internal/indexer/types/types.go @@ -91,9 +91,9 @@ type AccountWithOperationID struct { type Transaction struct { Hash string `json:"hash,omitempty" db:"hash"` ToID int64 `json:"toId,omitempty" db:"to_id"` - EnvelopeXDR string `json:"envelopeXdr,omitempty" db:"envelope_xdr"` + EnvelopeXDR *string `json:"envelopeXdr,omitempty" db:"envelope_xdr"` ResultXDR string `json:"resultXdr,omitempty" db:"result_xdr"` - MetaXDR string `json:"metaXdr,omitempty" db:"meta_xdr"` + MetaXDR *string `json:"metaXdr,omitempty" db:"meta_xdr"` LedgerNumber uint32 `json:"ledgerNumber,omitempty" db:"ledger_number"` LedgerCreatedAt time.Time `json:"ledgerCreatedAt,omitempty" db:"ledger_created_at"` IngestedAt time.Time `json:"ingestedAt,omitempty" db:"ingested_at"` @@ -101,6 +101,10 @@ type Transaction struct { Operations []Operation `json:"operations,omitempty"` Accounts []Account `json:"accounts,omitempty"` StateChanges []StateChange `json:"stateChanges,omitempty"` + // InnerTransactionHash is the hash of the inner transaction for fee bump transactions, + // or the transaction hash for regular transactions. + // This field is transient and not stored in the database. + InnerTransactionHash string `json:"innerTransactionHash,omitempty" db:"-"` } type TransactionWithCursor struct { diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index ce2bc4bc9..6f3df4944 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -35,11 +35,16 @@ const ( // LedgerBackendType represents the type of ledger backend to use type LedgerBackendType string +// IngestionMode represents the mode of ingestion to use +type IngestionMode string + const ( // LedgerBackendTypeRPC uses RPC to fetch ledgers LedgerBackendTypeRPC LedgerBackendType = "rpc" // LedgerBackendTypeDatastore uses cloud storage (S3/GCS) to fetch ledgers LedgerBackendTypeDatastore LedgerBackendType = "datastore" + IngestionModeLive IngestionMode = "live" + IngestionModeBackfill IngestionMode = "backfill" ) // StorageBackendConfig holds configuration for the datastore-based ledger backend @@ -49,27 +54,47 @@ type StorageBackendConfig struct { } type Configs struct { - DatabaseURL string - RedisHost string - RedisPort int - ServerPort int - LedgerCursorName string - StartLedger int - EndLedger int - LogLevel logrus.Level - AppTracker apptracker.AppTracker - RPCURL string - Network string - NetworkPassphrase string - GetLedgersLimit int - AdminPort int - AccountTokensCursorName string - ArchiveURL string - CheckpointFrequency int + IngestionMode string + LatestLedgerCursorName string + OldestLedgerCursorName string + DatabaseURL string + RedisHost string + RedisPort int + ServerPort int + StartLedger int + EndLedger int + LogLevel logrus.Level + AppTracker apptracker.AppTracker + RPCURL string + Network string + NetworkPassphrase string + GetLedgersLimit int + AdminPort int + ArchiveURL string + CheckpointFrequency int // LedgerBackendType specifies which backend to use for fetching ledgers LedgerBackendType LedgerBackendType // DatastoreConfigPath is the path to the TOML config file for datastore backend DatastoreConfigPath string + // SkipTxMeta skips storing transaction metadata (meta_xdr) to reduce storage space + SkipTxMeta bool + // SkipTxEnvelope skips storing transaction envelope (envelope_xdr) to reduce storage space + SkipTxEnvelope bool + // EnableParticipantFiltering controls whether to filter ingested data by pre-registered accounts. + // When false (default), all data is stored. When true, only data for pre-registered accounts is stored. + EnableParticipantFiltering bool + // BackfillWorkers limits concurrent batch processing during backfill. + // Defaults to runtime.NumCPU(). Lower values reduce RAM usage. + BackfillWorkers int + // BackfillBatchSize is the number of ledgers processed per batch during backfill. + // Defaults to 250. Lower values reduce RAM usage at cost of more DB transactions. + BackfillBatchSize int + // BackfillDBInsertBatchSize is the number of ledgers to process before flushing to DB. + // Defaults to 50. Lower values reduce RAM usage at cost of more DB transactions. + BackfillDBInsertBatchSize int + // CatchupThreshold is the number of ledgers behind network tip that triggers fast catchup. + // Defaults to 100. + CatchupThreshold int } func Ingest(cfg Configs) error { @@ -88,15 +113,37 @@ func Ingest(cfg Configs) error { } func setupDeps(cfg Configs) (services.IngestService, error) { - dbConnectionPool, err := db.OpenDBConnectionPool(cfg.DatabaseURL) - if err != nil { - return nil, fmt.Errorf("connecting to the database: %w", err) + ctx := context.Background() + + var dbConnectionPool db.ConnectionPool + var err error + switch cfg.IngestionMode { + // Use optimized connection pool for backfill mode with async commit and increased work_mem + case string(IngestionModeBackfill): + dbConnectionPool, err = db.OpenDBConnectionPoolForBackfill(cfg.DatabaseURL) + if err != nil { + return nil, fmt.Errorf("connecting to the database (backfill mode): %w", err) + } + + // Disable FK constraint checking for faster inserts (requires elevated privileges) + if fkErr := db.ConfigureBackfillSession(ctx, dbConnectionPool); fkErr != nil { + log.Ctx(ctx).Warnf("Could not disable FK checks (may require superuser privileges): %v", fkErr) + // Continue anyway - other optimizations (async commit, work_mem) still apply + } else { + log.Ctx(ctx).Info("Backfill session configured: FK checks disabled, async commit enabled, work_mem=256MB") + } + default: + dbConnectionPool, err = db.OpenDBConnectionPool(cfg.DatabaseURL) + if err != nil { + return nil, fmt.Errorf("connecting to the database: %w", err) + } } - db, err := dbConnectionPool.SqlxDB(context.Background()) + sqlxDB, err := dbConnectionPool.SqlxDB(ctx) if err != nil { return nil, fmt.Errorf("getting sqlx db: %w", err) } - metricsService := metrics.NewMetricsService(db) + + metricsService := metrics.NewMetricsService(sqlxDB) models, err := data.NewModels(dbConnectionPool, metricsService) if err != nil { return nil, fmt.Errorf("creating models: %w", err) @@ -148,8 +195,36 @@ func setupDeps(cfg Configs) (services.IngestService, error) { return nil, fmt.Errorf("instantiating account token service: %w", err) } - ingestService, err := services.NewIngestService( - models, cfg.LedgerCursorName, cfg.AccountTokensCursorName, cfg.AppTracker, rpcService, ledgerBackend, chAccStore, accountTokenService, contractMetadataService, metricsService, cfg.GetLedgersLimit, cfg.Network, cfg.NetworkPassphrase, archive) + // Create a factory function for parallel backfill (each batch needs its own backend) + ledgerBackendFactory := func(ctx context.Context) (ledgerbackend.LedgerBackend, error) { + return NewLedgerBackend(ctx, cfg) + } + + ingestService, err := services.NewIngestService(services.IngestServiceConfig{ + IngestionMode: cfg.IngestionMode, + Models: models, + LatestLedgerCursorName: cfg.LatestLedgerCursorName, + OldestLedgerCursorName: cfg.OldestLedgerCursorName, + AppTracker: cfg.AppTracker, + RPCService: rpcService, + LedgerBackend: ledgerBackend, + LedgerBackendFactory: ledgerBackendFactory, + ChannelAccountStore: chAccStore, + AccountTokenService: accountTokenService, + ContractMetadataService: contractMetadataService, + MetricsService: metricsService, + GetLedgersLimit: cfg.GetLedgersLimit, + Network: cfg.Network, + NetworkPassphrase: cfg.NetworkPassphrase, + Archive: archive, + SkipTxMeta: cfg.SkipTxMeta, + SkipTxEnvelope: cfg.SkipTxEnvelope, + EnableParticipantFiltering: cfg.EnableParticipantFiltering, + BackfillWorkers: cfg.BackfillWorkers, + BackfillBatchSize: cfg.BackfillBatchSize, + BackfillDBInsertBatchSize: cfg.BackfillDBInsertBatchSize, + CatchupThreshold: cfg.CatchupThreshold, + }) if err != nil { return nil, fmt.Errorf("instantiating ingest service: %w", err) } diff --git a/internal/integrationtests/accounts_register_test.go b/internal/integrationtests/accounts_register_test.go new file mode 100644 index 000000000..b7b9ae7f7 --- /dev/null +++ b/internal/integrationtests/accounts_register_test.go @@ -0,0 +1,87 @@ +package integrationtests + +import ( + "context" + + "github.com/stretchr/testify/suite" + + "github.com/stellar/go/keypair" + + "github.com/stellar/wallet-backend/internal/integrationtests/infrastructure" + "github.com/stellar/wallet-backend/pkg/wbclient/types" +) + +type AccountRegisterTestSuite struct { + suite.Suite + testEnv *infrastructure.TestEnvironment +} + +func (suite *AccountRegisterTestSuite) TestParticipantFiltering() { + ctx := context.Background() + client := suite.testEnv.WBClient + + // 1. Enable participant filtering + err := suite.testEnv.RestartIngestContainer(ctx, map[string]string{ + "ENABLE_PARTICIPANT_FILTERING": "true", + }) + suite.Require().NoError(err) + + // 2. Create a new random account (unregistered) + unregisteredKP := keypair.MustRandom() + + // Fund the account (this creates an operation on the network) + suite.testEnv.Containers.CreateAndFundAccounts(ctx, suite.T(), []*keypair.Full{unregisteredKP}) + + // Wait for the ledger to be ingested + suite.testEnv.WaitForLedgers(ctx, 2) + + // 3. Verify operations are NOT stored for unregistered account + limit := int32(10) + ops, err := client.GetAccountOperations(ctx, unregisteredKP.Address(), &limit, nil, nil, nil) + suite.Require().NoError(err) + suite.Require().Empty(ops.Edges, "Expected no operations for unregistered account") + + // 4. Register the account + suite.registerAndVerify(ctx, unregisteredKP.Address()) + + // Check for duplicate registration + suite.verifyDuplicateRegistration(ctx, unregisteredKP.Address()) + + // 5. Perform a payment operation from the registered account to another account + // Execute payment from the registered account (now it should be tracked) + suite.testEnv.Containers.SubmitPaymentOp(ctx, suite.T(), unregisteredKP.Address(), "100") + + // Wait for ledger + suite.testEnv.WaitForLedgers(ctx, 2) + + // 6. Verify operations ARE stored now for the registered account + ops, err = client.GetAccountOperations(ctx, unregisteredKP.Address(), &limit, nil, nil, nil) + suite.Require().NoError(err) + suite.Require().NotEmpty(ops.Edges, "Expected operations for registered account") + + // Verify the payment operation was ingested correctly + suite.Require().Len(ops.Edges, 1, "Expected exactly one operation") + paymentOp := ops.Edges[0].Node + suite.Require().NotNil(paymentOp, "Operation node should not be nil") + suite.Require().Equal(types.OperationTypePayment, paymentOp.OperationType, "Expected PAYMENT operation type") +} + +func (suite *AccountRegisterTestSuite) registerAndVerify(ctx context.Context, address string) { + client := suite.testEnv.WBClient + account, err := client.RegisterAccount(ctx, address) + suite.Require().NoError(err) + suite.Require().True(account.Success) + suite.Require().Equal(address, account.Account.Address) + + // Fetch the address to make sure it was registered + fetchedAccount, err := client.GetAccountByAddress(ctx, address) + suite.Require().NoError(err) + suite.Require().Equal(address, fetchedAccount.Address) +} + +func (suite *AccountRegisterTestSuite) verifyDuplicateRegistration(ctx context.Context, address string) { + client := suite.testEnv.WBClient + _, err := client.RegisterAccount(ctx, address) + suite.Require().Error(err) + suite.Require().ErrorContains(err, "Account is already registered") +} diff --git a/internal/integrationtests/accounts_test.go b/internal/integrationtests/accounts_test.go deleted file mode 100644 index ae9742f53..000000000 --- a/internal/integrationtests/accounts_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package integrationtests - -import ( - "context" - - "github.com/stretchr/testify/suite" - - "github.com/stellar/wallet-backend/internal/integrationtests/infrastructure" -) - -type AccountRegisterTestSuite struct { - suite.Suite - testEnv *infrastructure.TestEnvironment -} - -func (suite *AccountRegisterTestSuite) TestAccountRegistration() { - ctx := context.Background() - - client := suite.testEnv.WBClient - addresses := []string{ - suite.testEnv.PrimaryAccountKP.Address(), - suite.testEnv.SecondaryAccountKP.Address(), - suite.testEnv.SponsoredNewAccountKP.Address(), - } - for _, address := range addresses { - account, err := client.RegisterAccount(ctx, address) - suite.Require().NoError(err) - suite.Require().True(account.Success) - suite.Require().Equal(address, account.Account.Address) - - // Fetch the address to make sure it was registered - fetchedAccount, err := client.GetAccountByAddress(ctx, address) - suite.Require().NoError(err) - suite.Require().Equal(address, fetchedAccount.Address) - } -} - -func (suite *AccountRegisterTestSuite) TestDuplicateAccountRegistration() { - ctx := context.Background() - - client := suite.testEnv.WBClient - address := suite.testEnv.PrimaryAccountKP.Address() - _, err := client.RegisterAccount(ctx, address) - suite.Require().Error(err) - suite.Require().ErrorContains(err, "Account is already registered") -} diff --git a/internal/integrationtests/backfill_test.go b/internal/integrationtests/backfill_test.go new file mode 100644 index 000000000..863dd71f1 --- /dev/null +++ b/internal/integrationtests/backfill_test.go @@ -0,0 +1,148 @@ +// backfill_test.go tests the parallel execution of live ingestion and backfilling. +package integrationtests + +import ( + "context" + "time" + + "github.com/stellar/go/support/log" + "github.com/stretchr/testify/suite" + + "github.com/stellar/wallet-backend/internal/integrationtests/infrastructure" +) + +// BackfillTestSuite tests that live ingestion and backfilling can run concurrently. +type BackfillTestSuite struct { + suite.Suite + testEnv *infrastructure.TestEnvironment +} + +// TestParallelLiveAndBackfill validates that backfilling works alongside live ingestion. +func (suite *BackfillTestSuite) TestParallelLiveAndBackfill() { + ctx := context.Background() + containers := suite.testEnv.Containers + + // Phase 1: Record initial state (live ingest is already running) + log.Ctx(ctx).Info("Phase 1: Recording initial state") + + oldestLedger, err := containers.GetIngestCursor(ctx, "oldest_ingest_ledger") + suite.Require().NoError(err, "failed to get oldest_ingest_ledger cursor") + + latestLedger, err := containers.GetIngestCursor(ctx, "latest_ingest_ledger") + suite.Require().NoError(err, "failed to get latest_ingest_ledger cursor") + + // In standalone mode, checkpoint frequency is 8 + // Live started from some checkpoint, so oldest_ingest_ledger > 8 + // There are older ledgers (8 to oldest-1) that were never processed + firstCheckpoint := uint32(8) + + log.Ctx(ctx).Infof("Initial state: oldest=%d, latest=%d, firstCheckpoint=%d", + oldestLedger, latestLedger, firstCheckpoint) + + // Skip test if there's no backfill range available + if oldestLedger <= firstCheckpoint { + suite.T().Skipf("No backfill range available: oldest_ingest_ledger (%d) <= firstCheckpoint (%d)", + oldestLedger, firstCheckpoint) + return + } + + backfillEndLedger := oldestLedger - 1 // Just before where live started + log.Ctx(ctx).Infof("Backfill range: %d to %d", firstCheckpoint, backfillEndLedger) + + // Phase 2: Start backfill container alongside live + log.Ctx(ctx).Info("Phase 2: Starting backfill container alongside live ingest") + + backfillContainer, err := containers.StartBackfillContainer(ctx, firstCheckpoint, backfillEndLedger) + suite.Require().NoError(err, "failed to start backfill container") + defer func() { + if err := backfillContainer.Terminate(ctx); err != nil { + log.Ctx(ctx).Warnf("Failed to terminate backfill container: %v", err) + } + }() + + log.Ctx(ctx).Info("Backfill container started alongside live ingest") + + // Phase 3: Wait for backfill completion + log.Ctx(ctx).Info("Phase 3: Waiting for backfill completion") + + err = containers.WaitForBackfillCompletion(ctx, firstCheckpoint, 5*time.Minute) + suite.Require().NoError(err, "backfill did not complete successfully") + + log.Ctx(ctx).Info("Backfill completed successfully") + + // Phase 4: Validate both ran successfully + log.Ctx(ctx).Info("Phase 4: Validating cursors") + + newOldestLedger, err := containers.GetIngestCursor(ctx, "oldest_ingest_ledger") + suite.Require().NoError(err, "failed to get new oldest_ingest_ledger cursor") + suite.Assert().LessOrEqual(newOldestLedger, firstCheckpoint, + "backfill should have updated oldest cursor to first checkpoint") + + newLatestLedger, err := containers.GetIngestCursor(ctx, "latest_ingest_ledger") + suite.Require().NoError(err, "failed to get new latest_ingest_ledger cursor") + suite.Assert().GreaterOrEqual(newLatestLedger, latestLedger, + "live ingest should continue processing") + + log.Ctx(ctx).Infof("Cursor validation passed: oldest=%d (was %d), latest=%d (was %d)", + newOldestLedger, oldestLedger, newLatestLedger, latestLedger) + + // Phase 5: Validate setup transactions were backfilled + log.Ctx(ctx).Info("Phase 5: Validating setup transactions were backfilled") + + testAccounts := []string{ + containers.GetClientAuthKeyPair(ctx).Address(), + containers.GetPrimarySourceAccountKeyPair(ctx).Address(), + containers.GetSecondarySourceAccountKeyPair(ctx).Address(), + containers.GetDistributionAccountKeyPair(ctx).Address(), + containers.GetBalanceTestAccount1KeyPair(ctx).Address(), + containers.GetBalanceTestAccount2KeyPair(ctx).Address(), + } + + // Verify transactions exist for funded accounts in the backfilled range + for _, accountAddr := range testAccounts { + txCount, err := containers.GetTransactionCountForAccount(ctx, accountAddr, firstCheckpoint, backfillEndLedger) + suite.Require().NoError(err, "failed to get transaction count for account %s", accountAddr) + suite.Assert().Greater(txCount, 0, + "should have transactions for account %s in backfilled range", accountAddr) + } + + log.Ctx(ctx).Info("Transaction validation passed for all test accounts") + + // Verify CREATE_ACCOUNT operations for funded accounts exist in backfilled range + for _, accountAddr := range testAccounts { + hasOp, err := containers.HasOperationForAccount(ctx, accountAddr, "CREATE_ACCOUNT", firstCheckpoint, backfillEndLedger) + suite.Require().NoError(err, "failed to check CREATE_ACCOUNT operation for account %s", accountAddr) + suite.Assert().True(hasOp, + "should have CREATE_ACCOUNT operation for account %s in backfilled range", accountAddr) + } + + log.Ctx(ctx).Info("CREATE_ACCOUNT operation validation passed for all test accounts") + + // Verify CHANGE_TRUST operations for trustlines exist + hasTrustOp, err := containers.HasOperationForAccount(ctx, + containers.GetBalanceTestAccount1KeyPair(ctx).Address(), + "CHANGE_TRUST", firstCheckpoint, backfillEndLedger) + suite.Require().NoError(err, "failed to check CHANGE_TRUST operation") + suite.Assert().True(hasTrustOp, "should have CHANGE_TRUST operation for USDC trustline") + + log.Ctx(ctx).Info("CHANGE_TRUST operation validation passed") + + // Verify state_changes were created for account creations + stateChangeCount, err := containers.GetStateChangeCountForLedgerRange(ctx, firstCheckpoint, backfillEndLedger) + suite.Require().NoError(err, "failed to get state change count") + suite.Assert().Greater(stateChangeCount, 0, "should have state changes in backfilled range") + + log.Ctx(ctx).Infof("State change validation passed: %d state changes in backfilled range", stateChangeCount) + + // Verify participant linking tables were populated + for _, accountAddr := range testAccounts { + linkCount, err := containers.GetTransactionAccountLinkCount(ctx, accountAddr, firstCheckpoint, backfillEndLedger) + suite.Require().NoError(err, "failed to get transaction-account link count for %s", accountAddr) + suite.Assert().Greater(linkCount, 0, + "should have transaction-account links for %s", accountAddr) + } + + log.Ctx(ctx).Info("Transaction-account link validation passed for all test accounts") + + log.Ctx(ctx).Info("All backfill validations passed successfully!") +} diff --git a/internal/integrationtests/catchup_test.go b/internal/integrationtests/catchup_test.go new file mode 100644 index 000000000..39e8ad2e7 --- /dev/null +++ b/internal/integrationtests/catchup_test.go @@ -0,0 +1,102 @@ +// catchup_test.go tests the catchup backfilling during live ingestion. +package integrationtests + +import ( + "context" + "time" + + "github.com/stellar/go/support/log" + "github.com/stretchr/testify/suite" + + "github.com/stellar/wallet-backend/internal/integrationtests/infrastructure" +) + +// CatchupTestSuite tests the automatic catchup backfilling that occurs when +// the ingest service falls behind the network tip by more than the catchup threshold. +type CatchupTestSuite struct { + suite.Suite + testEnv *infrastructure.TestEnvironment +} + +// TestCatchupDuringLiveIngestion validates that when the ingest service is behind +// the network by more than the catchup threshold, it triggers parallel backfilling +// to catch up efficiently. +func (suite *CatchupTestSuite) TestCatchupDuringLiveIngestion() { + ctx := context.Background() + containers := suite.testEnv.Containers + + // Phase 1: Record initial state (live ingest is already running) + log.Ctx(ctx).Info("Phase 1: Recording initial state") + + initialOldest, err := containers.GetIngestCursor(ctx, "oldest_ingest_ledger") + suite.Require().NoError(err, "failed to get initial oldest_ingest_ledger cursor") + + initialLatest, err := containers.GetIngestCursor(ctx, "latest_ingest_ledger") + suite.Require().NoError(err, "failed to get initial latest_ingest_ledger cursor") + + log.Ctx(ctx).Infof("Initial state: oldest=%d, latest=%d", initialOldest, initialLatest) + + // Phase 2: Stop ingest container to simulate falling behind + log.Ctx(ctx).Info("Phase 2: Stopping ingest container") + + err = containers.StopIngestContainer(ctx) + suite.Require().NoError(err, "failed to stop ingest container") + + // Phase 3: Wait for network to advance beyond catchup threshold + // We'll use a low threshold of 5 ledgers for faster testing + log.Ctx(ctx).Info("Phase 3: Waiting for network to advance") + + targetLedger := initialLatest + 20 // Need to be behind by more than catchup threshold (5) + err = containers.WaitForNetworkAdvance(ctx, suite.testEnv.RPCService, targetLedger, 2*time.Minute) + suite.Require().NoError(err, "network did not advance to target ledger") + + log.Ctx(ctx).Infof("Network advanced to ledger %d", targetLedger) + + // Phase 4: Restart ingest container with low catchup threshold + log.Ctx(ctx).Info("Phase 4: Restarting ingest container with low catchup threshold") + + err = containers.RestartIngestContainer(ctx, map[string]string{ + "CATCHUP_THRESHOLD": "5", // Trigger catchup if behind by 5+ ledgers + }) + suite.Require().NoError(err, "failed to restart ingest container") + + // Phase 5: Wait for catchup to complete + log.Ctx(ctx).Info("Phase 5: Waiting for catchup to complete") + + err = containers.WaitForLatestLedgerToReach(ctx, targetLedger, 3*time.Minute) + suite.Require().NoError(err, "catchup did not complete in time") + + log.Ctx(ctx).Info("Catchup completed") + + // Phase 6: Validate results + log.Ctx(ctx).Info("Phase 6: Validating results") + + // Verify catchup was triggered by checking logs + logs, err := containers.GetIngestContainerLogs(ctx) + suite.Require().NoError(err, "failed to get ingest container logs") + suite.Assert().Contains(logs, "Doing optimized catchup to the tip", + "should see catchup triggered log message") + + // Verify oldest cursor stayed the same (catchup doesn't change oldest) + newOldest, err := containers.GetIngestCursor(ctx, "oldest_ingest_ledger") + suite.Require().NoError(err, "failed to get new oldest_ingest_ledger cursor") + suite.Assert().Equal(initialOldest, newOldest, + "oldest cursor should not change during catchup") + + // Verify latest cursor caught up + newLatest, err := containers.GetIngestCursor(ctx, "latest_ingest_ledger") + suite.Require().NoError(err, "failed to get new latest_ingest_ledger cursor") + suite.Assert().GreaterOrEqual(newLatest, targetLedger, + "should have caught up to target ledger") + + // Verify no gaps exist in the ledger range we caught up + gapCount, err := containers.GetLedgerGapCount(ctx, initialLatest, newLatest) + suite.Require().NoError(err, "failed to get ledger gap count") + suite.Assert().Equal(0, gapCount, + "should have no gaps after catchup") + + log.Ctx(ctx).Infof("Validation passed: oldest=%d, latest=%d (was %d), gaps=%d", + newOldest, newLatest, initialLatest, gapCount) + + log.Ctx(ctx).Info("All catchup validations passed successfully!") +} diff --git a/internal/integrationtests/data_validation_test.go b/internal/integrationtests/data_validation_test.go index 352a5eac9..377beb234 100644 --- a/internal/integrationtests/data_validation_test.go +++ b/internal/integrationtests/data_validation_test.go @@ -342,7 +342,7 @@ func (suite *DataValidationTestSuite) validatePaymentStateChanges(ctx context.Co stateChanges, err := suite.testEnv.WBClient.GetTransactionStateChanges(ctx, txHash, &first, nil, nil, nil) suite.Require().NoError(err, "failed to get transaction state changes") suite.Require().NotNil(stateChanges, "state changes should not be nil") - suite.Require().Len(stateChanges.Edges, 2, "should have exactly 2 state changes") + suite.Require().Len(stateChanges.Edges, 3, "should have exactly 2 state changes") for _, edge := range stateChanges.Edges { jsonBytes, err := json.MarshalIndent(edge.Node, "", " ") @@ -430,7 +430,7 @@ func (suite *DataValidationTestSuite) validateSponsoredAccountCreationStateChang stateChanges, err := suite.testEnv.WBClient.GetTransactionStateChanges(ctx, txHash, &first, nil, nil, nil) suite.Require().NoError(err, "failed to get transaction state changes") suite.Require().NotNil(stateChanges, "state changes should not be nil") - suite.Require().Len(stateChanges.Edges, 7, "should have exactly 7 total state changes") + suite.Require().Len(stateChanges.Edges, 8, "should have exactly 7 total state changes") for _, edge := range stateChanges.Edges { jsonBytes, err := json.MarshalIndent(edge.Node, "", " ") @@ -571,7 +571,7 @@ func (suite *DataValidationTestSuite) validateCustomAssetsStateChanges(ctx conte stateChanges, err := suite.testEnv.WBClient.GetTransactionStateChanges(ctx, txHash, &first, nil, nil, nil) suite.Require().NoError(err, "failed to get transaction state changes") suite.Require().NotNil(stateChanges, "state changes should not be nil") - suite.Require().Len(stateChanges.Edges, 25, "should have exactly 25 state changes") + suite.Require().Len(stateChanges.Edges, 26, "should have exactly 25 state changes") // Validate base fields for all state changes for _, edge := range stateChanges.Edges { @@ -772,7 +772,7 @@ func (suite *DataValidationTestSuite) validateAuthRequiredIssuerSetupStateChange stateChanges, err := suite.testEnv.WBClient.GetTransactionStateChanges(ctx, txHash, &first, nil, nil, nil) suite.Require().NoError(err, "failed to get transaction state changes") suite.Require().NotNil(stateChanges, "state changes should not be nil") - suite.Require().Len(stateChanges.Edges, 1, "should have exactly 1 state change") + suite.Require().Len(stateChanges.Edges, 2, "should have exactly 1 state change") // Validate base fields for all state changes for _, edge := range stateChanges.Edges { @@ -830,7 +830,7 @@ func (suite *DataValidationTestSuite) validateAuthRequiredAssetStateChanges(ctx stateChanges, err := suite.testEnv.WBClient.GetTransactionStateChanges(ctx, txHash, &first, nil, nil, nil) suite.Require().NoError(err, "failed to get transaction state changes") suite.Require().NotNil(stateChanges, "state changes should not be nil") - suite.Require().Len(stateChanges.Edges, 9, "should have exactly 9 state changes") + suite.Require().Len(stateChanges.Edges, 10, "should have exactly 9 state changes") // Validate base fields for all state changes for _, edge := range stateChanges.Edges { @@ -984,7 +984,7 @@ func (suite *DataValidationTestSuite) validateAccountMergeStateChanges(ctx conte stateChanges, err := suite.testEnv.WBClient.GetTransactionStateChanges(ctx, txHash, &first, nil, nil, nil) suite.Require().NoError(err, "failed to get transaction state changes") suite.Require().NotNil(stateChanges, "state changes should not be nil") - suite.Require().Len(stateChanges.Edges, 5, "should have exactly 5 state changes") + suite.Require().Len(stateChanges.Edges, 6, "should have exactly 5 state changes") for _, edge := range stateChanges.Edges { jsonBytes, err := json.MarshalIndent(edge.Node, "", " ") @@ -1104,7 +1104,7 @@ func (suite *DataValidationTestSuite) validateInvokeContractStateChanges(ctx con stateChanges, err := suite.testEnv.WBClient.GetTransactionStateChanges(ctx, txHash, &first, nil, nil, nil) suite.Require().NoError(err, "failed to get transaction state changes") suite.Require().NotNil(stateChanges, "state changes should not be nil") - suite.Require().Len(stateChanges.Edges, 2, "should have exactly 11 state changes") + suite.Require().Len(stateChanges.Edges, 3, "should have exactly 11 state changes") for _, edge := range stateChanges.Edges { jsonBytes, err := json.MarshalIndent(edge.Node, "", " ") @@ -1309,7 +1309,7 @@ func (suite *DataValidationTestSuite) validateClaimClaimableBalanceStateChanges( stateChanges, err := suite.testEnv.WBClient.GetTransactionStateChanges(ctx, txHash, &first, nil, nil, nil) suite.Require().NoError(err, "failed to get transaction state changes") suite.Require().NotNil(stateChanges, "state changes should not be nil") - suite.Require().Len(stateChanges.Edges, 2, "should have exactly 2 state changes") + suite.Require().Len(stateChanges.Edges, 3, "should have exactly 2 state changes") // Validate base fields for all state changes for _, edge := range stateChanges.Edges { @@ -1391,7 +1391,7 @@ func (suite *DataValidationTestSuite) validateClawbackClaimableBalanceStateChang stateChanges, err := suite.testEnv.WBClient.GetTransactionStateChanges(ctx, txHash, &first, nil, nil, nil) suite.Require().NoError(err, "failed to get transaction state changes") suite.Require().NotNil(stateChanges, "state changes should not be nil") - suite.Require().Len(stateChanges.Edges, 2, "should have exactly 2 state change") + suite.Require().Len(stateChanges.Edges, 3, "should have exactly 2 state change") // Validate base fields for all state changes for _, edge := range stateChanges.Edges { @@ -1473,7 +1473,7 @@ func (suite *DataValidationTestSuite) validateClearAuthFlagsStateChanges(ctx con stateChanges, err := suite.testEnv.WBClient.GetTransactionStateChanges(ctx, txHash, &first, nil, nil, nil) suite.Require().NoError(err, "failed to get transaction state changes") suite.Require().NotNil(stateChanges, "state changes should not be nil") - suite.Require().Len(stateChanges.Edges, 1, "should have exactly 1 state change") + suite.Require().Len(stateChanges.Edges, 2, "should have exactly 1 state change") // Validate base fields for all state changes for _, edge := range stateChanges.Edges { @@ -1549,7 +1549,7 @@ func (suite *DataValidationTestSuite) validateLiquidityPoolStateChanges(ctx cont stateChanges, err := suite.testEnv.WBClient.GetTransactionStateChanges(ctx, txHash, &first, nil, nil, nil) suite.Require().NoError(err, "failed to get transaction state changes") suite.Require().NotNil(stateChanges, "state changes should not be nil") - suite.Require().Len(stateChanges.Edges, 7, "should have exactly 7 state changes") + suite.Require().Len(stateChanges.Edges, 8, "should have exactly 7 state changes") // Validate base fields for all state changes for _, edge := range stateChanges.Edges { @@ -1695,7 +1695,7 @@ func (suite *DataValidationTestSuite) validateRevokeSponsorshipStateChanges(ctx stateChanges, err := suite.testEnv.WBClient.GetTransactionStateChanges(ctx, txHash, &first, nil, nil, nil) suite.Require().NoError(err, "failed to get transaction state changes") suite.Require().NotNil(stateChanges, "state changes should not be nil") - suite.Require().Len(stateChanges.Edges, 4, "should have exactly 4 state changes") + suite.Require().Len(stateChanges.Edges, 5, "should have exactly 4 state changes") // Validate base fields for all state changes for _, edge := range stateChanges.Edges { diff --git a/internal/integrationtests/infrastructure/account_setup.go b/internal/integrationtests/infrastructure/account_setup.go index 9bb87c6e3..269d19cb6 100644 --- a/internal/integrationtests/infrastructure/account_setup.go +++ b/internal/integrationtests/infrastructure/account_setup.go @@ -11,8 +11,8 @@ import ( "github.com/stretchr/testify/require" ) -// createAndFundAccounts creates and funds multiple accounts in a single transaction using the master account -func (s *SharedContainers) createAndFundAccounts(ctx context.Context, t *testing.T, accounts []*keypair.Full) { +// CreateAndFundAccounts creates and funds multiple accounts in a single transaction using the master account +func (s *SharedContainers) CreateAndFundAccounts(ctx context.Context, t *testing.T, accounts []*keypair.Full) { // Build CreateAccount operations for all accounts ops := make([]txnbuild.Operation, len(accounts)) for i, kp := range accounts { @@ -124,3 +124,20 @@ func (s *SharedContainers) createEURCTrustlines(ctx context.Context, t *testing. log.Ctx(ctx).Infof("🔗 Created EURC trustline for balance test account 1 %s: %s:%s", s.balanceTestAccount1KeyPair.Address(), eurcAsset.Code, eurcAsset.Issuer) } + +// SubmitPaymentOp executes a native XLM payment from the master account to a destination +func (s *SharedContainers) SubmitPaymentOp(ctx context.Context, t *testing.T, to string, amount string) { + ops := []txnbuild.Operation{ + &txnbuild.Payment{ + Destination: to, + Amount: amount, + Asset: txnbuild.NativeAsset{}, + SourceAccount: s.masterKeyPair.Address(), + }, + } + + _, err := executeClassicOperation(ctx, t, s, ops, []*keypair.Full{s.masterKeyPair}) + require.NoError(t, err, "failed to execute payment") + + log.Ctx(ctx).Infof("💸 Payment of %s XLM from %s to %s", amount, s.masterKeyPair.Address(), to) +} diff --git a/internal/integrationtests/infrastructure/backfill_helpers.go b/internal/integrationtests/infrastructure/backfill_helpers.go new file mode 100644 index 000000000..ec5532dc7 --- /dev/null +++ b/internal/integrationtests/infrastructure/backfill_helpers.go @@ -0,0 +1,252 @@ +// backfill_helpers.go provides helper functions for backfill integration testing. +package infrastructure + +import ( + "context" + "database/sql" + "fmt" + "io" + "strconv" + "time" + + "github.com/stellar/go/support/log" +) + +// GetIngestCursor retrieves a cursor value from the ingest_store table. +func (s *SharedContainers) GetIngestCursor(ctx context.Context, cursorName string) (uint32, error) { + dbURL, err := s.GetWalletDBConnectionString(ctx) + if err != nil { + return 0, fmt.Errorf("getting database connection string: %w", err) + } + db, err := sql.Open("postgres", dbURL) + if err != nil { + return 0, fmt.Errorf("opening database connection: %w", err) + } + defer db.Close() //nolint:errcheck + + var valueStr string + query := `SELECT value FROM ingest_store WHERE key = $1` + err = db.QueryRowContext(ctx, query, cursorName).Scan(&valueStr) + if err != nil { + return 0, fmt.Errorf("querying ingest_store for %s: %w", cursorName, err) + } + + value, err := strconv.ParseUint(valueStr, 10, 32) + if err != nil { + return 0, fmt.Errorf("parsing cursor value %s: %w", valueStr, err) + } + + return uint32(value), nil +} + +// GetTransactionCountForAccount counts transactions involving a specific account in a ledger range. +func (s *SharedContainers) GetTransactionCountForAccount(ctx context.Context, accountAddr string, startLedger, endLedger uint32) (int, error) { + dbURL, err := s.GetWalletDBConnectionString(ctx) + if err != nil { + return 0, fmt.Errorf("getting database connection string: %w", err) + } + db, err := sql.Open("postgres", dbURL) + if err != nil { + return 0, fmt.Errorf("opening database connection: %w", err) + } + defer db.Close() //nolint:errcheck + + var count int + query := ` + SELECT COUNT(DISTINCT t.hash) + FROM transactions t + INNER JOIN transactions_accounts ta ON t.hash = ta.tx_hash + WHERE ta.account_id = $1 + AND t.ledger_number BETWEEN $2 AND $3 + ` + err = db.QueryRowContext(ctx, query, accountAddr, startLedger, endLedger).Scan(&count) + if err != nil { + return 0, fmt.Errorf("counting transactions for account %s: %w", accountAddr, err) + } + + return count, nil +} + +// HasOperationForAccount checks if an operation of a specific type exists for an account in a ledger range. +func (s *SharedContainers) HasOperationForAccount(ctx context.Context, accountAddr, opType string, startLedger, endLedger uint32) (bool, error) { + dbURL, err := s.GetWalletDBConnectionString(ctx) + if err != nil { + return false, fmt.Errorf("getting database connection string: %w", err) + } + db, err := sql.Open("postgres", dbURL) + if err != nil { + return false, fmt.Errorf("opening database connection: %w", err) + } + defer db.Close() //nolint:errcheck + + var exists bool + query := ` + SELECT EXISTS ( + SELECT 1 FROM operations o + INNER JOIN operations_accounts oa ON o.id = oa.operation_id + WHERE oa.account_id = $1 + AND o.operation_type = $2 + AND o.ledger_number BETWEEN $3 AND $4 + ) + ` + err = db.QueryRowContext(ctx, query, accountAddr, opType, startLedger, endLedger).Scan(&exists) + if err != nil { + return false, fmt.Errorf("checking operation for account %s: %w", accountAddr, err) + } + + return exists, nil +} + +// GetTransactionAccountLinkCount counts transaction-account links for an account in a ledger range. +func (s *SharedContainers) GetTransactionAccountLinkCount(ctx context.Context, accountAddr string, startLedger, endLedger uint32) (int, error) { + dbURL, err := s.GetWalletDBConnectionString(ctx) + if err != nil { + return 0, fmt.Errorf("getting database connection string: %w", err) + } + db, err := sql.Open("postgres", dbURL) + if err != nil { + return 0, fmt.Errorf("opening database connection: %w", err) + } + defer db.Close() //nolint:errcheck + + var count int + query := ` + SELECT COUNT(*) + FROM transactions_accounts ta + INNER JOIN transactions t ON ta.tx_hash = t.hash + WHERE ta.account_id = $1 + AND t.ledger_number BETWEEN $2 AND $3 + ` + err = db.QueryRowContext(ctx, query, accountAddr, startLedger, endLedger).Scan(&count) + if err != nil { + return 0, fmt.Errorf("counting transaction-account links for %s: %w", accountAddr, err) + } + + return count, nil +} + +// GetStateChangeCountForLedgerRange counts state changes in a ledger range. +func (s *SharedContainers) GetStateChangeCountForLedgerRange(ctx context.Context, startLedger, endLedger uint32) (int, error) { + dbURL, err := s.GetWalletDBConnectionString(ctx) + if err != nil { + return 0, fmt.Errorf("getting database connection string: %w", err) + } + db, err := sql.Open("postgres", dbURL) + if err != nil { + return 0, fmt.Errorf("opening database connection: %w", err) + } + defer db.Close() //nolint:errcheck + + var count int + query := `SELECT COUNT(*) FROM state_changes WHERE ledger_number BETWEEN $1 AND $2` + err = db.QueryRowContext(ctx, query, startLedger, endLedger).Scan(&count) + if err != nil { + return 0, fmt.Errorf("counting state changes: %w", err) + } + + return count, nil +} + +// GetLedgerGapCount counts the number of missing ledgers (gaps) in the transactions table +// within the specified ledger range. Returns 0 if there are no gaps. +func (s *SharedContainers) GetLedgerGapCount(ctx context.Context, startLedger, endLedger uint32) (int, error) { + dbURL, err := s.GetWalletDBConnectionString(ctx) + if err != nil { + return 0, fmt.Errorf("getting database connection string: %w", err) + } + db, err := sql.Open("postgres", dbURL) + if err != nil { + return 0, fmt.Errorf("opening database connection: %w", err) + } + defer db.Close() //nolint:errcheck + + // Count the number of distinct ledgers that have transactions in the range + // Then compare with expected count to find gaps + var distinctLedgers int + query := ` + SELECT COUNT(DISTINCT ledger_number) + FROM transactions + WHERE ledger_number BETWEEN $1 AND $2 + ` + err = db.QueryRowContext(ctx, query, startLedger, endLedger).Scan(&distinctLedgers) + if err != nil { + return 0, fmt.Errorf("counting distinct ledgers: %w", err) + } + + // Note: Not all ledgers will have transactions, so we can't simply compare + // against expected range. Instead, use window function to find actual gaps. + var gapCount int + gapQuery := ` + WITH ledger_sequence AS ( + SELECT DISTINCT ledger_number + FROM transactions + WHERE ledger_number BETWEEN $1 AND $2 + ORDER BY ledger_number + ), + gaps AS ( + SELECT + ledger_number, + LEAD(ledger_number) OVER (ORDER BY ledger_number) AS next_ledger, + LEAD(ledger_number) OVER (ORDER BY ledger_number) - ledger_number - 1 AS gap_size + FROM ledger_sequence + ) + SELECT COALESCE(SUM(gap_size), 0) FROM gaps WHERE gap_size > 0 + ` + err = db.QueryRowContext(ctx, gapQuery, startLedger, endLedger).Scan(&gapCount) + if err != nil { + return 0, fmt.Errorf("counting ledger gaps: %w", err) + } + + return gapCount, nil +} + +// WaitForBackfillCompletion polls until the oldest_ingest_ledger cursor reaches the expected value. +func (s *SharedContainers) WaitForBackfillCompletion(ctx context.Context, expectedOldestLedger uint32, timeout time.Duration) error { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + timeoutChan := time.After(timeout) + + for { + select { + case <-ctx.Done(): + return fmt.Errorf("context cancelled: %w", ctx.Err()) + case <-timeoutChan: + return fmt.Errorf("backfill did not complete within %v", timeout) + case <-ticker.C: + oldest, err := s.GetIngestCursor(ctx, "oldest_ingest_ledger") + if err != nil { + log.Ctx(ctx).Warnf("Error getting oldest cursor during backfill wait: %v", err) + continue + } + + log.Ctx(ctx).Infof("Backfill progress: oldest_ingest_ledger=%d, expected=%d", oldest, expectedOldestLedger) + + if oldest <= expectedOldestLedger { + log.Ctx(ctx).Infof("Backfill completed: oldest_ingest_ledger=%d reached expected=%d", oldest, expectedOldestLedger) + return nil + } + + // Check if backfill container has exited (it exits on completion or error) + if s.BackfillContainer != nil { + state, stateErr := s.BackfillContainer.Container.State(ctx) + if stateErr == nil && !state.Running { + if state.ExitCode != 0 { + // Log container output for debugging + logs, logErr := s.BackfillContainer.Container.Logs(ctx) + if logErr == nil { + logBytes, _ := io.ReadAll(logs) //nolint:errcheck + logs.Close() //nolint:errcheck + log.Ctx(ctx).Errorf("Backfill container logs:\n%s", string(logBytes)) + } + return fmt.Errorf("backfill container exited with code %d", state.ExitCode) + } + // Container exited successfully, check cursor one more time + oldest, err = s.GetIngestCursor(ctx, "oldest_ingest_ledger") + if err == nil && oldest <= expectedOldestLedger { + return nil + } + } + } + } + } +} diff --git a/internal/integrationtests/infrastructure/containers.go b/internal/integrationtests/infrastructure/containers.go index 613b4cd8c..09cadd538 100644 --- a/internal/integrationtests/infrastructure/containers.go +++ b/internal/integrationtests/infrastructure/containers.go @@ -381,6 +381,7 @@ func createWalletDBContainer(ctx context.Context, testNetwork *testcontainers.Do // createWalletBackendIngestContainer creates a new wallet-backend ingest container using the shared network func createWalletBackendIngestContainer(ctx context.Context, name string, imageName string, testNetwork *testcontainers.DockerNetwork, clientAuthKeyPair *keypair.Full, distributionAccountKeyPair *keypair.Full, + extraEnv map[string]string, ) (*TestContainer, error) { // Prepare container request containerRequest := testcontainers.ContainerRequest{ @@ -403,6 +404,11 @@ func createWalletBackendIngestContainer(ctx context.Context, name string, imageN "ARCHIVE_URL": "http://stellar-core:1570", "CHECKPOINT_FREQUENCY": "8", "GET_LEDGERS_LIMIT": "200", + "SKIP_TX_META": "false", + "SKIP_TX_ENVELOPE": "false", + "LEDGER_BACKEND_TYPE": "rpc", + "START_LEDGER": "0", + "END_LEDGER": "0", "NETWORK_PASSPHRASE": networkPassphrase, "CLIENT_AUTH_PUBLIC_KEYS": clientAuthKeyPair.Address(), "DISTRIBUTION_ACCOUNT_PUBLIC_KEY": distributionAccountKeyPair.Address(), @@ -415,6 +421,10 @@ func createWalletBackendIngestContainer(ctx context.Context, name string, imageN Networks: []string{testNetwork.Name}, } + for k, v := range extraEnv { + containerRequest.Env[k] = v + } + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: containerRequest, Reuse: true, @@ -503,6 +513,87 @@ func createWalletBackendAPIContainer(ctx context.Context, name string, imageName }, nil } +// backfillContainerCounter is used to generate unique container names for backfill containers +var backfillContainerCounter int64 + +// StartBackfillContainer creates and starts a new backfill container alongside the live ingest container. +// It processes historical ledgers in the specified range. +func (s *SharedContainers) StartBackfillContainer(ctx context.Context, startLedger, endLedger uint32) (*TestContainer, error) { + // Use a unique container name + backfillContainerCounter++ + containerName := fmt.Sprintf("wallet-backend-backfill-%d", backfillContainerCounter) + + // Prepare container request with backfill-specific configuration + containerRequest := testcontainers.ContainerRequest{ + Name: containerName, + Image: s.walletBackendImage, + Labels: map[string]string{ + "org.testcontainers.session-id": "wallet-backend-integration-tests", + }, + Entrypoint: []string{"sh", "-c"}, + Cmd: []string{ + "./wallet-backend ingest", // No migrations needed, DB already set up + }, + ExposedPorts: []string{fmt.Sprintf("%s/tcp", walletBackendContainerIngestPort)}, + Env: map[string]string{ + "RPC_URL": "http://stellar-rpc:8000", + "DATABASE_URL": "postgres://postgres@wallet-backend-db:5432/wallet-backend?sslmode=disable", + "PORT": walletBackendContainerIngestPort, + "LOG_LEVEL": "DEBUG", + "NETWORK": "standalone", + "ARCHIVE_URL": "http://stellar-core:1570", + "CHECKPOINT_FREQUENCY": "8", + "GET_LEDGERS_LIMIT": "5", + "SKIP_TX_META": "false", + "SKIP_TX_ENVELOPE": "false", + "LEDGER_BACKEND_TYPE": "rpc", + "NETWORK_PASSPHRASE": networkPassphrase, + // Backfill-specific configuration + "INGESTION_MODE": "backfill", + "START_LEDGER": fmt.Sprintf("%d", startLedger), + "END_LEDGER": fmt.Sprintf("%d", endLedger), + "CLIENT_AUTH_PUBLIC_KEYS": s.clientAuthKeyPair.Address(), + "DISTRIBUTION_ACCOUNT_PUBLIC_KEY": s.distributionAccountKeyPair.Address(), + "DISTRIBUTION_ACCOUNT_PRIVATE_KEY": s.distributionAccountKeyPair.Seed(), + "DISTRIBUTION_ACCOUNT_SIGNATURE_PROVIDER": "ENV", + "STELLAR_ENVIRONMENT": "integration-test", + "REDIS_HOST": redisContainerName, + "REDIS_PORT": "6379", + }, + Networks: []string{s.TestNetwork.Name}, + } + + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: containerRequest, + Reuse: false, // Don't reuse - each backfill run is unique + Started: true, + }) + if err != nil { + // If we got a container reference, print its logs + if container != nil { + logs, logErr := container.Logs(ctx) + if logErr == nil { + defer logs.Close() //nolint:errcheck + logBytes, _ := io.ReadAll(logs) //nolint:errcheck + log.Ctx(ctx).Errorf("Backfill container logs:\n%s", string(logBytes)) + } + } + return nil, fmt.Errorf("creating backfill container: %w", err) + } + + log.Ctx(ctx).Infof("🔄 Created Backfill container (ledgers %d-%d)", startLedger, endLedger) + + testContainer := &TestContainer{ + Container: container, + MappedPortStr: walletBackendContainerIngestPort, + } + + // Store reference for WaitForBackfillCompletion + s.BackfillContainer = testContainer + + return testContainer, nil +} + // triggerProtocolUpgrade triggers a protocol upgrade on Stellar Core func triggerProtocolUpgrade(ctx context.Context, container *TestContainer, version int) error { // Get container's HTTP endpoint diff --git a/internal/integrationtests/infrastructure/env_helpers.go b/internal/integrationtests/infrastructure/env_helpers.go new file mode 100644 index 000000000..62cb63d7b --- /dev/null +++ b/internal/integrationtests/infrastructure/env_helpers.go @@ -0,0 +1,21 @@ +package infrastructure + +import ( + "context" + "time" + + "github.com/stellar/go/support/log" +) + +// WaitForLedgers waits for a number of ledgers to close (approximated by sleep) +func (e *TestEnvironment) WaitForLedgers(ctx context.Context, ledgers int) { + // Assume 1 second per ledger for standalone + buffer + duration := time.Duration(ledgers) * 2 * time.Second + log.Ctx(ctx).Infof("⏳ Waiting for %d ledgers (%s)...", ledgers, duration) + time.Sleep(duration) +} + +// RestartIngestContainer restarts the ingest container with extra environment variables +func (e *TestEnvironment) RestartIngestContainer(ctx context.Context, extraEnv map[string]string) error { + return e.Containers.RestartIngestContainer(ctx, extraEnv) +} diff --git a/internal/integrationtests/infrastructure/main_setup.go b/internal/integrationtests/infrastructure/main_setup.go index ad9913321..cf9246d7f 100644 --- a/internal/integrationtests/infrastructure/main_setup.go +++ b/internal/integrationtests/infrastructure/main_setup.go @@ -14,6 +14,7 @@ import ( "github.com/stellar/go/support/log" "github.com/stellar/go/txnbuild" "github.com/stellar/go/xdr" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/network" @@ -27,6 +28,7 @@ import ( // SharedContainers provides shared container management for integration tests type SharedContainers struct { + walletBackendImage string // Docker infrastructure TestNetwork *testcontainers.DockerNetwork PostgresContainer *TestContainer @@ -35,6 +37,7 @@ type SharedContainers struct { WalletDBContainer *TestContainer RedisContainer *TestContainer WalletBackendContainer *WalletBackendContainer + BackfillContainer *TestContainer // Separate container for backfill testing // HTTP client for RPC calls (reusable, safe for concurrent use) httpClient *http.Client @@ -68,6 +71,13 @@ type SharedContainers struct { func (s *SharedContainers) initializeContainerInfrastructure(ctx context.Context) error { var err error + // Build wallet-backend image first so that by the time we start the containers, + // Stellar Core has advanced enough ledgers for the health check to pass. + s.walletBackendImage, err = ensureWalletBackendImage(ctx, walletBackendContainerTag) + if err != nil { + return fmt.Errorf("ensuring wallet backend image: %w", err) + } + // Create network s.TestNetwork, err = network.New(ctx) if err != nil { @@ -124,7 +134,7 @@ func (s *SharedContainers) setupTestAccounts(ctx context.Context, t *testing.T) s.sponsoredNewAccountKeyPair = keypair.MustRandom() // Create and fund all accounts in a single transaction - s.createAndFundAccounts(ctx, t, []*keypair.Full{ + s.CreateAndFundAccounts(ctx, t, []*keypair.Full{ s.clientAuthKeyPair, s.primarySourceAccountKeyPair, s.secondarySourceAccountKeyPair, @@ -261,7 +271,7 @@ func (s *SharedContainers) setupContracts(ctx context.Context, t *testing.T, dir // setupWalletBackend initializes wallet-backend database and service containers. // //nolint:unparam // t kept for API consistency with other setup methods -func (s *SharedContainers) setupWalletBackend(ctx context.Context, t *testing.T) error { +func (s *SharedContainers) setupWalletBackend(ctx context.Context) error { var err error // Start PostgreSQL for wallet-backend @@ -270,23 +280,22 @@ func (s *SharedContainers) setupWalletBackend(ctx context.Context, t *testing.T) return fmt.Errorf("creating wallet DB container: %w", err) } - // Build or verify wallet-backend Docker image - walletBackendImage, err := ensureWalletBackendImage(ctx, walletBackendContainerTag) - if err != nil { - return fmt.Errorf("ensuring wallet backend image: %w", err) - } - // Start wallet-backend ingest s.WalletBackendContainer = &WalletBackendContainer{} s.WalletBackendContainer.Ingest, err = createWalletBackendIngestContainer(ctx, walletBackendIngestContainerName, - walletBackendImage, s.TestNetwork, s.clientAuthKeyPair, s.distributionAccountKeyPair) + s.walletBackendImage, s.TestNetwork, s.clientAuthKeyPair, s.distributionAccountKeyPair, nil) if err != nil { return fmt.Errorf("creating wallet backend ingest container: %w", err) } + // Wait for ingest to catch up with RPC before starting API + if err := s.waitForIngestSync(ctx); err != nil { + return fmt.Errorf("waiting for ingest sync: %w", err) + } + // Start wallet-backend service s.WalletBackendContainer.API, err = createWalletBackendAPIContainer(ctx, walletBackendAPIContainerName, - walletBackendImage, s.TestNetwork, s.clientAuthKeyPair, s.distributionAccountKeyPair) + s.walletBackendImage, s.TestNetwork, s.clientAuthKeyPair, s.distributionAccountKeyPair) if err != nil { return fmt.Errorf("creating wallet backend API container: %w", err) } @@ -294,6 +303,91 @@ func (s *SharedContainers) setupWalletBackend(ctx context.Context, t *testing.T) return nil } +// waitForIngestSync waits until the ingest service has caught up with RPC. +// It polls every 2 seconds until the gap between RPC's latest ledger and the +// backend's live_ingest_cursor is within the health threshold (50 ledgers). +func (s *SharedContainers) waitForIngestSync(ctx context.Context) error { + const ( + pollInterval = 2 * time.Second + timeout = 5 * time.Minute + ledgerHealthThreshold = uint32(50) + ledgerCursorName = "latest_ingest_ledger" + ) + + // Get RPC connection + rpcURL, err := s.RPCContainer.GetConnectionString(ctx) + if err != nil { + return fmt.Errorf("getting RPC connection string: %w", err) + } + + // Get DB connection + dbHost, err := s.WalletDBContainer.GetHost(ctx) + if err != nil { + return fmt.Errorf("getting database host: %w", err) + } + dbPort, err := s.WalletDBContainer.GetPort(ctx) + if err != nil { + return fmt.Errorf("getting database port: %w", err) + } + dbURL := fmt.Sprintf("postgres://postgres@%s:%s/wallet-backend?sslmode=disable", dbHost, dbPort) + dbConnectionPool, err := db.OpenDBConnectionPool(dbURL) + if err != nil { + return fmt.Errorf("opening database connection pool: %w", err) + } + defer dbConnectionPool.Close() //nolint:errcheck + + // Create RPC service for health checks + httpClient := s.httpClient + metricsService := metrics.NewMockMetricsService() + metricsService.On("IncRPCMethodCalls", "GetHealth") + metricsService.On("IncRPCEndpointSuccess", "getHealth") + metricsService.On("ObserveRPCMethodDuration", "GetHealth", mock.AnythingOfType("float64")) + metricsService.On("ObserveRPCRequestDuration", "getHealth", mock.AnythingOfType("float64")) + metricsService.On("IncRPCRequests", "getHealth") + rpcService, err := services.NewRPCService(rpcURL, networkPassphrase, httpClient, metricsService) + if err != nil { + return fmt.Errorf("creating RPC service: %w", err) + } + + log.Ctx(ctx).Info("⏳ Waiting for ingest to sync with RPC...") + + ticker := time.NewTicker(pollInterval) + defer ticker.Stop() + timeoutChan := time.After(timeout) + + for { + select { + case <-timeoutChan: + return fmt.Errorf("timeout waiting for ingest to sync with RPC after %v", timeout) + case <-ticker.C: + // Get RPC latest ledger + health, err := rpcService.GetHealth() + if err != nil { + log.Ctx(ctx).Warnf("Failed to get RPC health: %v", err) + continue + } + + // Get backend's latest ledger from database + var backendLatestLedger uint32 + err = dbConnectionPool.GetContext(ctx, &backendLatestLedger, + `SELECT COALESCE(value::integer, 0) FROM ingest_store WHERE key = $1`, ledgerCursorName) + if err != nil { + log.Ctx(ctx).Warnf("Failed to get backend latest ledger: %v", err) + continue + } + + gap := health.LatestLedger - backendLatestLedger + log.Ctx(ctx).Infof("🔄 Ingest sync status: RPC=%d, Backend=%d, Gap=%d (threshold=%d)", + health.LatestLedger, backendLatestLedger, gap, ledgerHealthThreshold) + + if gap <= 5 { + log.Ctx(ctx).Info("✅ Ingest is synced with RPC") + return nil + } + } + } +} + // NewSharedContainers creates and starts all containers needed for integration tests func NewSharedContainers(t *testing.T) *SharedContainers { // Get the directory of the current source file @@ -322,7 +416,7 @@ func NewSharedContainers(t *testing.T) *SharedContainers { require.NoError(t, err, "failed to setup contracts") // Setup wallet backend - err = shared.setupWalletBackend(ctx, t) + err = shared.setupWalletBackend(ctx) require.NoError(t, err, "failed to setup wallet backend") return shared @@ -363,6 +457,24 @@ func (s *SharedContainers) GetMasterKeyPair(ctx context.Context) *keypair.Full { return s.masterKeyPair } +// GetDistributionAccountKeyPair returns the distribution account keypair +func (s *SharedContainers) GetDistributionAccountKeyPair(ctx context.Context) *keypair.Full { + return s.distributionAccountKeyPair +} + +// GetWalletDBConnectionString returns the connection string for the wallet backend database +func (s *SharedContainers) GetWalletDBConnectionString(ctx context.Context) (string, error) { + dbHost, err := s.WalletDBContainer.GetHost(ctx) + if err != nil { + return "", fmt.Errorf("getting database host: %w", err) + } + dbPort, err := s.WalletDBContainer.GetPort(ctx) + if err != nil { + return "", fmt.Errorf("getting database port: %w", err) + } + return fmt.Sprintf("postgres://postgres@%s:%s/wallet-backend?sslmode=disable", dbHost, dbPort), nil +} + // Cleanup cleans up shared containers after all tests complete func (s *SharedContainers) Cleanup(ctx context.Context) { if s.WalletBackendContainer.API != nil { @@ -393,6 +505,7 @@ func (s *SharedContainers) Cleanup(ctx context.Context) { // TestEnvironment holds all initialized services and clients for integration tests type TestEnvironment struct { + Containers *SharedContainers WBClient *wbclient.Client RPCService services.RPCService PrimaryAccountKP *keypair.Full @@ -528,6 +641,7 @@ func NewTestEnvironment(ctx context.Context, containers *SharedContainers) (*Tes log.Ctx(ctx).Info("✅ Test environment setup complete") return &TestEnvironment{ + Containers: containers, WBClient: wbClient, RPCService: rpcService, PrimaryAccountKP: primaryAccountKP, diff --git a/internal/integrationtests/infrastructure/restart_ingest.go b/internal/integrationtests/infrastructure/restart_ingest.go new file mode 100644 index 000000000..c5a47097b --- /dev/null +++ b/internal/integrationtests/infrastructure/restart_ingest.go @@ -0,0 +1,139 @@ +package infrastructure + +import ( + "context" + "fmt" + "io" + "sync/atomic" + "time" + + "github.com/stellar/go/support/log" + + "github.com/stellar/wallet-backend/internal/services" +) + +// restartCounter is used to generate unique container names for restarted ingest containers +var restartCounter atomic.Int64 + +// RestartIngestContainer stops the current ingest container and starts a new one with extra environment variables +func (s *SharedContainers) RestartIngestContainer(ctx context.Context, extraEnv map[string]string) error { + // Terminate existing container + if s.WalletBackendContainer.Ingest != nil { + if err := s.WalletBackendContainer.Ingest.Terminate(ctx); err != nil { + return fmt.Errorf("terminating ingest container: %w", err) + } + } + + // Rebuild or verify wallet-backend Docker image (reuses existing check) + walletBackendImage, err := ensureWalletBackendImage(ctx, walletBackendContainerTag) + if err != nil { + return fmt.Errorf("ensuring wallet backend image: %w", err) + } + + // Use a unique container name to avoid reusing the terminated container + counter := restartCounter.Add(1) + containerName := fmt.Sprintf("%s-restart-%d", walletBackendIngestContainerName, counter) + + // Start new ingest container + s.WalletBackendContainer.Ingest, err = createWalletBackendIngestContainer(ctx, containerName, + walletBackendImage, s.TestNetwork, s.clientAuthKeyPair, s.distributionAccountKeyPair, extraEnv) + if err != nil { + return fmt.Errorf("creating wallet backend ingest container: %w", err) + } + + return nil +} + +// StopIngestContainer stops the ingest container without terminating it. +// This allows the network to advance while the container is stopped. +func (s *SharedContainers) StopIngestContainer(ctx context.Context) error { + if s.WalletBackendContainer.Ingest == nil { + return fmt.Errorf("ingest container is not running") + } + + if err := s.WalletBackendContainer.Ingest.Stop(ctx, nil); err != nil { + return fmt.Errorf("stopping ingest container: %w", err) + } + + log.Ctx(ctx).Info("🛑 Stopped ingest container") + return nil +} + +// GetIngestContainerLogs returns the logs from the ingest container. +func (s *SharedContainers) GetIngestContainerLogs(ctx context.Context) (string, error) { + if s.WalletBackendContainer.Ingest == nil { + return "", fmt.Errorf("ingest container is not running") + } + + logsReader, err := s.WalletBackendContainer.Ingest.Logs(ctx) + if err != nil { + return "", fmt.Errorf("getting ingest container logs: %w", err) + } + defer logsReader.Close() //nolint:errcheck + + logBytes, err := io.ReadAll(logsReader) + if err != nil { + return "", fmt.Errorf("reading ingest container logs: %w", err) + } + + return string(logBytes), nil +} + +// WaitForNetworkAdvance polls the RPC service until the network reaches the target ledger. +func (s *SharedContainers) WaitForNetworkAdvance(ctx context.Context, rpcService services.RPCService, targetLedger uint32, timeout time.Duration) error { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + timeoutChan := time.After(timeout) + + for { + select { + case <-ctx.Done(): + return fmt.Errorf("context cancelled: %w", ctx.Err()) + case <-timeoutChan: + return fmt.Errorf("network did not reach ledger %d within %v", targetLedger, timeout) + case <-ticker.C: + health, err := rpcService.GetHealth() + if err != nil { + log.Ctx(ctx).Warnf("Error getting RPC health during network advance wait: %v", err) + continue + } + + currentLedger := health.LatestLedger + log.Ctx(ctx).Infof("Network advance progress: current=%d, target=%d", currentLedger, targetLedger) + + if currentLedger >= targetLedger { + log.Ctx(ctx).Infof("Network reached target ledger %d", targetLedger) + return nil + } + } + } +} + +// WaitForLatestLedgerToReach polls the database until the latest_ingest_ledger cursor reaches the target. +func (s *SharedContainers) WaitForLatestLedgerToReach(ctx context.Context, targetLedger uint32, timeout time.Duration) error { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + timeoutChan := time.After(timeout) + + for { + select { + case <-ctx.Done(): + return fmt.Errorf("context cancelled: %w", ctx.Err()) + case <-timeoutChan: + return fmt.Errorf("latest_ingest_ledger did not reach %d within %v", targetLedger, timeout) + case <-ticker.C: + latest, err := s.GetIngestCursor(ctx, "latest_ingest_ledger") + if err != nil { + log.Ctx(ctx).Warnf("Error getting latest cursor during catchup wait: %v", err) + continue + } + + log.Ctx(ctx).Infof("Catchup progress: latest=%d, target=%d", latest, targetLedger) + + if latest >= targetLedger { + log.Ctx(ctx).Infof("Catchup completed: latest_ingest_ledger=%d reached target=%d", latest, targetLedger) + return nil + } + } + } +} diff --git a/internal/integrationtests/main_test.go b/internal/integrationtests/main_test.go index 3453836af..fc0859e1b 100644 --- a/internal/integrationtests/main_test.go +++ b/internal/integrationtests/main_test.go @@ -35,16 +35,19 @@ func TestIntegrationTests(t *testing.T) { t.Fatalf("Failed to initialize test environment: %v", err) } - t.Run("AccountRegisterTestSuite", func(t *testing.T) { - suite.Run(t, &AccountRegisterTestSuite{ + // Phase 2: Test parallel live + backfill ingestion + t.Run("BackfillTestSuite", func(t *testing.T) { + suite.Run(t, &BackfillTestSuite{ testEnv: testEnv, }) }) - // Only proceed if account registration succeeded - if t.Failed() { - t.Fatal("AccountRegisterTestSuite failed, skipping remaining tests") - } + // Test catchup backfilling during live ingestion + t.Run("CatchupTestSuite", func(t *testing.T) { + suite.Run(t, &CatchupTestSuite{ + testEnv: testEnv, + }) + }) // Phase 1: Validate balances from checkpoint before fixture transactions t.Run("AccountBalancesAfterCheckpointTestSuite", func(t *testing.T) { @@ -79,10 +82,20 @@ func TestIntegrationTests(t *testing.T) { }) }) - // Phase 2: Validate balances after live ingestion processes fixture transactions + // Phase 3: Validate balances after live ingestion processes fixture transactions t.Run("AccountBalancesAfterLiveIngestionTestSuite", func(t *testing.T) { suite.Run(t, &AccountBalancesAfterLiveIngestionTestSuite{ testEnv: testEnv, }) }) + + if t.Failed() { + t.Fatal("AccountBalancesAfterLiveIngestionTestSuite failed, skipping remaining tests") + } + + t.Run("AccountRegisterTestSuite", func(t *testing.T) { + suite.Run(t, &AccountRegisterTestSuite{ + testEnv: testEnv, + }) + }) } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index f0d66076a..3c14a2427 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -14,6 +14,7 @@ type MetricsService interface { RegisterPoolMetrics(channel string, pool pond.Pool) GetRegistry() *prometheus.Registry SetLatestLedgerIngested(value float64) + SetOldestLedgerIngested(value float64) ObserveIngestionDuration(duration float64) IncActiveAccount() DecActiveAccount() @@ -51,6 +52,14 @@ type MetricsService interface { IncGraphQLField(operationName, fieldName string, success bool) ObserveGraphQLComplexity(operationName string, complexity int) IncGraphQLError(operationName, errorType string) + // Backfill Metrics - all methods require instance ID + IncBackfillBatchesCompleted(instance string) + IncBackfillBatchesFailed(instance string) + ObserveBackfillPhaseDuration(instance string, phase string, duration float64) + IncBackfillLedgersProcessed(instance string, count int) + IncBackfillTransactionsProcessed(instance string, count int) + IncBackfillOperationsProcessed(instance string, count int) + SetBackfillElapsed(instance string, seconds float64) } // MetricsService handles all metrics for the wallet-backend @@ -60,6 +69,7 @@ type metricsService struct { // Ingest Service Metrics latestLedgerIngested prometheus.Gauge + oldestLedgerIngested prometheus.Gauge ingestionDuration *prometheus.HistogramVec // Account Metrics @@ -110,6 +120,25 @@ type metricsService struct { graphqlFieldsTotal *prometheus.CounterVec graphqlComplexity *prometheus.SummaryVec graphqlErrorsTotal *prometheus.CounterVec + + // Backfill Metrics (Progress) - all GaugeVec with instance label + backfillBatchesTotal *prometheus.GaugeVec + backfillBatchesCompleted *prometheus.GaugeVec + backfillBatchesFailed *prometheus.GaugeVec + + // Backfill Metrics (Phase Durations) + backfillPhaseDuration *prometheus.HistogramVec + + // Backfill Metrics (Counters) - all CounterVec with instance label + backfillLedgersProcessed *prometheus.CounterVec + backfillTransactionsProcessed *prometheus.CounterVec + backfillOperationsProcessed *prometheus.CounterVec + backfillRetriesTotal *prometheus.CounterVec + + // Backfill Metrics (Performance) - HistogramVec with instance label + backfillBatchSize *prometheus.HistogramVec + backfillBatchLedgersProcessed *prometheus.HistogramVec + backfillElapsed *prometheus.GaugeVec } // NewMetricsService creates a new metrics service with all metrics registered @@ -126,11 +155,17 @@ func NewMetricsService(db *sqlx.DB) MetricsService { Help: "Latest ledger ingested", }, ) + m.oldestLedgerIngested = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "ingestion_ledger_oldest", + Help: "Oldest ledger ingested (backfill boundary)", + }, + ) m.ingestionDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "ingestion_duration_seconds", Help: "Duration of ledger ingestion", - Buckets: []float64{0.01, 0.02, 0.03, 0.05, 0.075, 0.1, 0.125, 0.15, 0.2, 0.3, 0.5, 1, 2, 5}, + Buckets: []float64{0.01, 0.02, 0.03, 0.05, 0.075, 0.1, 0.125, 0.15, 0.2, 0.3, 0.5, 1, 2, 3, 4, 5, 6, 7, 8, 10}, }, []string{}, ) @@ -294,7 +329,7 @@ func NewMetricsService(db *sqlx.DB) MetricsService { ) m.stateChangesTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: "state_changes_total", + Name: "ingestion_state_changes_total", Help: "Total number of state changes persisted to database by type and category", }, []string{"type", "category"}, @@ -305,7 +340,7 @@ func NewMetricsService(db *sqlx.DB) MetricsService { prometheus.HistogramOpts{ Name: "ingestion_phase_duration_seconds", Help: "Duration of each ingestion phase", - Buckets: []float64{0.01, 0.05, 0.1, 0.15, 0.2, 0.3, 0.5, 1, 2, 5, 10, 30, 60, 120}, + Buckets: []float64{0.01, 0.05, 0.1, 0.15, 0.2, 0.3, 0.5, 1, 2, 3, 4, 5, 6, 7, 10, 30, 60}, }, []string{"phase"}, ) @@ -373,6 +408,94 @@ func NewMetricsService(db *sqlx.DB) MetricsService { }, []string{"operation_name", "error_type"}, ) + // Backfill Progress Gauges (with backfill_instance label to avoid conflict with Prometheus's instance label) + m.backfillBatchesTotal = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "backfill_batches_total", + Help: "Total number of batches to process in this backfill instance", + }, + []string{"backfill_instance"}, + ) + m.backfillBatchesCompleted = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "backfill_batches_completed", + Help: "Number of batches successfully completed in this backfill instance", + }, + []string{"backfill_instance"}, + ) + m.backfillBatchesFailed = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "backfill_batches_failed", + Help: "Number of batches that failed in this backfill instance", + }, + []string{"backfill_instance"}, + ) + + // Backfill Phase Duration (with backfill_instance AND phase labels) + // Exponential buckets: 0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6, 51.2, 102.4, 204.8 seconds + m.backfillPhaseDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "backfill_phase_duration_seconds", + Help: "Duration of each backfill phase", + Buckets: prometheus.ExponentialBuckets(0.1, 2, 12), + }, + []string{"backfill_instance", "phase"}, + ) + + // Backfill Counters (with backfill_instance label) + m.backfillLedgersProcessed = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "backfill_ledgers_processed_total", + Help: "Total ledgers processed during backfill", + }, + []string{"backfill_instance"}, + ) + m.backfillTransactionsProcessed = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "backfill_transactions_processed_total", + Help: "Total transactions processed during backfill", + }, + []string{"backfill_instance"}, + ) + m.backfillOperationsProcessed = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "backfill_operations_processed_total", + Help: "Total operations processed during backfill", + }, + []string{"backfill_instance"}, + ) + m.backfillRetriesTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "backfill_retries_total", + Help: "Total number of retry attempts for ledger fetch during backfill", + }, + []string{"backfill_instance"}, + ) + + // Backfill Performance Metrics (with backfill_instance label) + m.backfillBatchSize = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "backfill_batch_size", + Help: "Number of ledgers per batch in backfill", + Buckets: prometheus.ExponentialBuckets(1, 2, 10), // 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 + }, + []string{"backfill_instance"}, + ) + m.backfillBatchLedgersProcessed = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "backfill_batch_ledgers_processed", + Help: "Actual ledgers processed per batch in backfill", + Buckets: prometheus.ExponentialBuckets(1, 2, 10), // 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 + }, + []string{"backfill_instance"}, + ) + m.backfillElapsed = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "backfill_elapsed_seconds", + Help: "Elapsed time of current/completed backfill job in seconds", + }, + []string{"backfill_instance"}, + ) m.registerMetrics() return m @@ -383,6 +506,7 @@ func (m *metricsService) registerMetrics() { m.registry.MustRegister( collector, m.latestLedgerIngested, + m.oldestLedgerIngested, m.ingestionDuration, m.activeAccounts, m.rpcRequestsTotal, @@ -415,6 +539,21 @@ func (m *metricsService) registerMetrics() { m.graphqlFieldsTotal, m.graphqlComplexity, m.graphqlErrorsTotal, + // Backfill Progress + m.backfillBatchesTotal, + m.backfillBatchesCompleted, + m.backfillBatchesFailed, + // Backfill Phases + m.backfillPhaseDuration, + // Backfill Counters + m.backfillLedgersProcessed, + m.backfillTransactionsProcessed, + m.backfillOperationsProcessed, + m.backfillRetriesTotal, + // Backfill Performance + m.backfillBatchSize, + m.backfillBatchLedgersProcessed, + m.backfillElapsed, ) } @@ -498,6 +637,10 @@ func (m *metricsService) SetLatestLedgerIngested(value float64) { m.latestLedgerIngested.Set(value) } +func (m *metricsService) SetOldestLedgerIngested(value float64) { + m.oldestLedgerIngested.Set(value) +} + func (m *metricsService) ObserveIngestionDuration(duration float64) { m.ingestionDuration.WithLabelValues().Observe(duration) } @@ -646,3 +789,32 @@ func (m *metricsService) ObserveGraphQLComplexity(operationName string, complexi func (m *metricsService) IncGraphQLError(operationName, errorType string) { m.graphqlErrorsTotal.WithLabelValues(operationName, errorType).Inc() } + +// Backfill Metrics +func (m *metricsService) IncBackfillBatchesCompleted(instance string) { + m.backfillBatchesCompleted.WithLabelValues(instance).Inc() +} + +func (m *metricsService) IncBackfillBatchesFailed(instance string) { + m.backfillBatchesFailed.WithLabelValues(instance).Inc() +} + +func (m *metricsService) ObserveBackfillPhaseDuration(instance string, phase string, duration float64) { + m.backfillPhaseDuration.WithLabelValues(instance, phase).Observe(duration) +} + +func (m *metricsService) IncBackfillLedgersProcessed(instance string, count int) { + m.backfillLedgersProcessed.WithLabelValues(instance).Add(float64(count)) +} + +func (m *metricsService) IncBackfillTransactionsProcessed(instance string, count int) { + m.backfillTransactionsProcessed.WithLabelValues(instance).Add(float64(count)) +} + +func (m *metricsService) IncBackfillOperationsProcessed(instance string, count int) { + m.backfillOperationsProcessed.WithLabelValues(instance).Add(float64(count)) +} + +func (m *metricsService) SetBackfillElapsed(instance string, seconds float64) { + m.backfillElapsed.WithLabelValues(instance).Set(seconds) +} diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 1c5d73106..7f9061aee 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -957,7 +957,7 @@ func TestStateChangeMetrics(t *testing.T) { typeCategoryValues := make(map[string]map[string]float64) for _, mf := range metricFamilies { - if mf.GetName() == "state_changes_total" { + if mf.GetName() == "ingestion_state_changes_total" { found = true for _, metric := range mf.GetMetric() { labels := make(map[string]string) @@ -976,7 +976,7 @@ func TestStateChangeMetrics(t *testing.T) { } } - assert.True(t, found, "state_changes_total metric not found") + assert.True(t, found, "ingestion_state_changes_total metric not found") assert.Equal(t, float64(30), typeCategoryValues["DEBIT"]["BALANCE"], "Expected 30 DEBIT/BALANCE state changes") assert.Equal(t, float64(25), typeCategoryValues["CREDIT"]["BALANCE"], "Expected 25 CREDIT/BALANCE state changes") assert.Equal(t, float64(10), typeCategoryValues["ADD"]["SIGNER"], "Expected 10 ADD/SIGNER state changes") @@ -1008,7 +1008,7 @@ func TestStateChangeMetrics(t *testing.T) { typeCategoryFound := make(map[string]map[string]bool) for _, mf := range metricFamilies { - if mf.GetName() == "state_changes_total" { + if mf.GetName() == "ingestion_state_changes_total" { for _, metric := range mf.GetMetric() { labels := make(map[string]string) for _, label := range metric.GetLabel() { @@ -1039,3 +1039,254 @@ func TestStateChangeMetrics(t *testing.T) { } }) } + +func TestBackfillMetrics(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + + t.Run("backfill batches completed gauge", func(t *testing.T) { + ms := NewMetricsService(db) + instance := "100-200" + + ms.IncBackfillBatchesCompleted(instance) + ms.IncBackfillBatchesCompleted(instance) + + metricFamilies, err := ms.GetRegistry().Gather() + require.NoError(t, err) + + found := false + for _, mf := range metricFamilies { + if mf.GetName() == "backfill_batches_completed" { + found = true + metric := mf.GetMetric()[0] + assert.Equal(t, float64(2), metric.GetGauge().GetValue()) + // Verify instance label + labels := make(map[string]string) + for _, label := range metric.GetLabel() { + labels[label.GetName()] = label.GetValue() + } + assert.Equal(t, instance, labels["backfill_instance"]) + } + } + assert.True(t, found, "backfill_batches_completed metric not found") + }) + + t.Run("backfill batches failed gauge", func(t *testing.T) { + ms := NewMetricsService(db) + instance := "100-200" + + ms.IncBackfillBatchesFailed(instance) + + metricFamilies, err := ms.GetRegistry().Gather() + require.NoError(t, err) + + found := false + for _, mf := range metricFamilies { + if mf.GetName() == "backfill_batches_failed" { + found = true + metric := mf.GetMetric()[0] + assert.Equal(t, float64(1), metric.GetGauge().GetValue()) + labels := make(map[string]string) + for _, label := range metric.GetLabel() { + labels[label.GetName()] = label.GetValue() + } + assert.Equal(t, instance, labels["backfill_instance"]) + } + } + assert.True(t, found, "backfill_batches_failed metric not found") + }) + + t.Run("backfill phase duration histogram", func(t *testing.T) { + ms := NewMetricsService(db) + instance := "100-200" + + ms.ObserveBackfillPhaseDuration(instance, "get_ledger_from_backend", 0.5) + ms.ObserveBackfillPhaseDuration(instance, "db_insert", 1.2) + ms.ObserveBackfillPhaseDuration(instance, "batch_processing", 2.5) + + metricFamilies, err := ms.GetRegistry().Gather() + require.NoError(t, err) + + found := false + phasesFound := make(map[string]bool) + for _, mf := range metricFamilies { + if mf.GetName() == "backfill_phase_duration_seconds" { + found = true + for _, metric := range mf.GetMetric() { + labels := make(map[string]string) + for _, label := range metric.GetLabel() { + labels[label.GetName()] = label.GetValue() + } + if labels["backfill_instance"] == instance { + phasesFound[labels["phase"]] = true + assert.Equal(t, uint64(1), metric.GetHistogram().GetSampleCount()) + } + } + } + } + assert.True(t, found, "backfill_phase_duration_seconds metric not found") + assert.True(t, phasesFound["get_ledger_from_backend"]) + assert.True(t, phasesFound["db_insert"]) + assert.True(t, phasesFound["batch_processing"]) + }) + + t.Run("backfill ledgers processed counter", func(t *testing.T) { + ms := NewMetricsService(db) + instance := "100-200" + + ms.IncBackfillLedgersProcessed(instance, 50) + ms.IncBackfillLedgersProcessed(instance, 30) + + metricFamilies, err := ms.GetRegistry().Gather() + require.NoError(t, err) + + found := false + for _, mf := range metricFamilies { + if mf.GetName() == "backfill_ledgers_processed_total" { + found = true + metric := mf.GetMetric()[0] + assert.Equal(t, float64(80), metric.GetCounter().GetValue()) + labels := make(map[string]string) + for _, label := range metric.GetLabel() { + labels[label.GetName()] = label.GetValue() + } + assert.Equal(t, instance, labels["backfill_instance"]) + } + } + assert.True(t, found, "backfill_ledgers_processed_total metric not found") + }) + + t.Run("backfill transactions processed counter", func(t *testing.T) { + ms := NewMetricsService(db) + instance := "100-200" + + ms.IncBackfillTransactionsProcessed(instance, 100) + ms.IncBackfillTransactionsProcessed(instance, 50) + + metricFamilies, err := ms.GetRegistry().Gather() + require.NoError(t, err) + + found := false + for _, mf := range metricFamilies { + if mf.GetName() == "backfill_transactions_processed_total" { + found = true + metric := mf.GetMetric()[0] + assert.Equal(t, float64(150), metric.GetCounter().GetValue()) + labels := make(map[string]string) + for _, label := range metric.GetLabel() { + labels[label.GetName()] = label.GetValue() + } + assert.Equal(t, instance, labels["backfill_instance"]) + } + } + assert.True(t, found, "backfill_transactions_processed_total metric not found") + }) + + t.Run("backfill operations processed counter", func(t *testing.T) { + ms := NewMetricsService(db) + instance := "100-200" + + ms.IncBackfillOperationsProcessed(instance, 200) + ms.IncBackfillOperationsProcessed(instance, 75) + + metricFamilies, err := ms.GetRegistry().Gather() + require.NoError(t, err) + + found := false + for _, mf := range metricFamilies { + if mf.GetName() == "backfill_operations_processed_total" { + found = true + metric := mf.GetMetric()[0] + assert.Equal(t, float64(275), metric.GetCounter().GetValue()) + labels := make(map[string]string) + for _, label := range metric.GetLabel() { + labels[label.GetName()] = label.GetValue() + } + assert.Equal(t, instance, labels["backfill_instance"]) + } + } + assert.True(t, found, "backfill_operations_processed_total metric not found") + }) + + t.Run("backfill elapsed gauge", func(t *testing.T) { + ms := NewMetricsService(db) + instance := "100-200" + + ms.SetBackfillElapsed(instance, 120.5) + + metricFamilies, err := ms.GetRegistry().Gather() + require.NoError(t, err) + + found := false + for _, mf := range metricFamilies { + if mf.GetName() == "backfill_elapsed_seconds" { + found = true + metric := mf.GetMetric()[0] + assert.Equal(t, float64(120.5), metric.GetGauge().GetValue()) + labels := make(map[string]string) + for _, label := range metric.GetLabel() { + labels[label.GetName()] = label.GetValue() + } + assert.Equal(t, instance, labels["backfill_instance"]) + } + } + assert.True(t, found, "backfill_elapsed_seconds metric not found") + + // Test update + ms.SetBackfillElapsed(instance, 240.0) + metricFamilies, err = ms.GetRegistry().Gather() + require.NoError(t, err) + + for _, mf := range metricFamilies { + if mf.GetName() == "backfill_elapsed_seconds" { + metric := mf.GetMetric()[0] + assert.Equal(t, float64(240.0), metric.GetGauge().GetValue()) + } + } + }) + + t.Run("multiple backfill instances tracked independently", func(t *testing.T) { + ms := NewMetricsService(db) + instance1 := "100-200" + instance2 := "300-400" + + ms.IncBackfillBatchesCompleted(instance1) + ms.IncBackfillBatchesCompleted(instance1) + ms.IncBackfillBatchesCompleted(instance2) + + ms.IncBackfillLedgersProcessed(instance1, 50) + ms.IncBackfillLedgersProcessed(instance2, 100) + + metricFamilies, err := ms.GetRegistry().Gather() + require.NoError(t, err) + + batchesValues := make(map[string]float64) + ledgersValues := make(map[string]float64) + + for _, mf := range metricFamilies { + switch mf.GetName() { + case "backfill_batches_completed": + for _, metric := range mf.GetMetric() { + labels := make(map[string]string) + for _, label := range metric.GetLabel() { + labels[label.GetName()] = label.GetValue() + } + batchesValues[labels["backfill_instance"]] = metric.GetGauge().GetValue() + } + case "backfill_ledgers_processed_total": + for _, metric := range mf.GetMetric() { + labels := make(map[string]string) + for _, label := range metric.GetLabel() { + labels[label.GetName()] = label.GetValue() + } + ledgersValues[labels["backfill_instance"]] = metric.GetCounter().GetValue() + } + } + } + + assert.Equal(t, float64(2), batchesValues[instance1], "Expected 2 batches for instance1") + assert.Equal(t, float64(1), batchesValues[instance2], "Expected 1 batch for instance2") + assert.Equal(t, float64(50), ledgersValues[instance1], "Expected 50 ledgers for instance1") + assert.Equal(t, float64(100), ledgersValues[instance2], "Expected 100 ledgers for instance2") + }) +} diff --git a/internal/metrics/mocks.go b/internal/metrics/mocks.go index 9e09bcd3d..7bc282ef7 100644 --- a/internal/metrics/mocks.go +++ b/internal/metrics/mocks.go @@ -162,3 +162,36 @@ func (m *MockMetricsService) ObserveGraphQLComplexity(operationName string, comp func (m *MockMetricsService) IncGraphQLError(operationName, errorType string) { m.Called(operationName, errorType) } + +func (m *MockMetricsService) SetOldestLedgerIngested(value float64) { + m.Called(value) +} + +// Backfill Metrics +func (m *MockMetricsService) IncBackfillBatchesCompleted(instance string) { + m.Called(instance) +} + +func (m *MockMetricsService) IncBackfillBatchesFailed(instance string) { + m.Called(instance) +} + +func (m *MockMetricsService) ObserveBackfillPhaseDuration(instance string, phase string, duration float64) { + m.Called(instance, phase, duration) +} + +func (m *MockMetricsService) IncBackfillLedgersProcessed(instance string, count int) { + m.Called(instance, count) +} + +func (m *MockMetricsService) IncBackfillTransactionsProcessed(instance string, count int) { + m.Called(instance, count) +} + +func (m *MockMetricsService) IncBackfillOperationsProcessed(instance string, count int) { + m.Called(instance, count) +} + +func (m *MockMetricsService) SetBackfillElapsed(instance string, seconds float64) { + m.Called(instance, seconds) +} diff --git a/internal/serve/graphql/generated/generated.go b/internal/serve/graphql/generated/generated.go index 293bb53e5..284d24afe 100644 --- a/internal/serve/graphql/generated/generated.go +++ b/internal/serve/graphql/generated/generated.go @@ -2260,7 +2260,7 @@ interface BaseStateChange { ledgerNumber: UInt32! # GraphQL Relationships - these fields use resolvers - # Related operation - nullable since fee state changes do not have operations associated with them + # Related account account: Account! @goField(forceResolver: true) # Related operation - nullable since fee state changes do not have operations associated with them @@ -2399,9 +2399,9 @@ type BalanceAuthorizationChange implements BaseStateChange{ # gqlgen generates Go structs from this schema definition type Transaction{ hash: String! - envelopeXdr: String! + envelopeXdr: String resultXdr: String! - metaXdr: String! + metaXdr: String ledgerNumber: UInt32! ledgerCreatedAt: Time! ingestedAt: Time! @@ -10492,14 +10492,11 @@ func (ec *executionContext) _Transaction_envelopeXdr(ctx context.Context, field return graphql.Null } if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } return graphql.Null } - res := resTmp.(string) + res := resTmp.(*string) fc.Result = res - return ec.marshalNString2string(ctx, field.Selections, res) + return ec.marshalOString2ᚖstring(ctx, field.Selections, res) } func (ec *executionContext) fieldContext_Transaction_envelopeXdr(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { @@ -10580,14 +10577,11 @@ func (ec *executionContext) _Transaction_metaXdr(ctx context.Context, field grap return graphql.Null } if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } return graphql.Null } - res := resTmp.(string) + res := resTmp.(*string) fc.Result = res - return ec.marshalNString2string(ctx, field.Selections, res) + return ec.marshalOString2ᚖstring(ctx, field.Selections, res) } func (ec *executionContext) fieldContext_Transaction_metaXdr(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { @@ -17932,9 +17926,6 @@ func (ec *executionContext) _Transaction(ctx context.Context, sel ast.SelectionS } case "envelopeXdr": out.Values[i] = ec._Transaction_envelopeXdr(ctx, field, obj) - if out.Values[i] == graphql.Null { - atomic.AddUint32(&out.Invalids, 1) - } case "resultXdr": out.Values[i] = ec._Transaction_resultXdr(ctx, field, obj) if out.Values[i] == graphql.Null { @@ -17942,9 +17933,6 @@ func (ec *executionContext) _Transaction(ctx context.Context, sel ast.SelectionS } case "metaXdr": out.Values[i] = ec._Transaction_metaXdr(ctx, field, obj) - if out.Values[i] == graphql.Null { - atomic.AddUint32(&out.Invalids, 1) - } case "ledgerNumber": out.Values[i] = ec._Transaction_ledgerNumber(ctx, field, obj) if out.Values[i] == graphql.Null { diff --git a/internal/serve/graphql/resolvers/account_balances_test.go b/internal/serve/graphql/resolvers/account_balances_test.go index 9349c882a..ec4098000 100644 --- a/internal/serve/graphql/resolvers/account_balances_test.go +++ b/internal/serve/graphql/resolvers/account_balances_test.go @@ -365,11 +365,17 @@ func TestQueryResolver_BalancesByAccountAddress(t *testing.T) { require.Len(t, balances, 1) // Verify it's a native balance - nativeBalance, ok := balances[0].(*graphql1.NativeBalance) - require.True(t, ok) - assert.Equal(t, "1000.0000000", nativeBalance.Balance) - assert.Equal(t, graphql1.TokenTypeNative, nativeBalance.TokenType) - assert.NotEmpty(t, nativeBalance.TokenID) // Native asset contract ID + for _, balance := range balances { + switch balance.GetTokenType() { + case graphql1.TokenTypeNative: + nativeBalance := balance.(*graphql1.NativeBalance) + assert.Equal(t, "1000.0000000", nativeBalance.Balance) + assert.Equal(t, graphql1.TokenTypeNative, nativeBalance.TokenType) + assert.NotEmpty(t, nativeBalance.TokenID) // Native asset contract ID + default: + t.Errorf("unexpected balance type: %v", balance.GetTokenType()) + } + } }) t.Run("success - account with classic trustlines", func(t *testing.T) { @@ -423,32 +429,35 @@ func TestQueryResolver_BalancesByAccountAddress(t *testing.T) { require.NoError(t, err) require.Len(t, balances, 3) // 1 native + 2 trustlines - // Verify native balance - nativeBalance, ok := balances[0].(*graphql1.NativeBalance) - require.True(t, ok) - assert.Equal(t, "500.0000000", nativeBalance.Balance) - - // Verify USDC trustline - usdcBalance, ok := balances[1].(*graphql1.TrustlineBalance) - require.True(t, ok) - assert.Equal(t, "100.0000000", usdcBalance.Balance) - assert.Equal(t, graphql1.TokenTypeClassic, usdcBalance.TokenType) - assert.Equal(t, "USDC", usdcBalance.Code) - assert.Equal(t, testUSDCIssuer, usdcBalance.Issuer) - assert.Equal(t, "1000.0000000", usdcBalance.Limit) - assert.Equal(t, "0.1000000", usdcBalance.BuyingLiabilities) - assert.Equal(t, "0.2000000", usdcBalance.SellingLiabilities) - assert.True(t, usdcBalance.IsAuthorized) - assert.False(t, usdcBalance.IsAuthorizedToMaintainLiabilities) - - // Verify EUR trustline - eurBalance, ok := balances[2].(*graphql1.TrustlineBalance) - require.True(t, ok) - assert.Equal(t, "500.0000000", eurBalance.Balance) - assert.Equal(t, "EUR", eurBalance.Code) - assert.Equal(t, testEURIssuer, eurBalance.Issuer) - assert.False(t, eurBalance.IsAuthorized) - assert.True(t, eurBalance.IsAuthorizedToMaintainLiabilities) + for _, balance := range balances { + switch balance.GetTokenType() { + case graphql1.TokenTypeNative: + nativeBalance := balance.(*graphql1.NativeBalance) + assert.Equal(t, "500.0000000", nativeBalance.Balance) + case graphql1.TokenTypeClassic: + trustlineBalance := balance.(*graphql1.TrustlineBalance) + switch trustlineBalance.Code { + case "USDC": + assert.Equal(t, "100.0000000", trustlineBalance.Balance) + assert.Equal(t, graphql1.TokenTypeClassic, trustlineBalance.TokenType) + assert.Equal(t, testUSDCIssuer, trustlineBalance.Issuer) + assert.Equal(t, "1000.0000000", trustlineBalance.Limit) + assert.Equal(t, "0.1000000", trustlineBalance.BuyingLiabilities) + assert.Equal(t, "0.2000000", trustlineBalance.SellingLiabilities) + assert.True(t, trustlineBalance.IsAuthorized) + assert.False(t, trustlineBalance.IsAuthorizedToMaintainLiabilities) + case "EUR": + assert.Equal(t, "500.0000000", trustlineBalance.Balance) + assert.Equal(t, testEURIssuer, trustlineBalance.Issuer) + assert.False(t, trustlineBalance.IsAuthorized) + assert.True(t, trustlineBalance.IsAuthorizedToMaintainLiabilities) + default: + t.Errorf("unexpected trustline code: %s", trustlineBalance.Code) + } + default: + t.Errorf("unexpected balance type: %v", balance.GetTokenType()) + } + } }) t.Run("success - account with SAC contract balances", func(t *testing.T) { @@ -489,16 +498,23 @@ func TestQueryResolver_BalancesByAccountAddress(t *testing.T) { require.NoError(t, err) require.Len(t, balances, 2) // 1 native + 1 SAC - // Verify SAC balance - sacBalance, ok := balances[1].(*graphql1.SACBalance) - require.True(t, ok) - assert.Equal(t, "2500.0000000", sacBalance.Balance) - assert.Equal(t, graphql1.TokenTypeSac, sacBalance.TokenType) - assert.Equal(t, testSACContractAddress, sacBalance.TokenID) - assert.Equal(t, "USDC", sacBalance.Code) - assert.Equal(t, testUSDCIssuer, sacBalance.Issuer) - assert.True(t, sacBalance.IsAuthorized) - assert.False(t, sacBalance.IsClawbackEnabled) + for _, balance := range balances { + switch balance.GetTokenType() { + case graphql1.TokenTypeNative: + // Native balance verified by count + case graphql1.TokenTypeSac: + sacBalance := balance.(*graphql1.SACBalance) + assert.Equal(t, "2500.0000000", sacBalance.Balance) + assert.Equal(t, graphql1.TokenTypeSac, sacBalance.TokenType) + assert.Equal(t, testSACContractAddress, sacBalance.TokenID) + assert.Equal(t, "USDC", sacBalance.Code) + assert.Equal(t, testUSDCIssuer, sacBalance.Issuer) + assert.True(t, sacBalance.IsAuthorized) + assert.False(t, sacBalance.IsClawbackEnabled) + default: + t.Errorf("unexpected balance type: %v", balance.GetTokenType()) + } + } }) t.Run("success - account with SEP-41 contract balances", func(t *testing.T) { @@ -539,15 +555,22 @@ func TestQueryResolver_BalancesByAccountAddress(t *testing.T) { require.NoError(t, err) require.Len(t, balances, 2) // 1 native + 1 SEP-41 - // Verify SEP-41 balance - sep41Balance, ok := balances[1].(*graphql1.SEP41Balance) - require.True(t, ok) - assert.Equal(t, "5000.0000000", sep41Balance.Balance) - assert.Equal(t, graphql1.TokenTypeSep41, sep41Balance.TokenType) - assert.Equal(t, testSEP41ContractAddress, sep41Balance.TokenID) - assert.Equal(t, "MyToken", sep41Balance.Name) - assert.Equal(t, "MTK", sep41Balance.Symbol) - assert.Equal(t, int32(7), sep41Balance.Decimals) + for _, balance := range balances { + switch balance.GetTokenType() { + case graphql1.TokenTypeNative: + // Native balance verified by count + case graphql1.TokenTypeSep41: + sep41Balance := balance.(*graphql1.SEP41Balance) + assert.Equal(t, "5000.0000000", sep41Balance.Balance) + assert.Equal(t, graphql1.TokenTypeSep41, sep41Balance.TokenType) + assert.Equal(t, testSEP41ContractAddress, sep41Balance.TokenID) + assert.Equal(t, "MyToken", sep41Balance.Name) + assert.Equal(t, "MTK", sep41Balance.Symbol) + assert.Equal(t, int32(7), sep41Balance.Decimals) + default: + t.Errorf("unexpected balance type: %v", balance.GetTokenType()) + } + } }) t.Run("success - mixed balances (native + trustlines + SAC + SEP-41)", func(t *testing.T) { @@ -594,22 +617,23 @@ func TestQueryResolver_BalancesByAccountAddress(t *testing.T) { require.NoError(t, err) require.Len(t, balances, 4) - // Verify all balance types are present - assert.IsType(t, &graphql1.NativeBalance{}, balances[0]) - assert.IsType(t, &graphql1.TrustlineBalance{}, balances[1]) - assert.IsType(t, &graphql1.SACBalance{}, balances[2]) - assert.IsType(t, &graphql1.SEP41Balance{}, balances[3]) - - // Verify SAC balance details - sacBalance := balances[2].(*graphql1.SACBalance) - assert.Equal(t, "EURC", sacBalance.Code) - assert.Equal(t, testEURIssuer, sacBalance.Issuer) - - // Verify SEP-41 balance details - sep41Balance := balances[3].(*graphql1.SEP41Balance) - assert.Equal(t, "CustomToken", sep41Balance.Name) - assert.Equal(t, "CTK", sep41Balance.Symbol) - assert.Equal(t, int32(6), sep41Balance.Decimals) + for _, balance := range balances { + switch balance.GetTokenType() { + case graphql1.TokenTypeNative: + // Native balance verified by count + case graphql1.TokenTypeClassic: + // Classic trustline verified by count + case graphql1.TokenTypeSep41: + sep41Balance := balance.(*graphql1.SEP41Balance) + assert.Equal(t, "CustomToken", sep41Balance.Name) + assert.Equal(t, "CTK", sep41Balance.Symbol) + assert.Equal(t, int32(6), sep41Balance.Decimals) + case graphql1.TokenTypeSac: + sacBalance := balance.(*graphql1.SACBalance) + assert.Equal(t, "EURC", sacBalance.Code) + assert.Equal(t, testEURIssuer, sacBalance.Issuer) + } + } }) t.Run("success - contract address (skips account and trustlines)", func(t *testing.T) { @@ -647,12 +671,18 @@ func TestQueryResolver_BalancesByAccountAddress(t *testing.T) { require.NoError(t, err) require.Len(t, balances, 1) - sep41Balance, ok := balances[0].(*graphql1.SEP41Balance) - require.True(t, ok) - assert.Equal(t, "1000.0000000", sep41Balance.Balance) - assert.Equal(t, "Token", sep41Balance.Name) - assert.Equal(t, "TKN", sep41Balance.Symbol) - assert.Equal(t, int32(7), sep41Balance.Decimals) + for _, balance := range balances { + switch balance.GetTokenType() { + case graphql1.TokenTypeSep41: + sep41Balance := balance.(*graphql1.SEP41Balance) + assert.Equal(t, "1000.0000000", sep41Balance.Balance) + assert.Equal(t, "Token", sep41Balance.Name) + assert.Equal(t, "TKN", sep41Balance.Symbol) + assert.Equal(t, int32(7), sep41Balance.Decimals) + default: + t.Errorf("unexpected balance type: %v", balance.GetTokenType()) + } + } }) t.Run("success - trustline with V0 extension (no liabilities)", func(t *testing.T) { @@ -688,9 +718,18 @@ func TestQueryResolver_BalancesByAccountAddress(t *testing.T) { balances, err := resolver.BalancesByAccountAddress(ctx, testAccountAddress) require.NoError(t, err) - trustlineBalance := balances[1].(*graphql1.TrustlineBalance) - assert.Equal(t, "0.0000000", trustlineBalance.BuyingLiabilities) - assert.Equal(t, "0.0000000", trustlineBalance.SellingLiabilities) + for _, balance := range balances { + switch balance.GetTokenType() { + case graphql1.TokenTypeNative: + // Native balance verified by count + case graphql1.TokenTypeClassic: + trustlineBalance := balance.(*graphql1.TrustlineBalance) + assert.Equal(t, "0.0000000", trustlineBalance.BuyingLiabilities) + assert.Equal(t, "0.0000000", trustlineBalance.SellingLiabilities) + default: + t.Errorf("unexpected balance type: %v", balance.GetTokenType()) + } + } }) // Error Cases @@ -1031,7 +1070,14 @@ func TestQueryResolver_BalancesByAccountAddress(t *testing.T) { require.NoError(t, err) // Should only have native balance, contract balance skipped require.Len(t, balances, 1) - assert.IsType(t, &graphql1.NativeBalance{}, balances[0]) + for _, balance := range balances { + switch balance.GetTokenType() { + case graphql1.TokenTypeNative: + assert.IsType(t, &graphql1.NativeBalance{}, balance) + default: + t.Errorf("unexpected balance type: %v", balance.GetTokenType()) + } + } }) t.Run("edge - trustline authorization flags combinations", func(t *testing.T) { @@ -1076,14 +1122,27 @@ func TestQueryResolver_BalancesByAccountAddress(t *testing.T) { require.NoError(t, err) require.Len(t, balances, 3) - // USDC - both flags set - usdcBalance := balances[1].(*graphql1.TrustlineBalance) - assert.True(t, usdcBalance.IsAuthorized) - assert.True(t, usdcBalance.IsAuthorizedToMaintainLiabilities) - - // EUR - no flags set - eurBalance := balances[2].(*graphql1.TrustlineBalance) - assert.False(t, eurBalance.IsAuthorized) - assert.False(t, eurBalance.IsAuthorizedToMaintainLiabilities) + for _, balance := range balances { + switch balance.GetTokenType() { + case graphql1.TokenTypeNative: + // Native balance verified by count + case graphql1.TokenTypeClassic: + trustlineBalance := balance.(*graphql1.TrustlineBalance) + switch trustlineBalance.Code { + case "USDC": + // USDC - both flags set + assert.True(t, trustlineBalance.IsAuthorized) + assert.True(t, trustlineBalance.IsAuthorizedToMaintainLiabilities) + case "EUR": + // EUR - no flags set + assert.False(t, trustlineBalance.IsAuthorized) + assert.False(t, trustlineBalance.IsAuthorizedToMaintainLiabilities) + default: + t.Errorf("unexpected trustline code: %s", trustlineBalance.Code) + } + default: + t.Errorf("unexpected balance type: %v", balance.GetTokenType()) + } + } }) } diff --git a/internal/serve/graphql/resolvers/queries.resolvers.go b/internal/serve/graphql/resolvers/queries.resolvers.go index 78b9c4aa3..07419e6ee 100644 --- a/internal/serve/graphql/resolvers/queries.resolvers.go +++ b/internal/serve/graphql/resolvers/queries.resolvers.go @@ -6,6 +6,8 @@ package resolvers import ( "context" + "database/sql" + "errors" "fmt" "strings" @@ -63,7 +65,17 @@ func (r *queryResolver) Transactions(ctx context.Context, first *int32, after *s // AccountByAddress is the resolver for the accountByAddress field. func (r *queryResolver) AccountByAddress(ctx context.Context, address string) (*types.Account, error) { - return r.models.Account.Get(ctx, address) + if address == "" { + return nil, fmt.Errorf("address cannot be empty") + } + acc, err := r.models.Account.Get(ctx, address) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return &types.Account{StellarAddress: address}, nil + } + return nil, err + } + return acc, nil } // Operations is the resolver for the operations field. diff --git a/internal/serve/graphql/resolvers/queries_resolvers_test.go b/internal/serve/graphql/resolvers/queries_resolvers_test.go index 5d360b7a7..82e18de54 100644 --- a/internal/serve/graphql/resolvers/queries_resolvers_test.go +++ b/internal/serve/graphql/resolvers/queries_resolvers_test.go @@ -38,9 +38,11 @@ func TestQueryResolver_TransactionByHash(t *testing.T) { require.NoError(t, err) assert.Equal(t, "tx1", tx.Hash) assert.Equal(t, toid.New(1000, 1, 0).ToInt64(), tx.ToID) - assert.Equal(t, "envelope1", tx.EnvelopeXDR) + require.NotNil(t, tx.EnvelopeXDR) + assert.Equal(t, "envelope1", *tx.EnvelopeXDR) assert.Equal(t, "result1", tx.ResultXDR) - assert.Equal(t, "meta1", tx.MetaXDR) + require.NotNil(t, tx.MetaXDR) + assert.Equal(t, "meta1", *tx.MetaXDR) assert.Equal(t, uint32(1), tx.LedgerNumber) }) @@ -240,8 +242,9 @@ func TestQueryResolver_Account(t *testing.T) { t.Run("non-existent account", func(t *testing.T) { acc, err := resolver.AccountByAddress(testCtx, "non-existent-account") - require.Error(t, err) - assert.Nil(t, acc) + require.NoError(t, err) + assert.NotNil(t, acc) + assert.Equal(t, "non-existent-account", acc.StellarAddress) }) t.Run("empty address", func(t *testing.T) { diff --git a/internal/serve/graphql/resolvers/test_utils.go b/internal/serve/graphql/resolvers/test_utils.go index 0d4399749..4fa08c53f 100644 --- a/internal/serve/graphql/resolvers/test_utils.go +++ b/internal/serve/graphql/resolvers/test_utils.go @@ -42,6 +42,10 @@ func getTestCtx(table string, columns []string) context.Context { return ctx } +func ptr[T any](v T) *T { + return &v +} + func setupDB(ctx context.Context, t *testing.T, dbConnectionPool db.ConnectionPool) { testLedger := int32(1000) parentAccount := &types.Account{StellarAddress: "test-account"} @@ -52,9 +56,9 @@ func setupDB(ctx context.Context, t *testing.T, dbConnectionPool db.ConnectionPo txn := &types.Transaction{ Hash: fmt.Sprintf("tx%d", i+1), ToID: toid.New(testLedger, int32(i+1), 0).ToInt64(), - EnvelopeXDR: fmt.Sprintf("envelope%d", i+1), + EnvelopeXDR: ptr(fmt.Sprintf("envelope%d", i+1)), ResultXDR: fmt.Sprintf("result%d", i+1), - MetaXDR: fmt.Sprintf("meta%d", i+1), + MetaXDR: ptr(fmt.Sprintf("meta%d", i+1)), LedgerNumber: 1, LedgerCreatedAt: time.Now(), } diff --git a/internal/serve/graphql/schema/statechange.graphqls b/internal/serve/graphql/schema/statechange.graphqls index bec14693d..8ee8c9a4f 100644 --- a/internal/serve/graphql/schema/statechange.graphqls +++ b/internal/serve/graphql/schema/statechange.graphqls @@ -7,7 +7,7 @@ interface BaseStateChange { ledgerNumber: UInt32! # GraphQL Relationships - these fields use resolvers - # Related operation - nullable since fee state changes do not have operations associated with them + # Related account account: Account! @goField(forceResolver: true) # Related operation - nullable since fee state changes do not have operations associated with them diff --git a/internal/serve/graphql/schema/transaction.graphqls b/internal/serve/graphql/schema/transaction.graphqls index 602a633b7..708332442 100644 --- a/internal/serve/graphql/schema/transaction.graphqls +++ b/internal/serve/graphql/schema/transaction.graphqls @@ -2,9 +2,9 @@ # gqlgen generates Go structs from this schema definition type Transaction{ hash: String! - envelopeXdr: String! + envelopeXdr: String resultXdr: String! - metaXdr: String! + metaXdr: String ledgerNumber: UInt32! ledgerCreatedAt: Time! ingestedAt: Time! diff --git a/internal/serve/httphandler/health.go b/internal/serve/httphandler/health.go index 58eb4e706..9675741fc 100644 --- a/internal/serve/httphandler/health.go +++ b/internal/serve/httphandler/health.go @@ -20,7 +20,7 @@ type HealthHandler struct { } const ( - ledgerCursorName = "live_ingest_cursor" + ledgerCursorName = "latest_ingest_ledger" ledgerHealthThreshold = uint32(50) ) diff --git a/internal/serve/httphandler/health_test.go b/internal/serve/httphandler/health_test.go index d22536ccb..d294336aa 100644 --- a/internal/serve/httphandler/health_test.go +++ b/internal/serve/httphandler/health_test.go @@ -31,8 +31,8 @@ func TestHealthHandler_GetHealth(t *testing.T) { defer dbConnectionPool.Close() mockMetricsService := metrics.NewMockMetricsService() - mockMetricsService.On("ObserveDBQueryDuration", "GetLatestLedgerSynced", "ingest_store", mock.AnythingOfType("float64")).Return() - mockMetricsService.On("IncDBQuery", "GetLatestLedgerSynced", "ingest_store").Return() + mockMetricsService.On("ObserveDBQueryDuration", "Get", "ingest_store", mock.AnythingOfType("float64")).Return() + mockMetricsService.On("IncDBQuery", "Get", "ingest_store").Return() defer mockMetricsService.AssertExpectations(t) models, err := data.NewModels(dbConnectionPool, mockMetricsService) @@ -40,9 +40,9 @@ func TestHealthHandler_GetHealth(t *testing.T) { t.Run("healthy - RPC and backend in sync", func(t *testing.T) { ctx := context.Background() - _, err := dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store WHERE key = 'live_ingest_cursor'") + _, err := dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store WHERE key = 'latest_ingest_ledger'") require.NoError(t, err) - _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO ingest_store (key, value) VALUES ('live_ingest_cursor', $1)", uint32(98)) + _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO ingest_store (key, value) VALUES ('latest_ingest_ledger', $1)", uint32(98)) require.NoError(t, err) mockRPCService := &services.RPCServiceMock{} @@ -75,13 +75,13 @@ func TestHealthHandler_GetHealth(t *testing.T) { assert.Equal(t, "ok", response["status"]) assert.Equal(t, float64(98), response["backend_latest_ledger"]) - _, cleanupErr := dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store WHERE key = 'live_ingest_cursor'") + _, cleanupErr := dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store WHERE key = 'latest_ingest_ledger'") require.NoError(t, cleanupErr) }) t.Run("unhealthy - RPC service error", func(t *testing.T) { ctx := context.Background() - _, err := dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store WHERE key = 'live_ingest_cursor'") + _, err := dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store WHERE key = 'latest_ingest_ledger'") require.NoError(t, err) mockRPCService := &services.RPCServiceMock{} @@ -104,15 +104,15 @@ func TestHealthHandler_GetHealth(t *testing.T) { assert.Equal(t, http.StatusInternalServerError, recorder.Code) assert.Contains(t, recorder.Body.String(), "failed to get RPC health") - _, cleanupErr := dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store WHERE key = 'live_ingest_cursor'") + _, cleanupErr := dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store WHERE key = 'latest_ingest_ledger'") require.NoError(t, cleanupErr) }) t.Run("unhealthy - RPC status not healthy", func(t *testing.T) { ctx := context.Background() - _, err := dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store WHERE key = 'live_ingest_cursor'") + _, err := dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store WHERE key = 'latest_ingest_ledger'") require.NoError(t, err) - _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO ingest_store (key, value) VALUES ('live_ingest_cursor', $1)", uint32(98)) + _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO ingest_store (key, value) VALUES ('latest_ingest_ledger', $1)", uint32(98)) require.NoError(t, err) mockRPCService := &services.RPCServiceMock{} @@ -138,15 +138,15 @@ func TestHealthHandler_GetHealth(t *testing.T) { handler.GetHealth(recorder, req) assert.Equal(t, http.StatusServiceUnavailable, recorder.Code) - _, cleanupErr := dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store WHERE key = 'live_ingest_cursor'") + _, cleanupErr := dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store WHERE key = 'latest_ingest_ledger'") require.NoError(t, cleanupErr) }) t.Run("unhealthy - backend significantly behind", func(t *testing.T) { ctx := context.Background() - _, err := dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store WHERE key = 'live_ingest_cursor'") + _, err := dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store WHERE key = 'latest_ingest_ledger'") require.NoError(t, err) - _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO ingest_store (key, value) VALUES ('live_ingest_cursor', $1)", uint32(900)) + _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO ingest_store (key, value) VALUES ('latest_ingest_ledger', $1)", uint32(900)) require.NoError(t, err) mockRPCService := &services.RPCServiceMock{} @@ -180,7 +180,7 @@ func TestHealthHandler_GetHealth(t *testing.T) { assert.Equal(t, float64(1000), response["extras"].(map[string]any)["rpc_latest_ledger"]) assert.Equal(t, float64(900), response["extras"].(map[string]any)["backend_latest_ledger"]) - _, cleanupErr := dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store WHERE key = 'live_ingest_cursor'") + _, cleanupErr := dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store WHERE key = 'latest_ingest_ledger'") require.NoError(t, cleanupErr) }) } diff --git a/internal/services/account_tokens.go b/internal/services/account_tokens.go index 801d2e9ea..3d3dda673 100644 --- a/internal/services/account_tokens.go +++ b/internal/services/account_tokens.go @@ -474,7 +474,6 @@ func (s *accountTokenService) enrichContractTypes( for wasmHash, contractCode := range contractCodesByWasmHash { contractType, err := s.contractValidator.ValidateFromContractCode(ctx, contractCode) if err != nil { - log.Ctx(ctx).Warnf("Failed to validate contract code for WASM hash %s: %v", wasmHash.HexString(), err) continue } if contractType == types.ContractTypeUnknown { diff --git a/internal/services/ingest.go b/internal/services/ingest.go index 46b5d2a92..d3b71bee3 100644 --- a/internal/services/ingest.go +++ b/internal/services/ingest.go @@ -6,22 +6,22 @@ import ( "fmt" "hash/fnv" "io" + "runtime" "sort" + "strings" "time" "github.com/alitto/pond/v2" set "github.com/deckarep/golang-set/v2" + "github.com/jackc/pgx/v5" "github.com/stellar/go/historyarchive" "github.com/stellar/go/ingest" "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/support/log" - "github.com/stellar/go/txnbuild" "github.com/stellar/go/xdr" "github.com/stellar/wallet-backend/internal/apptracker" "github.com/stellar/wallet-backend/internal/data" - "github.com/stellar/wallet-backend/internal/db" - "github.com/stellar/wallet-backend/internal/entities" "github.com/stellar/wallet-backend/internal/indexer" "github.com/stellar/wallet-backend/internal/indexer/types" "github.com/stellar/wallet-backend/internal/metrics" @@ -31,6 +31,88 @@ import ( var ErrAlreadyInSync = errors.New("ingestion is already in sync") +const ( + // HistoricalBufferLedgers is the number of ledgers to keep before latestRPCLedger + // to avoid racing with live finalization during parallel processing. + HistoricalBufferLedgers uint32 = 5 + // maxLedgerFetchRetries is the maximum number of retry attempts when fetching a ledger fails. + maxLedgerFetchRetries = 10 + // maxRetryBackoff is the maximum backoff duration between retry attempts. + maxRetryBackoff = 30 * time.Second + // IngestionModeLive represents continuous ingestion from the latest ledger onwards. + IngestionModeLive = "live" + // IngestionModeBackfill represents historical ledger ingestion for a specified range. + IngestionModeBackfill = "backfill" +) + +// LedgerBackendFactory creates new LedgerBackend instances for parallel batch processing. +// Each batch needs its own backend because LedgerBackend is not thread-safe. +type LedgerBackendFactory func(ctx context.Context) (ledgerbackend.LedgerBackend, error) + +// IngestServiceConfig holds the configuration for creating an IngestService. +type IngestServiceConfig struct { + IngestionMode string + Models *data.Models + LatestLedgerCursorName string + OldestLedgerCursorName string + AppTracker apptracker.AppTracker + RPCService RPCService + LedgerBackend ledgerbackend.LedgerBackend + LedgerBackendFactory LedgerBackendFactory + ChannelAccountStore store.ChannelAccountStore + AccountTokenService AccountTokenService + ContractMetadataService ContractMetadataService + MetricsService metrics.MetricsService + GetLedgersLimit int + Network string + NetworkPassphrase string + Archive historyarchive.ArchiveInterface + SkipTxMeta bool + SkipTxEnvelope bool + EnableParticipantFiltering bool + BackfillWorkers int + BackfillBatchSize int + BackfillDBInsertBatchSize int + CatchupThreshold int +} + +// BackfillBatch represents a contiguous range of ledgers to process as a unit. +type BackfillBatch struct { + StartLedger uint32 + EndLedger uint32 +} + +// BackfillResult tracks the outcome of processing a single batch. +type BackfillResult struct { + Batch BackfillBatch + LedgersCount int + Duration time.Duration + Error error +} + +// batchAnalysis holds the aggregated results from processing multiple backfill batches. +type batchAnalysis struct { + failedBatches []BackfillBatch + successCount int + totalLedgers int +} + +// analyzeBatchResults aggregates backfill batch results and logs any failures. +func analyzeBatchResults(ctx context.Context, results []BackfillResult) batchAnalysis { + var analysis batchAnalysis + for _, result := range results { + if result.Error != nil { + analysis.failedBatches = append(analysis.failedBatches, result.Batch) + log.Ctx(ctx).Errorf("Batch [%d-%d] failed: %v", + result.Batch.StartLedger, result.Batch.EndLedger, result.Error) + } else { + analysis.successCount++ + analysis.totalLedgers += result.LedgersCount + } + } + return analysis +} + // generateAdvisoryLockID creates a deterministic advisory lock ID based on the network name. // This ensures different networks (mainnet, testnet) get separate locks while being consistent across restarts. func generateAdvisoryLockID(network string) int { @@ -46,417 +128,365 @@ type IngestService interface { var _ IngestService = (*ingestService)(nil) type ingestService struct { - models *data.Models - ledgerCursorName string - accountTokensCursorName string - advisoryLockID int - appTracker apptracker.AppTracker - rpcService RPCService - ledgerBackend ledgerbackend.LedgerBackend - chAccStore store.ChannelAccountStore - accountTokenService AccountTokenService - contractMetadataService ContractMetadataService - metricsService metrics.MetricsService - networkPassphrase string - getLedgersLimit int - ledgerIndexer *indexer.Indexer - archive historyarchive.ArchiveInterface - backfillMode bool + ingestionMode string + models *data.Models + latestLedgerCursorName string + oldestLedgerCursorName string + advisoryLockID int + appTracker apptracker.AppTracker + rpcService RPCService + ledgerBackend ledgerbackend.LedgerBackend + ledgerBackendFactory LedgerBackendFactory + chAccStore store.ChannelAccountStore + accountTokenService AccountTokenService + contractMetadataService ContractMetadataService + metricsService metrics.MetricsService + networkPassphrase string + getLedgersLimit int + ledgerIndexer *indexer.Indexer + archive historyarchive.ArchiveInterface + enableParticipantFiltering bool + backfillPool pond.Pool + backfillBatchSize uint32 + backfillDBInsertBatchSize uint32 + catchupThreshold uint32 + backfillInstanceID string } -func NewIngestService( - models *data.Models, - ledgerCursorName string, - accountTokensCursorName string, - appTracker apptracker.AppTracker, - rpcService RPCService, - ledgerBackend ledgerbackend.LedgerBackend, - chAccStore store.ChannelAccountStore, - accountTokenService AccountTokenService, - contractMetadataService ContractMetadataService, - metricsService metrics.MetricsService, - getLedgersLimit int, - network string, - networkPassphrase string, - archive historyarchive.ArchiveInterface, -) (*ingestService, error) { +func NewIngestService(cfg IngestServiceConfig) (*ingestService, error) { // Create worker pool for the ledger indexer (parallel transaction processing within a ledger) ledgerIndexerPool := pond.NewPool(0) - metricsService.RegisterPoolMetrics("ledger_indexer", ledgerIndexerPool) + cfg.MetricsService.RegisterPoolMetrics("ledger_indexer", ledgerIndexerPool) + + // Create backfill pool with bounded size to control memory usage. + // Default to NumCPU if not specified. + backfillWorkers := cfg.BackfillWorkers + if backfillWorkers <= 0 { + backfillWorkers = runtime.NumCPU() + } + backfillPool := pond.NewPool(backfillWorkers) + cfg.MetricsService.RegisterPoolMetrics("backfill", backfillPool) return &ingestService{ - models: models, - ledgerCursorName: ledgerCursorName, - accountTokensCursorName: accountTokensCursorName, - advisoryLockID: generateAdvisoryLockID(network), - appTracker: appTracker, - rpcService: rpcService, - ledgerBackend: ledgerBackend, - chAccStore: chAccStore, - accountTokenService: accountTokenService, - contractMetadataService: contractMetadataService, - metricsService: metricsService, - networkPassphrase: networkPassphrase, - getLedgersLimit: getLedgersLimit, - ledgerIndexer: indexer.NewIndexer(networkPassphrase, ledgerIndexerPool, metricsService), - archive: archive, - backfillMode: false, + ingestionMode: cfg.IngestionMode, + models: cfg.Models, + latestLedgerCursorName: cfg.LatestLedgerCursorName, + oldestLedgerCursorName: cfg.OldestLedgerCursorName, + advisoryLockID: generateAdvisoryLockID(cfg.Network), + appTracker: cfg.AppTracker, + rpcService: cfg.RPCService, + ledgerBackend: cfg.LedgerBackend, + ledgerBackendFactory: cfg.LedgerBackendFactory, + chAccStore: cfg.ChannelAccountStore, + accountTokenService: cfg.AccountTokenService, + contractMetadataService: cfg.ContractMetadataService, + metricsService: cfg.MetricsService, + networkPassphrase: cfg.NetworkPassphrase, + getLedgersLimit: cfg.GetLedgersLimit, + ledgerIndexer: indexer.NewIndexer(cfg.NetworkPassphrase, ledgerIndexerPool, cfg.MetricsService, cfg.SkipTxMeta, cfg.SkipTxEnvelope), + archive: cfg.Archive, + enableParticipantFiltering: cfg.EnableParticipantFiltering, + backfillPool: backfillPool, + backfillBatchSize: uint32(cfg.BackfillBatchSize), + backfillDBInsertBatchSize: uint32(cfg.BackfillDBInsertBatchSize), + catchupThreshold: uint32(cfg.CatchupThreshold), }, nil } +// Run starts the ingestion service in the configured mode (live or backfill). +// For live mode, startLedger and endLedger are ignored and ingestion runs continuously from the last checkpoint. +// For backfill mode, processes ledgers in the range [startLedger, endLedger]. func (m *ingestService) Run(ctx context.Context, startLedger uint32, endLedger uint32) error { - // Acquire advisory lock to prevent multiple ingestion instances from running concurrently - if lockAcquired, err := db.AcquireAdvisoryLock(ctx, m.models.DB, m.advisoryLockID); err != nil { - return fmt.Errorf("acquiring advisory lock: %w", err) - } else if !lockAcquired { - return errors.New("advisory lock not acquired") + switch m.ingestionMode { + case IngestionModeLive: + return m.startLiveIngestion(ctx) + case IngestionModeBackfill: + return m.startBackfilling(ctx, startLedger, endLedger, BackfillModeHistorical) + default: + return fmt.Errorf("unsupported ingestion mode %q, must be %q or %q", m.ingestionMode, IngestionModeLive, IngestionModeBackfill) } - defer func() { - if err := db.ReleaseAdvisoryLock(ctx, m.models.DB, m.advisoryLockID); err != nil { - err = fmt.Errorf("releasing advisory lock: %w", err) - log.Ctx(ctx).Error(err) +} + +// getLedgerWithRetry fetches a ledger with exponential backoff retry logic. +// It respects context cancellation and limits retries to maxLedgerFetchRetries attempts. +func (m *ingestService) getLedgerWithRetry(ctx context.Context, backend ledgerbackend.LedgerBackend, ledgerSeq uint32) (xdr.LedgerCloseMeta, error) { + var lastErr error + for attempt := 0; attempt < maxLedgerFetchRetries; attempt++ { + select { + case <-ctx.Done(): + return xdr.LedgerCloseMeta{}, fmt.Errorf("context cancelled: %w", ctx.Err()) + default: } - }() - // Check if account tokens cache is populated - latestIngestedLedger, err := m.models.IngestStore.Get(ctx, m.ledgerCursorName) + ledgerMeta, err := backend.GetLedger(ctx, ledgerSeq) + if err == nil { + return ledgerMeta, nil + } + lastErr = err + + backoff := time.Duration(1< maxRetryBackoff { + backoff = maxRetryBackoff + } + log.Ctx(ctx).Warnf("Error fetching ledger %d (attempt %d/%d): %v, retrying in %v...", + ledgerSeq, attempt+1, maxLedgerFetchRetries, err, backoff) + + select { + case <-ctx.Done(): + return xdr.LedgerCloseMeta{}, fmt.Errorf("context cancelled during backoff: %w", ctx.Err()) + case <-time.After(backoff): + } + } + return xdr.LedgerCloseMeta{}, fmt.Errorf("failed after %d attempts: %w", maxLedgerFetchRetries, lastErr) +} + +func (m *ingestService) getLedgerTransactions(ctx context.Context, xdrLedgerCloseMeta xdr.LedgerCloseMeta) ([]ingest.LedgerTransaction, error) { + ledgerTxReader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(m.networkPassphrase, xdrLedgerCloseMeta) if err != nil { - return fmt.Errorf("getting latest account-tokens ledger cursor: %w", err) + return nil, fmt.Errorf("creating ledger transaction reader: %w", err) } + defer utils.DeferredClose(ctx, ledgerTxReader, "closing ledger transaction reader") - // If latestIngestedLedger == 0, then its an empty db. In that case, we get the latest checkpoint ledger - // and start from there. - if latestIngestedLedger == 0 { - startLedger, err = m.calculateCheckpointLedger(startLedger) + transactions := make([]ingest.LedgerTransaction, 0) + for { + tx, err := ledgerTxReader.Read() if err != nil { - return fmt.Errorf("calculating checkpoint ledger: %w", err) + if errors.Is(err, io.EOF) { + break + } + return nil, fmt.Errorf("reading ledger: %w", err) } - log.Ctx(ctx).Infof("Account tokens cache not populated, using checkpoint ledger: %d", startLedger) - - if populateErr := m.accountTokenService.PopulateAccountTokens(ctx, startLedger); populateErr != nil { - return fmt.Errorf("populating account tokens cache: %w", populateErr) - } - } else { - // If we already have data ingested currently, then we check the start ledger value supplied by the user. - // If it is 0 or beyond the current ingested ledger, we just start from where we left off. - if startLedger == 0 || startLedger >= latestIngestedLedger { - startLedger = latestIngestedLedger + 1 - } else { - // If start ledger is some value less than latest ingested ledger, we go into backfilling mode. In this mode - // we dont update the account token cache (since it is already populated with recent checkpoint ledger) and we dont - // update the latest ledger ingested cursor. - // NOTE: Currently we dont have the functionality of detecting gaps and intelligently backfilling so we would process the same - // ledgers again during backfilling. However the db insertions have ON CONFLICT DO NOTHING, so we would not do repeated insertions - m.backfillMode = true - } + transactions = append(transactions, tx) } - // Prepare backend range - err = m.prepareBackendRange(ctx, startLedger, endLedger) + return transactions, nil +} + +// filteredIngestionData holds the filtered data for ingestion +type filteredIngestionData struct { + txs []*types.Transaction + txParticipants map[string]set.Set[string] + ops []*types.Operation + opParticipants map[int64]set.Set[string] + stateChanges []types.StateChange +} + +// filterByRegisteredAccounts filters ingestion data to only include items +// where at least one participant is a registered account. +// If a transaction/operation has ANY registered participant, it is included with ALL its participants. +func (m *ingestService) filterByRegisteredAccounts( + ctx context.Context, + txs []*types.Transaction, + txParticipants map[string]set.Set[string], + ops []*types.Operation, + opParticipants map[int64]set.Set[string], + stateChanges []types.StateChange, + allParticipants []string, +) (*filteredIngestionData, error) { + // Get registered accounts from DB + existing, err := m.models.Account.BatchGetByIDs(ctx, allParticipants) if err != nil { - return fmt.Errorf("preparing backend range: %w", err) + return nil, fmt.Errorf("getting registered accounts: %w", err) } + registeredAccounts := set.NewSet(existing...) - currentLedger := startLedger - log.Ctx(ctx).Infof("Starting ingestion loop from ledger: %d", currentLedger) - for endLedger == 0 || currentLedger < endLedger { - ledgerMeta, ledgerErr := m.ledgerBackend.GetLedger(ctx, currentLedger) - if ledgerErr != nil { - if endLedger > 0 && currentLedger > endLedger { - log.Ctx(ctx).Infof("Backfill complete: processed ledgers %d to %d", startLedger, endLedger) - return nil + log.Ctx(ctx).Infof("filtering enabled: %d/%d participants are registered", len(existing), len(allParticipants)) + + // Filter transactions: include if ANY participant is registered + txHashesToInclude := set.NewSet[string]() + for txHash, participants := range txParticipants { + for p := range participants.Iter() { + if registeredAccounts.Contains(p) { + txHashesToInclude.Add(txHash) + break } - log.Ctx(ctx).Warnf("Error fetching ledger %d: %v, retrying...", currentLedger, ledgerErr) - time.Sleep(time.Second) - continue } + } - totalStart := time.Now() - if processErr := m.processLedger(ctx, ledgerMeta); processErr != nil { - return fmt.Errorf("processing ledger %d: %w", currentLedger, processErr) + filteredTxs := make([]*types.Transaction, 0, txHashesToInclude.Cardinality()) + filteredTxParticipants := make(map[string]set.Set[string]) + for _, tx := range txs { + if txHashesToInclude.Contains(tx.Hash) { + filteredTxs = append(filteredTxs, tx) + filteredTxParticipants[tx.Hash] = txParticipants[tx.Hash] } + } - // Update cursor only for live ingestion - if !m.backfillMode { - err := m.updateCursor(ctx, currentLedger) - if err != nil { - return fmt.Errorf("updating cursor for ledger %d: %w", currentLedger, err) + // Filter operations: include if ANY participant is registered + opIDsToInclude := set.NewSet[int64]() + for opID, participants := range opParticipants { + for p := range participants.Iter() { + if registeredAccounts.Contains(p) { + opIDsToInclude.Add(opID) + break } } - m.metricsService.ObserveIngestionDuration(time.Since(totalStart).Seconds()) - m.metricsService.IncIngestionLedgersProcessed(1) - - log.Ctx(ctx).Infof("Processed ledger %d in %v", currentLedger, time.Since(totalStart)) - currentLedger++ - - // Once we have backfilled data and caught up to the tip, we should set the backfill mode to - // false. This is because when backfilling data, we are not updating the latest ledger cursor - // and not processing any account token cache changes. Remember that the account token cache was - // already populated using a more recent checkpoint ledger so we dont need to process older data. - if m.backfillMode && currentLedger > latestIngestedLedger { - m.backfillMode = false - } - if endLedger > 0 && currentLedger > endLedger { - log.Ctx(ctx).Infof("Backfill complete: processed ledgers %d to %d", startLedger, endLedger) - return nil - } } - return nil -} -func (m *ingestService) updateCursor(ctx context.Context, currentLedger uint32) error { - cursorStart := time.Now() - err := db.RunInTransaction(ctx, m.models.DB, nil, func(dbTx db.Transaction) error { - if updateErr := m.models.IngestStore.Update(ctx, dbTx, m.ledgerCursorName, currentLedger); updateErr != nil { - return fmt.Errorf("updating latest synced ledger: %w", updateErr) + filteredOps := make([]*types.Operation, 0, opIDsToInclude.Cardinality()) + filteredOpParticipants := make(map[int64]set.Set[string]) + for _, op := range ops { + if opIDsToInclude.Contains(op.ID) { + filteredOps = append(filteredOps, op) + filteredOpParticipants[op.ID] = opParticipants[op.ID] } - m.metricsService.SetLatestLedgerIngested(float64(currentLedger)) - return nil - }) - if err != nil { - return fmt.Errorf("updating cursors: %w", err) } - m.metricsService.ObserveIngestionPhaseDuration("cursor_update", time.Since(cursorStart).Seconds()) - return nil -} -// prepareBackendRange prepares the ledger backend with the appropriate range type. -// Returns the operating mode (live streaming vs backfill). -func (m *ingestService) prepareBackendRange(ctx context.Context, startLedger, endLedger uint32) error { - var ledgerRange ledgerbackend.Range - if endLedger == 0 { - ledgerRange = ledgerbackend.UnboundedRange(startLedger) - log.Ctx(ctx).Infof("Prepared backend with unbounded range starting from ledger %d", startLedger) - } else { - ledgerRange = ledgerbackend.BoundedRange(startLedger, endLedger) - log.Ctx(ctx).Infof("Prepared backend with bounded range [%d, %d]", startLedger, endLedger) + // Filter state changes: include if account is registered + filteredSC := make([]types.StateChange, 0) + for _, sc := range stateChanges { + if registeredAccounts.Contains(sc.AccountID) { + filteredSC = append(filteredSC, sc) + } } - if err := m.ledgerBackend.PrepareRange(ctx, ledgerRange); err != nil { - return fmt.Errorf("preparing datastore backend unbounded range from %d: %w", startLedger, err) - } - return nil -} + log.Ctx(ctx).Infof("after filtering: %d txs, %d ops, %d state_changes", + len(filteredTxs), len(filteredOps), len(filteredSC)) -// calculateCheckpointLedger determines the appropriate checkpoint ledger for account token cache population. -// If startLedger is 0, it returns the latest checkpoint from the archive. -// If startLedger is specified, it returns startLedger if it's a checkpoint, otherwise the previous checkpoint. -func (m *ingestService) calculateCheckpointLedger(startLedger uint32) (uint32, error) { - archiveManager := m.archive.GetCheckpointManager() + return &filteredIngestionData{ + txs: filteredTxs, + txParticipants: filteredTxParticipants, + ops: filteredOps, + opParticipants: filteredOpParticipants, + stateChanges: filteredSC, + }, nil +} - if startLedger == 0 { - // Get latest checkpoint from archive - latestLedger, err := m.archive.GetLatestLedgerSequence() +func (m *ingestService) ingestProcessedData(ctx context.Context, indexerBuffer indexer.IndexerBufferInterface) error { + // Get data from indexer buffer + txs := indexerBuffer.GetTransactions() + txParticipants := indexerBuffer.GetTransactionsParticipants() + ops := indexerBuffer.GetOperations() + opParticipants := indexerBuffer.GetOperationsParticipants() + stateChanges := indexerBuffer.GetStateChanges() + + // When filtering is enabled, only store data for registered accounts + if m.enableParticipantFiltering { + filtered, err := m.filterByRegisteredAccounts( + ctx, txs, txParticipants, ops, opParticipants, stateChanges, + indexerBuffer.GetAllParticipants(), + ) if err != nil { - return 0, fmt.Errorf("getting latest ledger sequence: %w", err) + return fmt.Errorf("filtering by registered accounts: %w", err) } - return latestLedger, nil - } - - // For specified startLedger, use it if it's a checkpoint, otherwise use previous checkpoint - if archiveManager.IsCheckpoint(startLedger) { - return startLedger, nil + txs = filtered.txs + txParticipants = filtered.txParticipants + ops = filtered.ops + opParticipants = filtered.opParticipants + stateChanges = filtered.stateChanges } - return archiveManager.PrevCheckpoint(startLedger), nil -} -// processLedger processes a single ledger through all ingestion phases. -// Phase 1: Get transactions and collect data using Indexer (parallel within ledger) -// Phase 2: Fetch existing accounts for participants (single DB call) -// Phase 3: Process transactions and populate buffer using Indexer (parallel within ledger) -// Phase 4: Insert all data into DB -func (m *ingestService) processLedger(ctx context.Context, ledgerMeta xdr.LedgerCloseMeta) error { - ledgerSeq := ledgerMeta.LedgerSequence() - - // Phase 1: Get transactions from ledger - start := time.Now() - transactions, err := m.getLedgerTransactions(ctx, ledgerMeta) + // Use pgx transaction for BatchCopy operations (binary COPY protocol) + pgxTx, err := m.models.DB.PgxPool().Begin(ctx) if err != nil { - return fmt.Errorf("getting transactions for ledger %d: %w", ledgerSeq, err) + return fmt.Errorf("beginning pgx transaction: %w", err) } + defer func() { + if err := pgxTx.Rollback(ctx); err != nil && !errors.Is(err, pgx.ErrTxClosed) { + log.Ctx(ctx).Errorf("error rolling back pgx transaction: %v", err) + } + }() - // Phase 1b: Collect all transaction data using Indexer (parallel within ledger) - precomputedData, allParticipants, err := m.ledgerIndexer.CollectAllTransactionData(ctx, transactions) - if err != nil { - return fmt.Errorf("collecting transaction data for ledger %d: %w", ledgerSeq, err) + if err := m.insertTransactions(ctx, pgxTx, txs, txParticipants); err != nil { + return err } - m.metricsService.ObserveIngestionPhaseDuration("collect_transaction_data", time.Since(start).Seconds()) - - // Phase 2: Fetch existing accounts for participants (single DB call) - start = time.Now() - existingAccounts, err := m.models.Account.BatchGetByIDs(ctx, allParticipants.ToSlice()) - if err != nil { - return fmt.Errorf("batch checking participants for ledger %d: %w", ledgerSeq, err) + if err := m.insertOperations(ctx, pgxTx, ops, opParticipants); err != nil { + return err + } + if err := m.insertStateChanges(ctx, pgxTx, stateChanges); err != nil { + return err } - existingAccountsSet := set.NewSet(existingAccounts...) - m.metricsService.ObserveIngestionParticipantsCount(allParticipants.Cardinality()) - m.metricsService.ObserveIngestionPhaseDuration("fetch_existing_accounts", time.Since(start).Seconds()) - // Phase 3: Process transactions using Indexer (parallel within ledger) - start = time.Now() - buffer := indexer.NewIndexerBuffer() - if err := m.ledgerIndexer.ProcessTransactions(ctx, precomputedData, existingAccountsSet, buffer); err != nil { - return fmt.Errorf("processing transactions for ledger %d: %w", ledgerSeq, err) + // Unlock channel accounts only during live ingestion (skip for historical backfill) + // This is done within the same pgxTx for atomicity - all inserts and unlocks succeed or fail together + if m.ingestionMode == IngestionModeLive { + if err := m.unlockChannelAccounts(ctx, pgxTx, txs); err != nil { + return err + } } - m.metricsService.ObserveIngestionPhaseDuration("process_and_buffer", time.Since(start).Seconds()) - // Phase 4: Insert all data into DB - start = time.Now() - if err := m.ingestProcessedData(ctx, buffer); err != nil { - return fmt.Errorf("ingesting processed data for ledger %d: %w", ledgerSeq, err) + if err := pgxTx.Commit(ctx); err != nil { + return fmt.Errorf("committing pgx transaction: %w", err) } - m.metricsService.ObserveIngestionPhaseDuration("db_insertion", time.Since(start).Seconds()) - // Metrics - m.metricsService.IncIngestionTransactionsProcessed(buffer.GetNumberOfTransactions()) - m.metricsService.IncIngestionOperationsProcessed(buffer.GetNumberOfOperations()) + // Process token changes only during live ingestion (not backfill) + if m.ingestionMode == IngestionModeLive { + return m.processLiveIngestionTokenChanges(ctx, indexerBuffer) + } return nil } -func (m *ingestService) getLedgerTransactions(ctx context.Context, xdrLedgerCloseMeta xdr.LedgerCloseMeta) ([]ingest.LedgerTransaction, error) { - ledgerTxReader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(m.networkPassphrase, xdrLedgerCloseMeta) +// insertTransactions batch inserts transactions with their participants into the database. +func (m *ingestService) insertTransactions(ctx context.Context, pgxTx pgx.Tx, txs []*types.Transaction, stellarAddressesByTxHash map[string]set.Set[string]) error { + if len(txs) == 0 { + return nil + } + insertedCount, err := m.models.Transactions.BatchCopy(ctx, pgxTx, txs, stellarAddressesByTxHash) if err != nil { - return nil, fmt.Errorf("creating ledger transaction reader: %w", err) + return fmt.Errorf("batch inserting transactions: %w", err) } - defer utils.DeferredClose(ctx, ledgerTxReader, "closing ledger transaction reader") - - transactions := make([]ingest.LedgerTransaction, 0) - for { - tx, err := ledgerTxReader.Read() - if err != nil { - if errors.Is(err, io.EOF) { - break - } - return nil, fmt.Errorf("reading ledger: %w", err) - } + log.Ctx(ctx).Infof("inserted %d transactions", insertedCount) + return nil +} - transactions = append(transactions, tx) +// insertOperations batch inserts operations with their participants into the database. +func (m *ingestService) insertOperations(ctx context.Context, pgxTx pgx.Tx, ops []*types.Operation, stellarAddressesByOpID map[int64]set.Set[string]) error { + if len(ops) == 0 { + return nil } - - return transactions, nil + insertedCount, err := m.models.Operations.BatchCopy(ctx, pgxTx, ops, stellarAddressesByOpID) + if err != nil { + return fmt.Errorf("batch inserting operations: %w", err) + } + log.Ctx(ctx).Infof("inserted %d operations", insertedCount) + return nil } -func (m *ingestService) ingestProcessedData(ctx context.Context, indexerBuffer indexer.IndexerBufferInterface) error { - dbTxErr := db.RunInTransaction(ctx, m.models.DB, nil, func(dbTx db.Transaction) error { - // 2. Insert queries - // 2.1. Insert transactions - txs := indexerBuffer.GetTransactions() - stellarAddressesByTxHash := indexerBuffer.GetTransactionsParticipants() - if len(txs) > 0 { - insertedHashes, err := m.models.Transactions.BatchInsert(ctx, dbTx, txs, stellarAddressesByTxHash) - if err != nil { - return fmt.Errorf("batch inserting transactions: %w", err) - } - log.Ctx(ctx).Infof("✅ inserted %d transactions with hashes %v", len(insertedHashes), insertedHashes) - } - - // 2.2. Insert operations - ops := indexerBuffer.GetOperations() - stellarAddressesByOpID := indexerBuffer.GetOperationsParticipants() - if len(ops) > 0 { - insertedOpIDs, err := m.models.Operations.BatchInsert(ctx, dbTx, ops, stellarAddressesByOpID) - if err != nil { - return fmt.Errorf("batch inserting operations: %w", err) - } - log.Ctx(ctx).Infof("✅ inserted %d operations with IDs %v", len(insertedOpIDs), insertedOpIDs) - } - - // 2.3. Insert state changes - stateChanges := indexerBuffer.GetStateChanges() - if len(stateChanges) > 0 { - insertedStateChangeIDs, err := m.models.StateChanges.BatchInsert(ctx, dbTx, stateChanges) - if err != nil { - return fmt.Errorf("batch inserting state changes: %w", err) - } - - // Count state changes by type and category - typeCategoryCount := make(map[string]map[string]int) - for _, sc := range stateChanges { - category := string(sc.StateChangeCategory) - scType := "" - if sc.StateChangeReason != nil { - scType = string(*sc.StateChangeReason) - } - - if typeCategoryCount[scType] == nil { - typeCategoryCount[scType] = make(map[string]int) - } - typeCategoryCount[scType][category]++ - } - - for scType, categories := range typeCategoryCount { - for category, count := range categories { - m.metricsService.IncStateChanges(scType, category, count) - } - } - - log.Ctx(ctx).Infof("✅ inserted %d state changes with IDs %v", len(insertedStateChangeIDs), insertedStateChangeIDs) - } - - // 3. Unlock channel accounts. - if !m.backfillMode { - err := m.unlockChannelAccounts(ctx, txs) - if err != nil { - return fmt.Errorf("unlocking channel accounts: %w", err) - } - } - +// insertStateChanges batch inserts state changes and records metrics. +func (m *ingestService) insertStateChanges(ctx context.Context, pgxTx pgx.Tx, stateChanges []types.StateChange) error { + if len(stateChanges) == 0 { return nil - }) - if dbTxErr != nil { - return fmt.Errorf("ingesting processed data: %w", dbTxErr) } + insertedCount, err := m.models.StateChanges.BatchCopy(ctx, pgxTx, stateChanges) + if err != nil { + return fmt.Errorf("batch inserting state changes: %w", err) + } + m.recordStateChangeMetrics(stateChanges) + log.Ctx(ctx).Infof("inserted %d state changes", insertedCount) + return nil +} - if !m.backfillMode { - trustlineChanges := indexerBuffer.GetTrustlineChanges() - // Insert trustline changes in the ascending order of operation IDs using batch processing - sort.Slice(trustlineChanges, func(i, j int) bool { - return trustlineChanges[i].OperationID < trustlineChanges[j].OperationID - }) - - contractChanges := indexerBuffer.GetContractChanges() - - // Process all trustline and contract changes in a single batch using Redis pipelining - if err := m.accountTokenService.ProcessTokenChanges(ctx, trustlineChanges, contractChanges); err != nil { - log.Ctx(ctx).Errorf("processing trustline changes batch: %v", err) - return fmt.Errorf("processing trustline changes batch: %w", err) - } - log.Ctx(ctx).Infof("✅ inserted %d trustline and %d contract changes", len(trustlineChanges), len(contractChanges)) - - // Fetch and store metadata for new SAC/SEP-41 contracts discovered during live ingestion - if m.contractMetadataService != nil { - newContractTypesByID := m.filterNewContractTokens(ctx, contractChanges) - if len(newContractTypesByID) > 0 { - log.Ctx(ctx).Infof("Fetching metadata for %d new contract tokens", len(newContractTypesByID)) - if err := m.contractMetadataService.FetchAndStoreMetadata(ctx, newContractTypesByID); err != nil { - log.Ctx(ctx).Warnf("fetching new contract metadata: %v", err) - // Don't return error - we don't want to block ingestion for metadata fetch failures - } - } +// recordStateChangeMetrics aggregates state changes by reason and category, then records metrics. +func (m *ingestService) recordStateChangeMetrics(stateChanges []types.StateChange) { + counts := make(map[string]int) // key: "reason|category" + for _, sc := range stateChanges { + reason := "" + if sc.StateChangeReason != nil { + reason = string(*sc.StateChangeReason) } + key := reason + "|" + string(sc.StateChangeCategory) + counts[key]++ + } + for key, count := range counts { + parts := strings.SplitN(key, "|", 2) + m.metricsService.IncStateChanges(parts[0], parts[1], count) } - - return nil } // unlockChannelAccounts unlocks the channel accounts associated with the given transaction XDRs. -func (m *ingestService) unlockChannelAccounts(ctx context.Context, txs []types.Transaction) error { +func (m *ingestService) unlockChannelAccounts(ctx context.Context, pgxTx pgx.Tx, txs []*types.Transaction) error { if len(txs) == 0 { return nil } innerTxHashes := make([]string, 0, len(txs)) for _, tx := range txs { - innerTxHash, err := m.extractInnerTxHash(tx.EnvelopeXDR) - if err != nil { - return fmt.Errorf("extracting inner tx hash: %w", err) - } - innerTxHashes = append(innerTxHashes, innerTxHash) + innerTxHashes = append(innerTxHashes, tx.InnerTransactionHash) } - if affectedRows, err := m.chAccStore.UnassignTxAndUnlockChannelAccounts(ctx, nil, innerTxHashes...); err != nil { + if affectedRows, err := m.chAccStore.UnassignTxAndUnlockChannelAccounts(ctx, pgxTx, innerTxHashes...); err != nil { return fmt.Errorf("unlocking channel accounts with txHashes %v: %w", innerTxHashes, err) } else if affectedRows > 0 { log.Ctx(ctx).Infof("🔓 unlocked %d channel accounts", affectedRows) @@ -465,55 +495,36 @@ func (m *ingestService) unlockChannelAccounts(ctx context.Context, txs []types.T return nil } -func (m *ingestService) GetLedgerTransactions(ledger int64) ([]entities.Transaction, error) { - var ledgerTransactions []entities.Transaction - var cursor string - lastLedgerSeen := ledger - for lastLedgerSeen == ledger { - getTxnsResp, err := m.rpcService.GetTransactions(ledger, cursor, 50) - if err != nil { - return []entities.Transaction{}, fmt.Errorf("getTransactions: %w", err) - } - cursor = getTxnsResp.Cursor - for _, tx := range getTxnsResp.Transactions { - if tx.Ledger == ledger { - ledgerTransactions = append(ledgerTransactions, tx) - lastLedgerSeen = tx.Ledger - } else { - lastLedgerSeen = tx.Ledger - break - } - } - } - return ledgerTransactions, nil -} +// processLiveIngestionTokenChanges processes trustline and contract changes for live ingestion. +// This updates the Redis cache and fetches metadata for new SAC/SEP-41 contracts. +func (m *ingestService) processLiveIngestionTokenChanges(ctx context.Context, buffer indexer.IndexerBufferInterface) error { + trustlineChanges := buffer.GetTrustlineChanges() + // Sort trustline changes by operation ID in ascending order + sort.Slice(trustlineChanges, func(i, j int) bool { + return trustlineChanges[i].OperationID < trustlineChanges[j].OperationID + }) -// extractInnerTxHash takes a transaction XDR string and returns the hash of its inner transaction. -// For fee bump transactions, it returns the hash of the inner transaction. -// For regular transactions, it returns the hash of the transaction itself. -func (m *ingestService) extractInnerTxHash(txXDR string) (string, error) { - genericTx, err := txnbuild.TransactionFromXDR(txXDR) - if err != nil { - return "", fmt.Errorf("deserializing envelope xdr %q: %w", txXDR, err) - } + contractChanges := buffer.GetContractChanges() - var innerTx *txnbuild.Transaction - feeBumpTx, ok := genericTx.FeeBump() - if ok { - innerTx = feeBumpTx.InnerTransaction() - } else { - innerTx, ok = genericTx.Transaction() - if !ok { - return "", errors.New("transaction is neither fee bump nor inner transaction") - } + // Process all trustline and contract changes in a single batch using Redis pipelining + if err := m.accountTokenService.ProcessTokenChanges(ctx, trustlineChanges, contractChanges); err != nil { + log.Ctx(ctx).Errorf("processing trustline changes batch: %v", err) + return fmt.Errorf("processing trustline changes batch: %w", err) } - - innerTxHash, err := innerTx.HashHex(m.rpcService.NetworkPassphrase()) - if err != nil { - return "", fmt.Errorf("generating hash hex: %w", err) + log.Ctx(ctx).Infof("✅ inserted %d trustline and %d contract changes", len(trustlineChanges), len(contractChanges)) + + // Fetch and store metadata for new SAC/SEP-41 contracts + if m.contractMetadataService != nil { + newContractTypesByID := m.filterNewContractTokens(ctx, contractChanges) + if len(newContractTypesByID) > 0 { + log.Ctx(ctx).Infof("Fetching metadata for %d new contract tokens", len(newContractTypesByID)) + if err := m.contractMetadataService.FetchAndStoreMetadata(ctx, newContractTypesByID); err != nil { + log.Ctx(ctx).Warnf("fetching new contract metadata: %v", err) + // Don't return error - we don't want to block ingestion for metadata fetch failures + } + } } - - return innerTxHash, nil + return nil } // filterNewContractTokens extracts unique SAC/SEP-41 contract IDs from contract changes, diff --git a/internal/services/ingest_backfill.go b/internal/services/ingest_backfill.go new file mode 100644 index 000000000..4c312575b --- /dev/null +++ b/internal/services/ingest_backfill.go @@ -0,0 +1,384 @@ +package services + +import ( + "context" + "fmt" + "time" + + "github.com/stellar/go/ingest/ledgerbackend" + "github.com/stellar/go/support/log" + "github.com/stellar/go/xdr" + + "github.com/stellar/wallet-backend/internal/data" + "github.com/stellar/wallet-backend/internal/db" + "github.com/stellar/wallet-backend/internal/indexer" +) + +// BackfillMode indicates the purpose of backfilling. +type BackfillMode int + +const ( + // BackfillModeHistorical fills gaps within already-ingested ledger range. + BackfillModeHistorical BackfillMode = iota + // BackfillModeCatchup fills forward gaps to catch up to network tip. + BackfillModeCatchup +) + +// startBackfilling processes ledgers in the specified range, identifying gaps +// and processing them in parallel batches. The mode parameter determines: +// - BackfillModeHistorical: fills gaps within already-ingested range +// - BackfillModeCatchup: catches up to network tip from latest ingested ledger +func (m *ingestService) startBackfilling(ctx context.Context, startLedger, endLedger uint32, mode BackfillMode) error { + if startLedger > endLedger { + return fmt.Errorf("start ledger cannot be greater than end ledger") + } + + latestIngestedLedger, err := m.models.IngestStore.Get(ctx, m.latestLedgerCursorName) + if err != nil { + return fmt.Errorf("getting latest ledger cursor: %w", err) + } + + // Validate based on mode + switch mode { + case BackfillModeHistorical: + if endLedger > latestIngestedLedger { + return fmt.Errorf("end ledger %d cannot be greater than latest ingested ledger %d for backfilling", endLedger, latestIngestedLedger) + } + case BackfillModeCatchup: + if startLedger != latestIngestedLedger+1 { + return fmt.Errorf("catchup must start from ledger %d (latestIngestedLedger + 1), got %d", latestIngestedLedger+1, startLedger) + } + } + + startTime := time.Now() + done := make(chan struct{}) + + // Start elapsed time updater goroutine (only for historical backfill) + if mode == BackfillModeHistorical { + go func() { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + m.metricsService.SetBackfillElapsed(m.backfillInstanceID, time.Since(startTime).Seconds()) + case <-done: + return + } + } + }() + } + + // Determine gaps to fill based on mode + var gaps []data.LedgerRange + if mode == BackfillModeCatchup { + // For catchup, treat entire range as a single gap (no existing data in this range) + gaps = []data.LedgerRange{{GapStart: startLedger, GapEnd: endLedger}} + } else { + m.backfillInstanceID = fmt.Sprintf("%d-%d", startLedger, endLedger) + gaps, err = m.calculateBackfillGaps(ctx, startLedger, endLedger) + if err != nil { + return fmt.Errorf("calculating backfill gaps: %w", err) + } + } + if len(gaps) == 0 { + log.Ctx(ctx).Infof("No gaps to backfill in range [%d - %d]", startLedger, endLedger) + return nil + } + + backfillBatches := m.splitGapsIntoBatches(gaps) + results := m.processBackfillBatchesParallel(ctx, backfillBatches, mode) + close(done) + duration := time.Since(startTime) + if mode == BackfillModeHistorical { + m.metricsService.SetBackfillElapsed(m.backfillInstanceID, duration.Seconds()) + } + + analysis := analyzeBatchResults(ctx, results) + + // Record failed batches metric (only for historical backfill) + if mode == BackfillModeHistorical { + for range analysis.failedBatches { + m.metricsService.IncBackfillBatchesFailed(m.backfillInstanceID) + } + } + if len(analysis.failedBatches) > 0 { + return fmt.Errorf("backfilling failed: %d/%d batches failed", len(analysis.failedBatches), len(backfillBatches)) + } + + // Update cursors based on mode + switch mode { + case BackfillModeHistorical: + if err := m.updateOldestLedgerCursor(ctx, startLedger); err != nil { + return fmt.Errorf("updating oldest cursor: %w", err) + } + case BackfillModeCatchup: + if err := m.updateLatestLedgerCursorAfterCatchup(ctx, endLedger); err != nil { + return fmt.Errorf("updating latest cursor after catchup: %w", err) + } + } + + log.Ctx(ctx).Infof("Backfilling completed in %v: %d batches, %d ledgers", duration, analysis.successCount, analysis.totalLedgers) + return nil +} + +// calculateBackfillGaps determines which ledger ranges need to be backfilled based on +// the requested range, oldest ingested ledger, and any existing gaps in the data. +func (m *ingestService) calculateBackfillGaps(ctx context.Context, startLedger, endLedger uint32) ([]data.LedgerRange, error) { + // Get oldest ledger ingested (endLedger <= latestIngestedLedger is guaranteed by caller) + oldestIngestedLedger, err := m.models.IngestStore.Get(ctx, m.oldestLedgerCursorName) + if err != nil { + return nil, fmt.Errorf("getting oldest ingest ledger: %w", err) + } + + currentGaps, err := m.models.IngestStore.GetLedgerGaps(ctx) + if err != nil { + return nil, fmt.Errorf("calculating gaps in ledger range: %w", err) + } + + newGaps := make([]data.LedgerRange, 0) + switch { + case endLedger <= oldestIngestedLedger: + // Case 1: End ledger matches/less than oldest - backfill [start, min(end, oldest-1)] + if oldestIngestedLedger > 0 { + newGaps = append(newGaps, data.LedgerRange{ + GapStart: startLedger, + GapEnd: min(endLedger, oldestIngestedLedger-1), + }) + } + + case startLedger < oldestIngestedLedger: + // Case 2: Overlaps with existing range - backfill before oldest + internal gaps + if oldestIngestedLedger > 0 { + newGaps = append(newGaps, data.LedgerRange{ + GapStart: startLedger, + GapEnd: oldestIngestedLedger - 1, + }) + } + for _, gap := range currentGaps { + if gap.GapStart > endLedger { + break + } + newGaps = append(newGaps, data.LedgerRange{ + GapStart: gap.GapStart, + GapEnd: min(gap.GapEnd, endLedger), + }) + } + + default: + // Case 3: Entirely within existing range - only fill internal gaps + for _, gap := range currentGaps { + if gap.GapEnd < startLedger { + continue + } + if gap.GapStart > endLedger { + break + } + newGaps = append(newGaps, data.LedgerRange{ + GapStart: max(gap.GapStart, startLedger), + GapEnd: min(gap.GapEnd, endLedger), + }) + } + } + + return newGaps, nil +} + +// splitGapsIntoBatches divides ledger gaps into fixed-size batches for parallel processing. +func (m *ingestService) splitGapsIntoBatches(gaps []data.LedgerRange) []BackfillBatch { + var batches []BackfillBatch + + for _, gap := range gaps { + start := gap.GapStart + for start <= gap.GapEnd { + end := min(start+m.backfillBatchSize-1, gap.GapEnd) + batches = append(batches, BackfillBatch{ + StartLedger: start, + EndLedger: end, + }) + start = end + 1 + } + } + + return batches +} + +// processBackfillBatchesParallel processes backfill batches in parallel using a worker pool. +func (m *ingestService) processBackfillBatchesParallel(ctx context.Context, batches []BackfillBatch, mode BackfillMode) []BackfillResult { + results := make([]BackfillResult, len(batches)) + group := m.backfillPool.NewGroupContext(ctx) + + for i, batch := range batches { + group.Submit(func() { + result := m.processSingleBatch(ctx, batch, mode) + results[i] = result + }) + } + + if err := group.Wait(); err != nil { + log.Ctx(ctx).Warnf("Backfill batch group wait returned error: %v", err) + } + return results +} + +// processSingleBatch processes a single backfill batch with its own ledger backend. +func (m *ingestService) processSingleBatch(ctx context.Context, batch BackfillBatch, mode BackfillMode) BackfillResult { + start := time.Now() + result := BackfillResult{Batch: batch} + recordMetrics := mode == BackfillModeHistorical + + // Create a new ledger backend for this batch + backend, err := m.ledgerBackendFactory(ctx) + if err != nil { + result.Error = fmt.Errorf("creating ledger backend: %w", err) + result.Duration = time.Since(start) + return result + } + defer func() { + if closeErr := backend.Close(); closeErr != nil { + log.Ctx(ctx).Warnf("Error closing ledger backend for batch [%d-%d]: %v", + batch.StartLedger, batch.EndLedger, closeErr) + } + }() + + // Prepare the range for this batch + ledgerRange := ledgerbackend.BoundedRange(batch.StartLedger, batch.EndLedger) + if err := backend.PrepareRange(ctx, ledgerRange); err != nil { + result.Error = fmt.Errorf("preparing backend range: %w", err) + result.Duration = time.Since(start) + return result + } + + // Process each ledger in the batch using a single shared buffer. + // Periodically flush to DB to control memory usage. + batchBuffer := indexer.NewIndexerBuffer() + ledgersInBuffer := uint32(0) + + // Process each ledger in the batch sequentially + for ledgerSeq := batch.StartLedger; ledgerSeq <= batch.EndLedger; ledgerSeq++ { + fetchStart := time.Now() + ledgerMeta, err := m.getLedgerWithRetry(ctx, backend, ledgerSeq) + if err != nil { + result.Error = fmt.Errorf("getting ledger %d: %w", ledgerSeq, err) + result.Duration = time.Since(start) + return result + } + if recordMetrics { + m.metricsService.ObserveBackfillPhaseDuration(m.backfillInstanceID, "get_ledger_from_backend", time.Since(fetchStart).Seconds()) + } + + err = m.processBackfillLedger(ctx, ledgerMeta, batchBuffer, recordMetrics) + if err != nil { + result.Error = fmt.Errorf("processing ledger %d: %w", ledgerSeq, err) + result.Duration = time.Since(start) + return result + } + result.LedgersCount++ + ledgersInBuffer++ + + // Flush buffer periodically to control memory usage + if ledgersInBuffer >= m.backfillDBInsertBatchSize { + if recordMetrics { + m.metricsService.IncBackfillTransactionsProcessed(m.backfillInstanceID, batchBuffer.GetNumberOfTransactions()) + m.metricsService.IncBackfillOperationsProcessed(m.backfillInstanceID, batchBuffer.GetNumberOfOperations()) + } + + insertStart := time.Now() + if err := m.ingestProcessedData(ctx, batchBuffer); err != nil { + result.Error = fmt.Errorf("ingesting data for ledgers ending at %d: %w", ledgerSeq, err) + result.Duration = time.Since(insertStart) + return result + } + if recordMetrics { + m.metricsService.ObserveBackfillPhaseDuration(m.backfillInstanceID, "insert_into_db", time.Since(insertStart).Seconds()) + } + batchBuffer.Clear() + ledgersInBuffer = 0 + } + } + + // Flush remaining data in buffer + if ledgersInBuffer > 0 { + if recordMetrics { + m.metricsService.IncBackfillTransactionsProcessed(m.backfillInstanceID, batchBuffer.GetNumberOfTransactions()) + m.metricsService.IncBackfillOperationsProcessed(m.backfillInstanceID, batchBuffer.GetNumberOfOperations()) + } + + insertStart := time.Now() + if err := m.ingestProcessedData(ctx, batchBuffer); err != nil { + result.Error = fmt.Errorf("ingesting final data for batch [%d - %d]: %w", batch.StartLedger, batch.EndLedger, err) + result.Duration = time.Since(insertStart) + return result + } + if recordMetrics { + m.metricsService.ObserveBackfillPhaseDuration(m.backfillInstanceID, "insert_into_db", time.Since(insertStart).Seconds()) + } + } + + result.Duration = time.Since(start) + if recordMetrics { + m.metricsService.ObserveBackfillPhaseDuration(m.backfillInstanceID, "batch_processing", result.Duration.Seconds()) + m.metricsService.IncBackfillBatchesCompleted(m.backfillInstanceID) + m.metricsService.IncBackfillLedgersProcessed(m.backfillInstanceID, result.LedgersCount) + } + log.Ctx(ctx).Infof("Batch [%d - %d] completed: %d ledgers in %v", + batch.StartLedger, batch.EndLedger, result.LedgersCount, result.Duration) + + return result +} + +// processBackfillLedger processes a ledger and populates the provided buffer. +func (m *ingestService) processBackfillLedger(ctx context.Context, ledgerMeta xdr.LedgerCloseMeta, buffer *indexer.IndexerBuffer, recordMetrics bool) error { + ledgerSeq := ledgerMeta.LedgerSequence() + + // Get transactions from ledger + start := time.Now() + transactions, err := m.getLedgerTransactions(ctx, ledgerMeta) + if err != nil { + return fmt.Errorf("getting transactions for ledger %d: %w", ledgerSeq, err) + } + + // Process transactions and populate buffer (combined collection + processing) + _, err = m.ledgerIndexer.ProcessLedgerTransactions(ctx, transactions, buffer) + if err != nil { + return fmt.Errorf("processing transactions for ledger %d: %w", ledgerSeq, err) + } + if recordMetrics { + m.metricsService.ObserveBackfillPhaseDuration(m.backfillInstanceID, "process_ledger_transactions", time.Since(start).Seconds()) + } + return nil +} + +// updateOldestLedgerCursor updates the oldest ledger cursor during backfill with metrics tracking. +func (m *ingestService) updateOldestLedgerCursor(ctx context.Context, currentLedger uint32) error { + cursorStart := time.Now() + err := db.RunInTransaction(ctx, m.models.DB, nil, func(dbTx db.Transaction) error { + if updateErr := m.models.IngestStore.UpdateMin(ctx, dbTx, m.oldestLedgerCursorName, currentLedger); updateErr != nil { + return fmt.Errorf("updating oldest synced ledger: %w", updateErr) + } + m.metricsService.SetOldestLedgerIngested(float64(currentLedger)) + return nil + }) + if err != nil { + return fmt.Errorf("updating cursors: %w", err) + } + m.metricsService.ObserveIngestionPhaseDuration("oldest_cursor_update", time.Since(cursorStart).Seconds()) + return nil +} + +// updateLatestLedgerCursorAfterCatchup updates the latest ledger cursor after catchup completes. +func (m *ingestService) updateLatestLedgerCursorAfterCatchup(ctx context.Context, ledger uint32) error { + cursorStart := time.Now() + err := db.RunInTransaction(ctx, m.models.DB, nil, func(dbTx db.Transaction) error { + if updateErr := m.models.IngestStore.Update(ctx, dbTx, m.latestLedgerCursorName, ledger); updateErr != nil { + return fmt.Errorf("updating latest synced ledger: %w", updateErr) + } + m.metricsService.SetLatestLedgerIngested(float64(ledger)) + return nil + }) + if err != nil { + return fmt.Errorf("updating cursors: %w", err) + } + m.metricsService.ObserveIngestionPhaseDuration("catchup_cursor_update", time.Since(cursorStart).Seconds()) + return nil +} diff --git a/internal/services/ingest_live.go b/internal/services/ingest_live.go new file mode 100644 index 000000000..0a741c131 --- /dev/null +++ b/internal/services/ingest_live.go @@ -0,0 +1,183 @@ +package services + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/stellar/go/ingest/ledgerbackend" + "github.com/stellar/go/support/log" + "github.com/stellar/go/xdr" + + "github.com/stellar/wallet-backend/internal/db" + "github.com/stellar/wallet-backend/internal/indexer" +) + +// startLiveIngestion begins continuous ingestion from the last checkpoint ledger, +// acquiring an advisory lock to prevent concurrent ingestion instances. +func (m *ingestService) startLiveIngestion(ctx context.Context) error { + // Acquire advisory lock to prevent multiple ingestion instances from running concurrently + if lockAcquired, err := db.AcquireAdvisoryLock(ctx, m.models.DB, m.advisoryLockID); err != nil { + return fmt.Errorf("acquiring advisory lock: %w", err) + } else if !lockAcquired { + return errors.New("advisory lock not acquired") + } + defer func() { + if err := db.ReleaseAdvisoryLock(ctx, m.models.DB, m.advisoryLockID); err != nil { + err = fmt.Errorf("releasing advisory lock: %w", err) + log.Ctx(ctx).Error(err) + } + }() + + // Get latest ingested ledger to determine DB state + latestIngestedLedger, err := m.models.IngestStore.Get(ctx, m.latestLedgerCursorName) + if err != nil { + return fmt.Errorf("getting latest ledger cursor: %w", err) + } + + startLedger := latestIngestedLedger + 1 + if latestIngestedLedger == 0 { + startLedger, err = m.archive.GetLatestLedgerSequence() + if err != nil { + return fmt.Errorf("getting latest ledger sequence: %w", err) + } + + err = m.accountTokenService.PopulateAccountTokens(ctx, startLedger) + if err != nil { + return fmt.Errorf("populating account tokens cache: %w", err) + } + + err := m.initializeCursors(ctx, startLedger) + if err != nil { + return fmt.Errorf("initializing cursors: %w", err) + } + } else { + // If we already have data in the DB, we will do an optimized catchup by parallely backfilling the ledgers. + health, err := m.rpcService.GetHealth() + if err != nil { + return fmt.Errorf("getting health check result from RPC: %w", err) + } + + networkLatestLedger := health.LatestLedger + if networkLatestLedger > startLedger && (networkLatestLedger-startLedger) >= m.catchupThreshold { + log.Ctx(ctx).Infof("Wallet backend has fallen behind network tip by %d ledgers. Doing optimized catchup to the tip: %d", networkLatestLedger-startLedger, networkLatestLedger) + err := m.startBackfilling(ctx, startLedger, networkLatestLedger, BackfillModeCatchup) + if err != nil { + return fmt.Errorf("catching up to network tip: %w", err) + } + // Update startLedger to continue from where catchup ended + startLedger = networkLatestLedger + 1 + } + } + + // Start unbounded ingestion from latest ledger ingested onwards + ledgerRange := ledgerbackend.UnboundedRange(startLedger) + if err := m.ledgerBackend.PrepareRange(ctx, ledgerRange); err != nil { + return fmt.Errorf("preparing unbounded ledger backend range from %d: %w", startLedger, err) + } + return m.ingestLiveLedgers(ctx, startLedger) +} + +// initializeCursors initializes both latest and oldest cursors to the same starting ledger. +func (m *ingestService) initializeCursors(ctx context.Context, ledger uint32) error { + err := db.RunInTransaction(ctx, m.models.DB, nil, func(dbTx db.Transaction) error { + if err := m.models.IngestStore.Update(ctx, dbTx, m.latestLedgerCursorName, ledger); err != nil { + return fmt.Errorf("initializing latest cursor: %w", err) + } + if err := m.models.IngestStore.Update(ctx, dbTx, m.oldestLedgerCursorName, ledger); err != nil { + return fmt.Errorf("initializing oldest cursor: %w", err) + } + return nil + }) + if err != nil { + return fmt.Errorf("initializing cursors: %w", err) + } + return nil +} + +// ingestLiveLedgers continuously processes ledgers starting from startLedger, +// updating cursors and metrics after each successful ledger. +func (m *ingestService) ingestLiveLedgers(ctx context.Context, startLedger uint32) error { + currentLedger := startLedger + log.Ctx(ctx).Infof("Starting ingestion from ledger: %d", currentLedger) + for { + totalStart := time.Now() + ledgerMeta, ledgerErr := m.getLedgerWithRetry(ctx, m.ledgerBackend, currentLedger) + if ledgerErr != nil { + return fmt.Errorf("fetching ledger %d: %w", currentLedger, ledgerErr) + } + m.metricsService.ObserveIngestionPhaseDuration("get_ledger_from_backend", time.Since(totalStart).Seconds()) + + if processErr := m.processLiveLedger(ctx, ledgerMeta); processErr != nil { + return fmt.Errorf("processing ledger %d: %w", currentLedger, processErr) + } + + // Update cursor only for live ingestion + err := m.updateLatestLedgerCursor(ctx, currentLedger) + if err != nil { + return fmt.Errorf("updating cursor for ledger %d: %w", currentLedger, err) + } + m.metricsService.ObserveIngestionDuration(time.Since(totalStart).Seconds()) + m.metricsService.IncIngestionLedgersProcessed(1) + + log.Ctx(ctx).Infof("Processed ledger %d in %v", currentLedger, time.Since(totalStart)) + currentLedger++ + } +} + +// processLiveLedger processes a single ledger through all ingestion phases. +// Phase 1: Get transactions from ledger +// Phase 2: Process transactions using Indexer (parallel within ledger) +// Phase 3: Insert all data into DB +// Note: Live ingestion includes Redis cache updates and channel account unlocks, +// while backfill mode skips these operations (determined by m.ingestionMode). +func (m *ingestService) processLiveLedger(ctx context.Context, ledgerMeta xdr.LedgerCloseMeta) error { + ledgerSeq := ledgerMeta.LedgerSequence() + + // Phase 1: Get transactions from ledger + start := time.Now() + transactions, err := m.getLedgerTransactions(ctx, ledgerMeta) + if err != nil { + return fmt.Errorf("getting transactions for ledger %d: %w", ledgerSeq, err) + } + + // Phase 2: Process transactions using Indexer (parallel within ledger) + buffer := indexer.NewIndexerBuffer() + participantCount, err := m.ledgerIndexer.ProcessLedgerTransactions(ctx, transactions, buffer) + if err != nil { + return fmt.Errorf("processing transactions for ledger %d: %w", ledgerSeq, err) + } + m.metricsService.ObserveIngestionParticipantsCount(participantCount) + m.metricsService.ObserveIngestionPhaseDuration("process_ledger_transactions", time.Since(start).Seconds()) + + // Phase 3: Insert all data into DB + start = time.Now() + if err := m.ingestProcessedData(ctx, buffer); err != nil { + return fmt.Errorf("ingesting processed data for ledger %d: %w", ledgerSeq, err) + } + m.metricsService.ObserveIngestionPhaseDuration("insert_into_db", time.Since(start).Seconds()) + + // Record transaction and operation processing metrics + m.metricsService.IncIngestionTransactionsProcessed(buffer.GetNumberOfTransactions()) + m.metricsService.IncIngestionOperationsProcessed(buffer.GetNumberOfOperations()) + + return nil +} + +// updateLatestLedgerCursor updates the latest ledger cursor during live ingestion with metrics tracking. +func (m *ingestService) updateLatestLedgerCursor(ctx context.Context, currentLedger uint32) error { + cursorStart := time.Now() + err := db.RunInTransaction(ctx, m.models.DB, nil, func(dbTx db.Transaction) error { + if updateErr := m.models.IngestStore.Update(ctx, dbTx, m.latestLedgerCursorName, currentLedger); updateErr != nil { + return fmt.Errorf("updating latest synced ledger: %w", updateErr) + } + m.metricsService.SetLatestLedgerIngested(float64(currentLedger)) + return nil + }) + if err != nil { + return fmt.Errorf("updating cursors: %w", err) + } + m.metricsService.ObserveIngestionPhaseDuration("latest_cursor_update", time.Since(cursorStart).Seconds()) + return nil +} diff --git a/internal/services/ingest_test.go b/internal/services/ingest_test.go index ce718a978..c6eec77a7 100644 --- a/internal/services/ingest_test.go +++ b/internal/services/ingest_test.go @@ -2,11 +2,11 @@ package services import ( "context" + "fmt" "testing" - "github.com/stellar/go/keypair" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/network" - "github.com/stellar/go/txnbuild" "github.com/stellar/go/xdr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -16,7 +16,6 @@ import ( "github.com/stellar/wallet-backend/internal/data" "github.com/stellar/wallet-backend/internal/db" "github.com/stellar/wallet-backend/internal/db/dbtest" - "github.com/stellar/wallet-backend/internal/entities" "github.com/stellar/wallet-backend/internal/metrics" "github.com/stellar/wallet-backend/internal/signing/store" ) @@ -25,164 +24,6 @@ const ( defaultGetLedgersLimit = 50 ) -func TestGetLedgerTransactions(t *testing.T) { - dbt := dbtest.Open(t) - defer dbt.Close() - - dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) - require.NoError(t, err) - defer dbConnectionPool.Close() - - mockMetricsService := metrics.NewMockMetricsService() - mockMetricsService.On("RegisterPoolMetrics", "ledger_indexer", mock.Anything).Return() - models, err := data.NewModels(dbConnectionPool, mockMetricsService) - require.NoError(t, err) - mockAppTracker := apptracker.MockAppTracker{} - mockRPCService := RPCServiceMock{} - mockRPCService.On("NetworkPassphrase").Return(network.TestNetworkPassphrase) - mockChAccStore := &store.ChannelAccountStoreMock{} - mockLedgerBackend := &LedgerBackendMock{} - mockArchive := &HistoryArchiveMock{} - ingestService, err := NewIngestService(models, "ingestionLedger", "accountTokensCursor", &mockAppTracker, &mockRPCService, mockLedgerBackend, mockChAccStore, nil, nil, mockMetricsService, defaultGetLedgersLimit, network.TestNetworkPassphrase, network.TestNetworkPassphrase, mockArchive) - require.NoError(t, err) - t.Run("all_ledger_transactions_in_single_gettransactions_call", func(t *testing.T) { - defer mockMetricsService.AssertExpectations(t) - - rpcGetTransactionsResult := entities.RPCGetTransactionsResult{ - Cursor: "51", - Transactions: []entities.Transaction{ - { - Status: entities.SuccessStatus, - Hash: "hash1", - Ledger: 1, - }, - { - Status: entities.FailedStatus, - Hash: "hash2", - Ledger: 2, - }, - }, - } - mockRPCService. - On("GetTransactions", int64(1), "", 50). - Return(rpcGetTransactionsResult, nil). - Once() - - txns, err := ingestService.GetLedgerTransactions(1) - assert.Equal(t, 1, len(txns)) - assert.Equal(t, txns[0].Hash, "hash1") - assert.NoError(t, err) - }) - - t.Run("ledger_transactions_split_between_multiple_gettransactions_calls", func(t *testing.T) { - defer mockMetricsService.AssertExpectations(t) - - rpcGetTransactionsResult1 := entities.RPCGetTransactionsResult{ - Cursor: "51", - Transactions: []entities.Transaction{ - { - Status: entities.SuccessStatus, - Hash: "hash1", - Ledger: 1, - }, - { - Status: entities.FailedStatus, - Hash: "hash2", - Ledger: 1, - }, - }, - } - rpcGetTransactionsResult2 := entities.RPCGetTransactionsResult{ - Cursor: "51", - Transactions: []entities.Transaction{ - { - Status: entities.SuccessStatus, - Hash: "hash3", - Ledger: 1, - }, - { - Status: entities.FailedStatus, - Hash: "hash4", - Ledger: 2, - }, - }, - } - - mockRPCService. - On("GetTransactions", int64(1), "", 50). - Return(rpcGetTransactionsResult1, nil). - Once() - - mockRPCService. - On("GetTransactions", int64(1), "51", 50). - Return(rpcGetTransactionsResult2, nil). - Once() - - txns, err := ingestService.GetLedgerTransactions(1) - assert.Equal(t, 3, len(txns)) - assert.Equal(t, txns[0].Hash, "hash1") - assert.Equal(t, txns[1].Hash, "hash2") - assert.Equal(t, txns[2].Hash, "hash3") - assert.NoError(t, err) - }) -} - -func Test_ingestService_extractInnerTxHash(t *testing.T) { - networkPassphrase := network.TestNetworkPassphrase - sourceAccountKP := keypair.MustRandom() - destAccountKP := keypair.MustRandom() - - // Create a simple inner transaction - innerTx, err := txnbuild.NewTransaction(txnbuild.TransactionParams{ - SourceAccount: &txnbuild.SimpleAccount{ - AccountID: sourceAccountKP.Address(), - Sequence: 1, - }, - Operations: []txnbuild.Operation{&txnbuild.CreateAccount{ - Destination: destAccountKP.Address(), - Amount: "1", - }}, - Preconditions: txnbuild.Preconditions{TimeBounds: txnbuild.NewTimeout(10)}, - IncrementSequenceNum: true, - BaseFee: txnbuild.MinBaseFee, - }) - require.NoError(t, err) - innerTx, err = innerTx.Sign(networkPassphrase, sourceAccountKP) - require.NoError(t, err) - - innerTxHash, err := innerTx.HashHex(networkPassphrase) - require.NoError(t, err) - innerTxXDR, err := innerTx.Base64() - require.NoError(t, err) - - // Create a fee bump transaction - feeBumpAccountKP := keypair.MustRandom() - feeBumpTx, err := txnbuild.NewFeeBumpTransaction(txnbuild.FeeBumpTransactionParams{ - Inner: innerTx, - FeeAccount: feeBumpAccountKP.Address(), - BaseFee: 2 * txnbuild.MinBaseFee, - }) - require.NoError(t, err) - feeBumpTx, err = feeBumpTx.Sign(networkPassphrase, feeBumpAccountKP) - require.NoError(t, err) - feeBumpTxXDR, err := feeBumpTx.Base64() - require.NoError(t, err) - - ingestSvc := ingestService{rpcService: &rpcService{networkPassphrase: networkPassphrase}} - - t.Run("🟢inner_tx_hash", func(t *testing.T) { - gotTxHash, err := ingestSvc.extractInnerTxHash(innerTxXDR) - require.NoError(t, err) - assert.Equal(t, innerTxHash, gotTxHash) - }) - - t.Run("🟢fee_bump_tx_hash", func(t *testing.T) { - gotTxHash, err := ingestSvc.extractInnerTxHash(feeBumpTxXDR) - require.NoError(t, err) - assert.Equal(t, innerTxHash, gotTxHash) - }) -} - func Test_ingestService_getLedgerTransactions(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -217,6 +58,7 @@ func Test_ingestService_getLedgerTransactions(t *testing.T) { t.Run(tc.name, func(t *testing.T) { mockMetricsService := metrics.NewMockMetricsService() mockMetricsService.On("RegisterPoolMetrics", "ledger_indexer", mock.Anything).Return() + mockMetricsService.On("RegisterPoolMetrics", "backfill", mock.Anything).Return() defer mockMetricsService.AssertExpectations(t) models, err := data.NewModels(dbConnectionPool, mockMetricsService) require.NoError(t, err) @@ -227,7 +69,25 @@ func Test_ingestService_getLedgerTransactions(t *testing.T) { mockChAccStore := &store.ChannelAccountStoreMock{} mockLedgerBackend := &LedgerBackendMock{} mockArchive := &HistoryArchiveMock{} - ingestService, err := NewIngestService(models, "testCursor", "accountTokensCursor", &mockAppTracker, &mockRPCService, mockLedgerBackend, mockChAccStore, nil, nil, mockMetricsService, defaultGetLedgersLimit, network.TestNetworkPassphrase, network.TestNetworkPassphrase, mockArchive) + ingestService, err := NewIngestService(IngestServiceConfig{ + IngestionMode: IngestionModeLive, + Models: models, + LatestLedgerCursorName: "testCursor", + AppTracker: &mockAppTracker, + RPCService: &mockRPCService, + LedgerBackend: mockLedgerBackend, + ChannelAccountStore: mockChAccStore, + AccountTokenService: nil, + ContractMetadataService: nil, + MetricsService: mockMetricsService, + GetLedgersLimit: defaultGetLedgersLimit, + Network: network.TestNetworkPassphrase, + NetworkPassphrase: network.TestNetworkPassphrase, + Archive: mockArchive, + SkipTxMeta: false, + SkipTxEnvelope: false, + EnableParticipantFiltering: false, + }) require.NoError(t, err) var xdrLedgerCloseMeta xdr.LedgerCloseMeta @@ -253,3 +113,433 @@ func Test_ingestService_getLedgerTransactions(t *testing.T) { }) } } + +func Test_generateAdvisoryLockID(t *testing.T) { + testCases := []struct { + name string + network string + expected int + }{ + { + name: "testnet_generates_consistent_id", + network: "testnet", + expected: generateAdvisoryLockID("testnet"), + }, + { + name: "mainnet_generates_consistent_id", + network: "mainnet", + expected: generateAdvisoryLockID("mainnet"), + }, + { + name: "different_networks_generate_different_ids", + network: "testnet", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := generateAdvisoryLockID(tc.network) + + if tc.name == "different_networks_generate_different_ids" { + mainnetID := generateAdvisoryLockID("mainnet") + testnetID := generateAdvisoryLockID("testnet") + assert.NotEqual(t, mainnetID, testnetID, "different networks should generate different lock IDs") + } else { + // Verify consistency - same network should always generate same ID + result2 := generateAdvisoryLockID(tc.network) + assert.Equal(t, result, result2, "same network should generate same lock ID") + assert.Equal(t, tc.expected, result) + } + }) + } +} + +func Test_ingestService_splitGapsIntoBatches(t *testing.T) { + svc := &ingestService{} + + testCases := []struct { + name string + gaps []data.LedgerRange + batchSize uint32 + expected []BackfillBatch + }{ + { + name: "empty_gaps", + gaps: []data.LedgerRange{}, + batchSize: 100, + expected: nil, + }, + { + name: "single_gap_smaller_than_batch", + gaps: []data.LedgerRange{ + {GapStart: 100, GapEnd: 150}, + }, + batchSize: 200, + expected: []BackfillBatch{ + {StartLedger: 100, EndLedger: 150}, + }, + }, + { + name: "single_gap_larger_than_batch", + gaps: []data.LedgerRange{ + {GapStart: 100, GapEnd: 399}, + }, + batchSize: 100, + expected: []BackfillBatch{ + {StartLedger: 100, EndLedger: 199}, + {StartLedger: 200, EndLedger: 299}, + {StartLedger: 300, EndLedger: 399}, + }, + }, + { + name: "single_gap_exact_batch_size", + gaps: []data.LedgerRange{ + {GapStart: 100, GapEnd: 199}, + }, + batchSize: 100, + expected: []BackfillBatch{ + {StartLedger: 100, EndLedger: 199}, + }, + }, + { + name: "multiple_gaps", + gaps: []data.LedgerRange{ + {GapStart: 100, GapEnd: 149}, + {GapStart: 300, GapEnd: 349}, + }, + batchSize: 100, + expected: []BackfillBatch{ + {StartLedger: 100, EndLedger: 149}, + {StartLedger: 300, EndLedger: 349}, + }, + }, + { + name: "multiple_gaps_with_splits", + gaps: []data.LedgerRange{ + {GapStart: 100, GapEnd: 249}, + {GapStart: 500, GapEnd: 599}, + }, + batchSize: 100, + expected: []BackfillBatch{ + {StartLedger: 100, EndLedger: 199}, + {StartLedger: 200, EndLedger: 249}, + {StartLedger: 500, EndLedger: 599}, + }, + }, + { + name: "single_ledger_gap", + gaps: []data.LedgerRange{ + {GapStart: 100, GapEnd: 100}, + }, + batchSize: 100, + expected: []BackfillBatch{ + {StartLedger: 100, EndLedger: 100}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + svc.backfillBatchSize = tc.batchSize + result := svc.splitGapsIntoBatches(tc.gaps) + assert.Equal(t, tc.expected, result) + }) + } +} + +func Test_ingestService_calculateBackfillGaps(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + + testCases := []struct { + name string + startLedger uint32 + endLedger uint32 + setupDB func(t *testing.T) + expectedGaps []data.LedgerRange + }{ + { + name: "entirely_before_oldest_ingested_ledger", + startLedger: 50, + endLedger: 80, + setupDB: func(t *testing.T) { + // Set oldest to 100, latest to 200 + _, err := dbConnectionPool.ExecContext(ctx, `INSERT INTO ingest_store (key, value) VALUES ('oldest_ledger_cursor', 100)`) + require.NoError(t, err) + _, err = dbConnectionPool.ExecContext(ctx, `INSERT INTO ingest_store (key, value) VALUES ('latest_ledger_cursor', 200)`) + require.NoError(t, err) + }, + expectedGaps: []data.LedgerRange{ + {GapStart: 50, GapEnd: 80}, + }, + }, + { + name: "overlaps_with_oldest_ingested_ledger", + startLedger: 50, + endLedger: 150, + setupDB: func(t *testing.T) { + // Set oldest to 100, latest to 200 + _, err := dbConnectionPool.ExecContext(ctx, `INSERT INTO ingest_store (key, value) VALUES ('oldest_ledger_cursor', 100)`) + require.NoError(t, err) + _, err = dbConnectionPool.ExecContext(ctx, `INSERT INTO ingest_store (key, value) VALUES ('latest_ledger_cursor', 200)`) + require.NoError(t, err) + // Insert transactions for 100-200 (no gaps) + for ledger := uint32(100); ledger <= 200; ledger++ { + _, err := dbConnectionPool.ExecContext(ctx, + `INSERT INTO transactions (hash, to_id, envelope_xdr, result_xdr, meta_xdr, ledger_number, ledger_created_at) + VALUES ($1, $2, 'env', 'res', 'meta', $3, NOW())`, + "hash"+string(rune(ledger)), ledger, ledger) + require.NoError(t, err) + } + }, + expectedGaps: []data.LedgerRange{ + {GapStart: 50, GapEnd: 99}, + }, + }, + { + name: "entirely_within_ingested_range_no_gaps", + startLedger: 110, + endLedger: 150, + setupDB: func(t *testing.T) { + // Set oldest to 100, latest to 200 + _, err := dbConnectionPool.ExecContext(ctx, `INSERT INTO ingest_store (key, value) VALUES ('oldest_ledger_cursor', 100)`) + require.NoError(t, err) + _, err = dbConnectionPool.ExecContext(ctx, `INSERT INTO ingest_store (key, value) VALUES ('latest_ledger_cursor', 200)`) + require.NoError(t, err) + // Insert transactions for 100-200 (no gaps) + for ledger := uint32(100); ledger <= 200; ledger++ { + _, err := dbConnectionPool.ExecContext(ctx, + `INSERT INTO transactions (hash, to_id, envelope_xdr, result_xdr, meta_xdr, ledger_number, ledger_created_at) + VALUES ($1, $2, 'env', 'res', 'meta', $3, NOW())`, + "hash"+string(rune(ledger)), ledger, ledger) + require.NoError(t, err) + } + }, + expectedGaps: []data.LedgerRange{}, + }, + { + name: "entirely_within_ingested_range_with_gaps", + startLedger: 110, + endLedger: 180, + setupDB: func(t *testing.T) { + // Set oldest to 100, latest to 200 + _, err := dbConnectionPool.ExecContext(ctx, `INSERT INTO ingest_store (key, value) VALUES ('oldest_ledger_cursor', 100)`) + require.NoError(t, err) + _, err = dbConnectionPool.ExecContext(ctx, `INSERT INTO ingest_store (key, value) VALUES ('latest_ledger_cursor', 200)`) + require.NoError(t, err) + // Insert transactions with gaps: 100-120, 150-200 (gap at 121-149) + for ledger := uint32(100); ledger <= 120; ledger++ { + _, err := dbConnectionPool.ExecContext(ctx, + `INSERT INTO transactions (hash, to_id, envelope_xdr, result_xdr, meta_xdr, ledger_number, ledger_created_at) + VALUES ($1, $2, 'env', 'res', 'meta', $3, NOW())`, + "hash"+string(rune(ledger)), ledger, ledger) + require.NoError(t, err) + } + for ledger := uint32(150); ledger <= 200; ledger++ { + _, err := dbConnectionPool.ExecContext(ctx, + `INSERT INTO transactions (hash, to_id, envelope_xdr, result_xdr, meta_xdr, ledger_number, ledger_created_at) + VALUES ($1, $2, 'env', 'res', 'meta', $3, NOW())`, + "hash"+string(rune(ledger)), ledger, ledger) + require.NoError(t, err) + } + }, + expectedGaps: []data.LedgerRange{ + {GapStart: 121, GapEnd: 149}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Clean up + _, err := dbConnectionPool.ExecContext(ctx, "DELETE FROM transactions") + require.NoError(t, err) + _, err = dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store") + require.NoError(t, err) + + mockMetricsService := metrics.NewMockMetricsService() + mockMetricsService.On("RegisterPoolMetrics", "ledger_indexer", mock.Anything).Return() + mockMetricsService.On("RegisterPoolMetrics", "backfill", mock.Anything).Return() + mockMetricsService.On("ObserveDBQueryDuration", mock.Anything, mock.Anything, mock.Anything).Return().Maybe() + mockMetricsService.On("IncDBQuery", mock.Anything, mock.Anything).Return().Maybe() + defer mockMetricsService.AssertExpectations(t) + + models, err := data.NewModels(dbConnectionPool, mockMetricsService) + require.NoError(t, err) + + if tc.setupDB != nil { + tc.setupDB(t) + } + + mockAppTracker := apptracker.MockAppTracker{} + mockRPCService := RPCServiceMock{} + mockRPCService.On("NetworkPassphrase").Return(network.TestNetworkPassphrase).Maybe() + mockChAccStore := &store.ChannelAccountStoreMock{} + mockLedgerBackend := &LedgerBackendMock{} + mockArchive := &HistoryArchiveMock{} + + svc, err := NewIngestService(IngestServiceConfig{ + IngestionMode: IngestionModeBackfill, + Models: models, + LatestLedgerCursorName: "latest_ledger_cursor", + OldestLedgerCursorName: "oldest_ledger_cursor", + AppTracker: &mockAppTracker, + RPCService: &mockRPCService, + LedgerBackend: mockLedgerBackend, + ChannelAccountStore: mockChAccStore, + MetricsService: mockMetricsService, + GetLedgersLimit: defaultGetLedgersLimit, + Network: network.TestNetworkPassphrase, + NetworkPassphrase: network.TestNetworkPassphrase, + Archive: mockArchive, + SkipTxMeta: false, + SkipTxEnvelope: false, + }) + require.NoError(t, err) + + gaps, err := svc.calculateBackfillGaps(ctx, tc.startLedger, tc.endLedger) + require.NoError(t, err) + assert.Equal(t, tc.expectedGaps, gaps) + }) + } +} + +func Test_BackfillMode_Validation(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + + testCases := []struct { + name string + mode BackfillMode + startLedger uint32 + endLedger uint32 + latestIngested uint32 + expectValidationError bool + errorContains string + }{ + { + name: "historical_mode_valid_range", + mode: BackfillModeHistorical, + startLedger: 50, + endLedger: 80, + latestIngested: 100, + expectValidationError: false, + }, + { + name: "historical_mode_end_exceeds_latest", + mode: BackfillModeHistorical, + startLedger: 50, + endLedger: 150, + latestIngested: 100, + expectValidationError: true, + errorContains: "end ledger 150 cannot be greater than latest ingested ledger 100", + }, + { + name: "catchup_mode_valid_range", + mode: BackfillModeCatchup, + startLedger: 101, + endLedger: 150, + latestIngested: 100, + expectValidationError: false, + }, + { + name: "catchup_mode_start_not_next_ledger", + mode: BackfillModeCatchup, + startLedger: 105, + endLedger: 150, + latestIngested: 100, + expectValidationError: true, + errorContains: "catchup must start from ledger 101", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Clean up + _, err := dbConnectionPool.ExecContext(ctx, "DELETE FROM transactions") + require.NoError(t, err) + _, err = dbConnectionPool.ExecContext(ctx, "DELETE FROM ingest_store") + require.NoError(t, err) + + // Set up latest ingested ledger cursor + _, err = dbConnectionPool.ExecContext(ctx, + `INSERT INTO ingest_store (key, value) VALUES ('latest_ledger_cursor', $1)`, + tc.latestIngested) + require.NoError(t, err) + _, err = dbConnectionPool.ExecContext(ctx, + `INSERT INTO ingest_store (key, value) VALUES ('oldest_ledger_cursor', $1)`, + tc.latestIngested) + require.NoError(t, err) + + mockMetricsService := metrics.NewMockMetricsService() + mockMetricsService.On("RegisterPoolMetrics", "ledger_indexer", mock.Anything).Return() + mockMetricsService.On("RegisterPoolMetrics", "backfill", mock.Anything).Return() + mockMetricsService.On("SetBackfillElapsed", mock.AnythingOfType("string"), mock.AnythingOfType("float64")).Return().Maybe() + mockMetricsService.On("IncBackfillBatchesFailed", mock.AnythingOfType("string")).Return().Maybe() + mockMetricsService.On("ObserveDBQueryDuration", mock.Anything, mock.Anything, mock.Anything).Return().Maybe() + mockMetricsService.On("IncDBQuery", mock.Anything, mock.Anything).Return().Maybe() + defer mockMetricsService.AssertExpectations(t) + + models, err := data.NewModels(dbConnectionPool, mockMetricsService) + require.NoError(t, err) + + mockAppTracker := apptracker.MockAppTracker{} + mockRPCService := RPCServiceMock{} + mockRPCService.On("NetworkPassphrase").Return(network.TestNetworkPassphrase).Maybe() + mockChAccStore := &store.ChannelAccountStoreMock{} + mockLedgerBackend := &LedgerBackendMock{} + mockArchive := &HistoryArchiveMock{} + + // Create a mock ledger backend factory that returns an error immediately + // This allows validation to pass but stops batch processing early + mockBackendFactory := func(ctx context.Context) (ledgerbackend.LedgerBackend, error) { + return nil, fmt.Errorf("mock backend factory error") + } + + svc, err := NewIngestService(IngestServiceConfig{ + IngestionMode: IngestionModeBackfill, + Models: models, + LatestLedgerCursorName: "latest_ledger_cursor", + OldestLedgerCursorName: "oldest_ledger_cursor", + AppTracker: &mockAppTracker, + RPCService: &mockRPCService, + LedgerBackend: mockLedgerBackend, + LedgerBackendFactory: mockBackendFactory, + ChannelAccountStore: mockChAccStore, + MetricsService: mockMetricsService, + GetLedgersLimit: defaultGetLedgersLimit, + Network: network.TestNetworkPassphrase, + NetworkPassphrase: network.TestNetworkPassphrase, + Archive: mockArchive, + SkipTxMeta: false, + SkipTxEnvelope: false, + BackfillBatchSize: 100, + }) + require.NoError(t, err) + + err = svc.startBackfilling(ctx, tc.startLedger, tc.endLedger, tc.mode) + if tc.expectValidationError { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errorContains) + } else { + // For valid cases, validation passes but batch processing fails + // The error will be wrapped as "backfilling failed: X/Y batches failed" + require.Error(t, err) + assert.Contains(t, err.Error(), "backfilling failed") + // Ensure it's NOT a validation error + assert.NotContains(t, err.Error(), "cannot be greater than latest ingested ledger") + assert.NotContains(t, err.Error(), "catchup must start from ledger") + } + }) + } +} diff --git a/internal/signing/store/channel_accounts_model.go b/internal/signing/store/channel_accounts_model.go index 842ca86fe..93192ec2b 100644 --- a/internal/signing/store/channel_accounts_model.go +++ b/internal/signing/store/channel_accounts_model.go @@ -7,6 +7,7 @@ import ( "fmt" "time" + "github.com/jackc/pgx/v5" "github.com/lib/pq" "github.com/stellar/wallet-backend/internal/db" @@ -93,11 +94,7 @@ func (ca *ChannelAccountModel) AssignTxToChannelAccount(ctx context.Context, pub return nil } -func (ca *ChannelAccountModel) UnassignTxAndUnlockChannelAccounts(ctx context.Context, sqlExec db.SQLExecuter, txHashes ...string) (int64, error) { - if sqlExec == nil { - sqlExec = ca.DB - } - +func (ca *ChannelAccountModel) UnassignTxAndUnlockChannelAccounts(ctx context.Context, pgxTx pgx.Tx, txHashes ...string) (int64, error) { if len(txHashes) == 0 { return 0, errors.New("txHashes cannot be empty") } @@ -111,16 +108,12 @@ func (ca *ChannelAccountModel) UnassignTxAndUnlockChannelAccounts(ctx context.Co WHERE locked_tx_hash = ANY($1) ` - res, err := sqlExec.ExecContext(ctx, query, pq.Array(txHashes)) + result, err := pgxTx.Exec(ctx, query, txHashes) if err != nil { return 0, fmt.Errorf("unlocking channel accounts %v: %w", txHashes, err) } - rowsAffected, err := res.RowsAffected() - if err != nil { - return 0, fmt.Errorf("getting rows affected: %w", err) - } - return rowsAffected, nil + return result.RowsAffected(), nil } func (ca *ChannelAccountModel) BatchInsert(ctx context.Context, sqlExec db.SQLExecuter, channelAccounts []*ChannelAccount) error { diff --git a/internal/signing/store/channel_accounts_model_test.go b/internal/signing/store/channel_accounts_model_test.go index 05813cc56..7786b73d0 100644 --- a/internal/signing/store/channel_accounts_model_test.go +++ b/internal/signing/store/channel_accounts_model_test.go @@ -182,7 +182,6 @@ func Test_ChannelAccountModel_UnassignTxAndUnlockChannelAccounts(t *testing.T) { testCases := []struct { name string - useDBTx bool numberOfFixtures int txHashes func(fixtures []*keypair.Full) []string expectedErrContains string @@ -209,14 +208,6 @@ func Test_ChannelAccountModel_UnassignTxAndUnlockChannelAccounts(t *testing.T) { return []string{"txhash_" + fixtures[0].Address()} }, }, - { - name: "🟢single_tx_hash_found_with_db_tx", - useDBTx: true, - numberOfFixtures: 1, - txHashes: func(fixtures []*keypair.Full) []string { - return []string{"txhash_" + fixtures[0].Address()} - }, - }, { name: "🟢multiple_tx_hashes_all_found", numberOfFixtures: 2, @@ -240,20 +231,12 @@ func Test_ChannelAccountModel_UnassignTxAndUnlockChannelAccounts(t *testing.T) { require.NoError(t, err) }() - var sqlExec db.SQLExecuter = dbConnectionPool - if tc.useDBTx { - dbTx, err := dbConnectionPool.BeginTxx(ctx, nil) - require.NoError(t, err) - defer dbTx.Rollback() - sqlExec = dbTx - } - // Create fixtures for this test case fixtures := make([]*keypair.Full, tc.numberOfFixtures) now := time.Now() for i := range fixtures { channelAccount := keypair.MustRandom() - createChannelAccountFixture(t, ctx, sqlExec, ChannelAccount{ + createChannelAccountFixture(t, ctx, dbConnectionPool, ChannelAccount{ PublicKey: channelAccount.Address(), EncryptedPrivateKey: channelAccount.Seed(), LockedTxHash: utils.SQLNullString("txhash_" + channelAccount.Address()), @@ -265,22 +248,29 @@ func Test_ChannelAccountModel_UnassignTxAndUnlockChannelAccounts(t *testing.T) { // 🔒 Channel accounts start locked for _, fixture := range fixtures { - chAccFromDB, err := m.Get(ctx, sqlExec, fixture.Address()) + chAccFromDB, err := m.Get(ctx, dbConnectionPool, fixture.Address()) require.NoError(t, err) require.True(t, chAccFromDB.LockedTxHash.Valid) require.True(t, chAccFromDB.LockedAt.Valid) require.True(t, chAccFromDB.LockedUntil.Valid) } - rowsAffected, err := m.UnassignTxAndUnlockChannelAccounts(ctx, sqlExec, tc.txHashes(fixtures)...) + // Start pgx transaction for the unlock call + pgxTx, err := dbConnectionPool.PgxPool().Begin(ctx) + require.NoError(t, err) + defer pgxTx.Rollback(ctx) //nolint:errcheck + + rowsAffected, err := m.UnassignTxAndUnlockChannelAccounts(ctx, pgxTx, tc.txHashes(fixtures)...) if tc.expectedErrContains != "" { require.ErrorContains(t, err, tc.expectedErrContains) } else { require.NoError(t, err) require.Equal(t, int64(tc.numberOfFixtures), rowsAffected) + // Commit the pgx transaction to persist the changes + require.NoError(t, pgxTx.Commit(ctx)) // 🔓 Channel accounts get unlocked for _, fixture := range fixtures { - chAccFromDB, err := m.Get(ctx, sqlExec, fixture.Address()) + chAccFromDB, err := m.Get(ctx, dbConnectionPool, fixture.Address()) require.NoError(t, err) require.False(t, chAccFromDB.LockedTxHash.Valid) require.False(t, chAccFromDB.LockedAt.Valid) diff --git a/internal/signing/store/mocks.go b/internal/signing/store/mocks.go index fac524d96..43e05192e 100644 --- a/internal/signing/store/mocks.go +++ b/internal/signing/store/mocks.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/jackc/pgx/v5" "github.com/stretchr/testify/mock" "github.com/stellar/wallet-backend/internal/db" @@ -44,8 +45,8 @@ func (s *ChannelAccountStoreMock) AssignTxToChannelAccount(ctx context.Context, return args.Error(0) } -func (s *ChannelAccountStoreMock) UnassignTxAndUnlockChannelAccounts(ctx context.Context, sqlExec db.SQLExecuter, txHashes ...string) (int64, error) { - _ca := []any{ctx, sqlExec} +func (s *ChannelAccountStoreMock) UnassignTxAndUnlockChannelAccounts(ctx context.Context, pgxTx pgx.Tx, txHashes ...string) (int64, error) { + _ca := []any{ctx, pgxTx} for _, txHash := range txHashes { _ca = append(_ca, txHash) } diff --git a/internal/signing/store/types.go b/internal/signing/store/types.go index 8b8fee2ba..7d1f3ce05 100644 --- a/internal/signing/store/types.go +++ b/internal/signing/store/types.go @@ -5,6 +5,8 @@ import ( "database/sql" "time" + "github.com/jackc/pgx/v5" + "github.com/stellar/wallet-backend/internal/db" ) @@ -24,7 +26,7 @@ type ChannelAccountStore interface { GetAll(ctx context.Context, sqlExec db.SQLExecuter, limit int) ([]*ChannelAccount, error) GetAllByPublicKey(ctx context.Context, sqlExec db.SQLExecuter, publicKeys ...string) ([]*ChannelAccount, error) AssignTxToChannelAccount(ctx context.Context, publicKey string, txHash string) error - UnassignTxAndUnlockChannelAccounts(ctx context.Context, sqlExec db.SQLExecuter, txHashes ...string) (int64, error) + UnassignTxAndUnlockChannelAccounts(ctx context.Context, pgxTx pgx.Tx, txHashes ...string) (int64, error) BatchInsert(ctx context.Context, sqlExec db.SQLExecuter, channelAccounts []*ChannelAccount) error Delete(ctx context.Context, sqlExec db.SQLExecuter, publicKeys ...string) (int64, error) Count(ctx context.Context) (int64, error) diff --git a/pkg/wbclient/types/types.go b/pkg/wbclient/types/types.go index 6e77a0a0d..bdad938f2 100644 --- a/pkg/wbclient/types/types.go +++ b/pkg/wbclient/types/types.go @@ -221,9 +221,9 @@ type Transaction struct { // GraphQLTransaction represents a transaction from the GraphQL API type GraphQLTransaction struct { Hash string `json:"hash"` - EnvelopeXdr string `json:"envelopeXdr"` + EnvelopeXdr *string `json:"envelopeXdr"` ResultXdr string `json:"resultXdr"` - MetaXdr string `json:"metaXdr"` + MetaXdr *string `json:"metaXdr,omitempty"` LedgerNumber uint32 `json:"ledgerNumber"` LedgerCreatedAt time.Time `json:"ledgerCreatedAt"` IngestedAt time.Time `json:"ingestedAt"`