From 37c03e84cff2e41d490a3024a653ea9821fc0037 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 2 Feb 2026 14:16:24 -0500 Subject: [PATCH 01/77] add timescale db hypertable --- docker-compose.yaml | 3 ++- .../migrations/2025-06-10.2-transactions.sql | 27 +++++++++++++------ .../db/migrations/2025-06-10.3-operations.sql | 22 ++++++++++----- .../migrations/2025-06-10.4-statechanges.sql | 26 +++++++++++++----- 4 files changed, 56 insertions(+), 22 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 0a251049a..c84af70bd 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,7 +7,7 @@ services: db: container_name: db - image: postgres:14-alpine + image: timescale/timescaledb:latest-pg14 healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres -d wallet-backend"] interval: 10s @@ -268,5 +268,6 @@ volumes: configs: postgres_init: content: | + CREATE EXTENSION IF NOT EXISTS timescaledb; CREATE SCHEMA IF NOT EXISTS wallet_backend_mainnet; CREATE SCHEMA IF NOT EXISTS wallet_backend_testnet; diff --git a/internal/db/migrations/2025-06-10.2-transactions.sql b/internal/db/migrations/2025-06-10.2-transactions.sql index 16cac0572..b8b7558e7 100644 --- a/internal/db/migrations/2025-06-10.2-transactions.sql +++ b/internal/db/migrations/2025-06-10.2-transactions.sql @@ -1,24 +1,35 @@ -- +migrate Up --- Table: transactions +-- Table: transactions (TimescaleDB hypertable with columnstore) CREATE TABLE transactions ( - to_id BIGINT PRIMARY KEY, - hash TEXT NOT NULL UNIQUE, + ledger_created_at TIMESTAMPTZ NOT NULL, + to_id BIGINT NOT NULL, + hash TEXT NOT NULL, envelope_xdr TEXT, fee_charged BIGINT NOT NULL, result_code TEXT NOT NULL, meta_xdr TEXT, ledger_number INTEGER NOT NULL, - ledger_created_at TIMESTAMPTZ NOT NULL, is_fee_bump BOOLEAN NOT NULL DEFAULT false, - ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (ledger_created_at, to_id), + UNIQUE (ledger_created_at, hash) +) WITH ( + tsdb.hypertable, + tsdb.partition_column = 'ledger_created_at', + tsdb.chunk_interval = '1 day', + tsdb.orderby = 'ledger_created_at DESC' ); -CREATE INDEX idx_transactions_ledger_created_at ON transactions(ledger_created_at); +-- Index for hash lookups (GetByHash queries) +CREATE INDEX idx_transactions_hash ON transactions(hash); + +-- Index for to_id lookups (TOID-based queries) +CREATE INDEX idx_transactions_to_id ON transactions(to_id); --- Table: transactions_accounts +-- Table: transactions_accounts (no FK - hypertable compound PK) CREATE TABLE transactions_accounts ( - tx_to_id BIGINT NOT NULL REFERENCES transactions(to_id) ON DELETE CASCADE, + tx_to_id BIGINT NOT NULL, account_id TEXT NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), PRIMARY KEY (account_id, tx_to_id) diff --git a/internal/db/migrations/2025-06-10.3-operations.sql b/internal/db/migrations/2025-06-10.3-operations.sql index 22ad0a854..1e6273921 100644 --- a/internal/db/migrations/2025-06-10.3-operations.sql +++ b/internal/db/migrations/2025-06-10.3-operations.sql @@ -1,8 +1,9 @@ -- +migrate Up --- Table: operations +-- Table: operations (TimescaleDB hypertable with columnstore) CREATE TABLE operations ( - id BIGINT PRIMARY KEY, + ledger_created_at TIMESTAMPTZ NOT NULL, + id BIGINT NOT NULL, operation_type TEXT NOT NULL CHECK ( operation_type IN ( 'CREATE_ACCOUNT', 'PAYMENT', 'PATH_PAYMENT_STRICT_RECEIVE', @@ -21,15 +22,24 @@ CREATE TABLE operations ( result_code TEXT NOT NULL, successful BOOLEAN NOT NULL, ledger_number INTEGER NOT NULL, - ledger_created_at TIMESTAMPTZ NOT NULL, - ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (ledger_created_at, id) +) WITH ( + tsdb.hypertable, + tsdb.partition_column = 'ledger_created_at', + tsdb.chunk_interval = '1 day', + tsdb.segmentby = 'operation_type', + tsdb.orderby = 'ledger_created_at DESC' ); CREATE INDEX idx_operations_ledger_created_at ON operations(ledger_created_at); --- Table: operations_accounts +-- Index for id lookups (TOID-based queries) +CREATE INDEX idx_operations_id ON operations(id); + +-- Table: operations_accounts (no FK - hypertable compound PK) CREATE TABLE operations_accounts ( - operation_id BIGINT NOT NULL REFERENCES operations(id) ON DELETE CASCADE, + operation_id BIGINT NOT NULL, account_id TEXT NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), PRIMARY KEY (account_id, operation_id) diff --git a/internal/db/migrations/2025-06-10.4-statechanges.sql b/internal/db/migrations/2025-06-10.4-statechanges.sql index 6dfcfacc0..75df06376 100644 --- a/internal/db/migrations/2025-06-10.4-statechanges.sql +++ b/internal/db/migrations/2025-06-10.4-statechanges.sql @@ -1,8 +1,11 @@ -- +migrate Up --- Table: state_changes +-- Table: state_changes (TimescaleDB hypertable with columnstore) +-- Note: FK to transactions removed (hypertable FKs not supported) CREATE TABLE state_changes ( - to_id BIGINT NOT NULL REFERENCES transactions(to_id) ON DELETE CASCADE, + ledger_created_at TIMESTAMPTZ NOT NULL, + to_id BIGINT NOT NULL, + operation_id BIGINT NOT NULL, state_change_order BIGINT NOT NULL CHECK (state_change_order >= 1), state_change_category TEXT NOT NULL CHECK ( state_change_category IN ( @@ -19,10 +22,8 @@ CREATE TABLE state_changes ( ) ), ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - ledger_created_at TIMESTAMPTZ NOT NULL, ledger_number INTEGER NOT NULL, account_id TEXT NOT NULL, - operation_id BIGINT NOT NULL, token_id TEXT, amount TEXT, signer_account_id TEXT, @@ -42,14 +43,25 @@ CREATE TABLE state_changes ( trustline_limit_new TEXT, flags SMALLINT, key_value JSONB, - - PRIMARY KEY (to_id, operation_id, state_change_order) + PRIMARY KEY (ledger_created_at, to_id, operation_id, state_change_order) +) WITH ( + tsdb.hypertable, + tsdb.partition_column = 'ledger_created_at', + tsdb.chunk_interval = '1 day', + tsdb.segmentby = 'state_change_category', + tsdb.orderby = 'ledger_created_at DESC' ); +-- Index for account_id lookups (most common query pattern) CREATE INDEX idx_state_changes_account_id ON state_changes(account_id); -CREATE INDEX idx_state_changes_operation_id ON state_changes(operation_id); CREATE INDEX idx_state_changes_ledger_created_at ON state_changes(ledger_created_at); +-- Index for to_id lookups (transaction-based queries) +CREATE INDEX idx_state_changes_to_id ON state_changes(to_id); + +-- Index for operation_id lookups +CREATE INDEX idx_state_changes_operation_id ON state_changes(operation_id) WHERE operation_id > 0; + -- +migrate Down -- Tables From a963d2396931281ff0c1e20c47aa6bbe6f96765b Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 2 Feb 2026 14:20:25 -0500 Subject: [PATCH 02/77] update docker and test files to use timescaledb --- docker-compose.yaml | 2 +- internal/db/dbtest/dbtest.go | 31 +++++++++++++++++-- .../infrastructure/containers.go | 4 +-- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index c84af70bd..5a28d0f09 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,7 +7,7 @@ services: db: container_name: db - image: timescale/timescaledb:latest-pg14 + image: timescale/timescaledb:latest-pg17 healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres -d wallet-backend"] interval: 10s diff --git a/internal/db/dbtest/dbtest.go b/internal/db/dbtest/dbtest.go index 9cbc9917e..2ddfe6eb7 100644 --- a/internal/db/dbtest/dbtest.go +++ b/internal/db/dbtest/dbtest.go @@ -1,3 +1,4 @@ +// Package dbtest provides test database utilities with TimescaleDB support. package dbtest import ( @@ -11,14 +12,22 @@ import ( "github.com/stellar/wallet-backend/internal/db/migrations" ) +// Open opens a test database with migrations applied. +// Requires TimescaleDB extension to be available on the PostgreSQL server. func Open(t *testing.T) *dbtest.DB { db := dbtest.Postgres(t) conn := db.Open() defer conn.Close() + // Enable TimescaleDB extension before running migrations + _, err := conn.Exec("CREATE EXTENSION IF NOT EXISTS timescaledb") + if err != nil { + t.Fatal(err) + } + migrateDirection := schema.MigrateUp m := migrate.HttpFileSystemMigrationSource{FileSystem: http.FS(migrations.FS)} - _, err := schema.Migrate(conn.DB, m, migrateDirection, 0) + _, err = schema.Migrate(conn.DB, m, migrateDirection, 0) if err != nil { t.Fatal(err) } @@ -26,20 +35,38 @@ func Open(t *testing.T) *dbtest.DB { return db } +// OpenWithoutMigrations opens a test database without running migrations +// but still enables TimescaleDB extension for manual migration testing. func OpenWithoutMigrations(t *testing.T) *dbtest.DB { db := dbtest.Postgres(t) + conn := db.Open() + defer conn.Close() + + // Enable TimescaleDB extension + _, err := conn.Exec("CREATE EXTENSION IF NOT EXISTS timescaledb") + if err != nil { + t.Fatal(err) + } + return db } // OpenB opens a test database for benchmarks with migrations applied. +// Requires TimescaleDB extension to be available on the PostgreSQL server. func OpenB(b *testing.B) *dbtest.DB { db := dbtest.Postgres(b) conn := db.Open() defer conn.Close() + // Enable TimescaleDB extension before running migrations + _, err := conn.Exec("CREATE EXTENSION IF NOT EXISTS timescaledb") + if err != nil { + b.Fatal(err) + } + migrateDirection := schema.MigrateUp m := migrate.HttpFileSystemMigrationSource{FileSystem: http.FS(migrations.FS)} - _, err := schema.Migrate(conn.DB, m, migrateDirection, 0) + _, err = schema.Migrate(conn.DB, m, migrateDirection, 0) if err != nil { b.Fatal(err) } diff --git a/internal/integrationtests/infrastructure/containers.go b/internal/integrationtests/infrastructure/containers.go index 6c295cf1c..80cbfcaa2 100644 --- a/internal/integrationtests/infrastructure/containers.go +++ b/internal/integrationtests/infrastructure/containers.go @@ -317,11 +317,11 @@ func createRPCContainer(ctx context.Context, testNetwork *testcontainers.DockerN }, nil } -// createWalletDBContainer starts a PostgreSQL container for wallet-backend +// createWalletDBContainer starts a TimescaleDB container for wallet-backend func createWalletDBContainer(ctx context.Context, testNetwork *testcontainers.DockerNetwork) (*TestContainer, error) { containerRequest := testcontainers.ContainerRequest{ Name: walletBackendDBContainerName, - Image: "postgres:14-alpine", + Image: "timescale/timescaledb:latest-pg17", Labels: map[string]string{ "org.testcontainers.session-id": "wallet-backend-integration-tests", }, From ce9d7d0271d9854f638d4a0e0c553d8c4fcf8d10 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 2 Feb 2026 14:35:55 -0500 Subject: [PATCH 03/77] Fix tests --- internal/data/operations.go | 2 +- internal/data/operations_test.go | 3 ++- internal/data/statechanges.go | 2 +- internal/data/statechanges_test.go | 3 ++- internal/data/transactions.go | 2 +- internal/data/transactions_test.go | 3 ++- 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/internal/data/operations.go b/internal/data/operations.go index fbdcb9fae..83742a4cc 100644 --- a/internal/data/operations.go +++ b/internal/data/operations.go @@ -329,7 +329,7 @@ func (m *OperationModel) BatchInsert( UNNEST($6::bigint[]) AS ledger_number, UNNEST($7::timestamptz[]) AS ledger_created_at ) o - ON CONFLICT (id) DO NOTHING + ON CONFLICT (ledger_created_at, id) DO NOTHING RETURNING id ), diff --git a/internal/data/operations_test.go b/internal/data/operations_test.go index cbf4dde82..a78387951 100644 --- a/internal/data/operations_test.go +++ b/internal/data/operations_test.go @@ -476,7 +476,8 @@ func Test_OperationModel_BatchCopy_DuplicateFails(t *testing.T) { // BatchCopy should fail with a unique constraint violation require.Error(t, err) - assert.Contains(t, err.Error(), "pgx CopyFrom operations: ERROR: duplicate key value violates unique constraint \"operations_pkey\"") + // TimescaleDB uses chunk-based constraint names like "2_3_operations_pkey" instead of "operations_pkey" + assert.Contains(t, err.Error(), "pgx CopyFrom operations: ERROR: duplicate key value violates unique constraint") // Rollback the failed transaction require.NoError(t, pgxTx.Rollback(ctx)) diff --git a/internal/data/statechanges.go b/internal/data/statechanges.go index 442d55447..a181dd8db 100644 --- a/internal/data/statechanges.go +++ b/internal/data/statechanges.go @@ -327,7 +327,7 @@ func (m *StateChangeModel) BatchInsert( sc.signer_weight_old, sc.signer_weight_new, sc.threshold_old, sc.threshold_new, sc.trustline_limit_old, sc.trustline_limit_new, sc.flags, sc.key_value FROM input_data sc - ON CONFLICT (to_id, operation_id, state_change_order) DO NOTHING + ON CONFLICT (ledger_created_at, to_id, operation_id, state_change_order) DO NOTHING RETURNING to_id, operation_id, state_change_order ) SELECT CONCAT(to_id, '-', operation_id, '-', state_change_order) FROM inserted_state_changes; diff --git a/internal/data/statechanges_test.go b/internal/data/statechanges_test.go index c4c99b06f..e36649687 100644 --- a/internal/data/statechanges_test.go +++ b/internal/data/statechanges_test.go @@ -460,7 +460,8 @@ func TestStateChangeModel_BatchCopy_DuplicateFails(t *testing.T) { // BatchCopy should fail with a unique constraint violation require.Error(t, err) - assert.Contains(t, err.Error(), "pgx CopyFrom state_changes: ERROR: duplicate key value violates unique constraint \"state_changes_pkey\"") + // TimescaleDB uses chunk-based constraint names like "2_3_state_changes_pkey" instead of "state_changes_pkey" + assert.Contains(t, err.Error(), "pgx CopyFrom state_changes: ERROR: duplicate key value violates unique constraint") // Rollback the failed transaction require.NoError(t, pgxTx.Rollback(ctx)) diff --git a/internal/data/transactions.go b/internal/data/transactions.go index e33b07cb1..5728ad6eb 100644 --- a/internal/data/transactions.go +++ b/internal/data/transactions.go @@ -243,7 +243,7 @@ func (m *TransactionModel) BatchInsert( UNNEST($8::timestamptz[]) AS ledger_created_at, UNNEST($9::boolean[]) AS is_fee_bump ) t - ON CONFLICT (to_id) DO NOTHING + ON CONFLICT (ledger_created_at, to_id) DO NOTHING RETURNING hash ), diff --git a/internal/data/transactions_test.go b/internal/data/transactions_test.go index 8a978ac9b..f04ffc38b 100644 --- a/internal/data/transactions_test.go +++ b/internal/data/transactions_test.go @@ -455,7 +455,8 @@ func Test_TransactionModel_BatchCopy_DuplicateFails(t *testing.T) { // BatchCopy should fail with a unique constraint violation require.Error(t, err) - assert.Contains(t, err.Error(), "pgx CopyFrom transactions: ERROR: duplicate key value violates unique constraint \"transactions_pkey\"") + // TimescaleDB uses chunk-based constraint names like "1_1_transactions_pkey" instead of "transactions_pkey" + assert.Contains(t, err.Error(), "pgx CopyFrom transactions: ERROR: duplicate key value violates unique constraint") // Rollback the failed transaction require.NoError(t, pgxTx.Rollback(ctx)) From 8badbeff641debeecab3185447e655fd4608cc49 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 2 Feb 2026 14:55:47 -0500 Subject: [PATCH 04/77] convert junction tables to hypertables --- internal/data/operations.go | 37 +++++++++++++++---- internal/data/transactions.go | 37 +++++++++++++++---- .../migrations/2025-06-10.2-transactions.sql | 12 ++++-- .../db/migrations/2025-06-10.3-operations.sql | 12 ++++-- 4 files changed, 78 insertions(+), 20 deletions(-) diff --git a/internal/data/operations.go b/internal/data/operations.go index 83742a4cc..eeec16b26 100644 --- a/internal/data/operations.go +++ b/internal/data/operations.go @@ -300,13 +300,22 @@ func (m *OperationModel) BatchInsert( ledgerCreatedAts[i] = op.LedgerCreatedAt } - // 2. Flatten the stellarAddressesByOpID into parallel slices + // 2. Build OpID -> LedgerCreatedAt lookup from operations + ledgerCreatedAtByOpID := make(map[int64]time.Time, len(operations)) + for _, op := range operations { + ledgerCreatedAtByOpID[op.ID] = op.LedgerCreatedAt + } + + // 3. Flatten the stellarAddressesByOpID into parallel slices var opIDs []int64 var stellarAddresses []string + var oaLedgerCreatedAts []time.Time for opID, addresses := range stellarAddressesByOpID { + ledgerCreatedAt := ledgerCreatedAtByOpID[opID] for address := range addresses.Iter() { opIDs = append(opIDs, opID) stellarAddresses = append(stellarAddresses, address) + oaLedgerCreatedAts = append(oaLedgerCreatedAts, ledgerCreatedAt) } } @@ -336,13 +345,14 @@ func (m *OperationModel) BatchInsert( -- Insert operations_accounts links inserted_operations_accounts AS ( INSERT INTO operations_accounts - (operation_id, account_id) + (ledger_created_at, operation_id, account_id) SELECT - oa.op_id, oa.account_id + oa.ledger_created_at, oa.op_id, oa.account_id FROM ( SELECT - UNNEST($8::bigint[]) AS op_id, - UNNEST($9::text[]) AS account_id + UNNEST($8::timestamptz[]) AS ledger_created_at, + UNNEST($9::bigint[]) AS op_id, + UNNEST($10::text[]) AS account_id ) oa ON CONFLICT DO NOTHING ) @@ -361,6 +371,7 @@ func (m *OperationModel) BatchInsert( pq.Array(successfulFlags), pq.Array(ledgerNumbers), pq.Array(ledgerCreatedAts), + pq.Array(oaLedgerCreatedAts), pq.Array(opIDs), pq.Array(stellarAddresses), ) @@ -432,18 +443,30 @@ func (m *OperationModel) BatchCopy( // COPY operations_accounts using pgx binary format with native pgtype types if len(stellarAddressesByOpID) > 0 { + // Build OpID -> LedgerCreatedAt lookup from operations + ledgerCreatedAtByOpID := make(map[int64]time.Time, len(operations)) + for _, op := range operations { + ledgerCreatedAtByOpID[op.ID] = op.LedgerCreatedAt + } + var oaRows [][]any for opID, addresses := range stellarAddressesByOpID { + ledgerCreatedAt := ledgerCreatedAtByOpID[opID] + ledgerCreatedAtPgtype := pgtype.Timestamptz{Time: ledgerCreatedAt, Valid: true} opIDPgtype := pgtype.Int8{Int64: opID, Valid: true} for _, addr := range addresses.ToSlice() { - oaRows = append(oaRows, []any{opIDPgtype, pgtype.Text{String: addr, Valid: true}}) + oaRows = append(oaRows, []any{ + ledgerCreatedAtPgtype, + opIDPgtype, + pgtype.Text{String: addr, Valid: true}, + }) } } _, err = pgxTx.CopyFrom( ctx, pgx.Identifier{"operations_accounts"}, - []string{"operation_id", "account_id"}, + []string{"ledger_created_at", "operation_id", "account_id"}, pgx.CopyFromRows(oaRows), ) if err != nil { diff --git a/internal/data/transactions.go b/internal/data/transactions.go index 5728ad6eb..51a6a88d9 100644 --- a/internal/data/transactions.go +++ b/internal/data/transactions.go @@ -212,13 +212,22 @@ func (m *TransactionModel) BatchInsert( isFeeBumps[i] = t.IsFeeBump } - // 2. Flatten the stellarAddressesByToID into parallel slices + // 2. Build ToID -> LedgerCreatedAt lookup from transactions + ledgerCreatedAtByToID := make(map[int64]time.Time, len(txs)) + for _, tx := range txs { + ledgerCreatedAtByToID[tx.ToID] = tx.LedgerCreatedAt + } + + // 3. Flatten the stellarAddressesByToID into parallel slices var txToIDs []int64 var stellarAddresses []string + var taLedgerCreatedAts []time.Time for toID, addresses := range stellarAddressesByToID { + ledgerCreatedAt := ledgerCreatedAtByToID[toID] for address := range addresses.Iter() { txToIDs = append(txToIDs, toID) stellarAddresses = append(stellarAddresses, address) + taLedgerCreatedAts = append(taLedgerCreatedAts, ledgerCreatedAt) } } @@ -250,13 +259,14 @@ func (m *TransactionModel) BatchInsert( -- Insert transactions_accounts links inserted_transactions_accounts AS ( INSERT INTO transactions_accounts - (tx_to_id, account_id) + (ledger_created_at, tx_to_id, account_id) SELECT - ta.tx_to_id, ta.account_id + ta.ledger_created_at, ta.tx_to_id, ta.account_id FROM ( SELECT - UNNEST($10::bigint[]) AS tx_to_id, - UNNEST($11::text[]) AS account_id + UNNEST($10::timestamptz[]) AS ledger_created_at, + UNNEST($11::bigint[]) AS tx_to_id, + UNNEST($12::text[]) AS account_id ) ta ON CONFLICT DO NOTHING ) @@ -277,6 +287,7 @@ func (m *TransactionModel) BatchInsert( pq.Array(ledgerNumbers), pq.Array(ledgerCreatedAts), pq.Array(isFeeBumps), + pq.Array(taLedgerCreatedAts), pq.Array(txToIDs), pq.Array(stellarAddresses), ) @@ -350,18 +361,30 @@ func (m *TransactionModel) BatchCopy( // COPY transactions_accounts using pgx binary format with native pgtype types if len(stellarAddressesByToID) > 0 { + // Build ToID -> LedgerCreatedAt lookup from transactions + ledgerCreatedAtByToID := make(map[int64]time.Time, len(txs)) + for _, tx := range txs { + ledgerCreatedAtByToID[tx.ToID] = tx.LedgerCreatedAt + } + var taRows [][]any for toID, addresses := range stellarAddressesByToID { + ledgerCreatedAt := ledgerCreatedAtByToID[toID] + ledgerCreatedAtPgtype := pgtype.Timestamptz{Time: ledgerCreatedAt, Valid: true} toIDPgtype := pgtype.Int8{Int64: toID, Valid: true} for _, addr := range addresses.ToSlice() { - taRows = append(taRows, []any{toIDPgtype, pgtype.Text{String: addr, Valid: true}}) + taRows = append(taRows, []any{ + ledgerCreatedAtPgtype, + toIDPgtype, + pgtype.Text{String: addr, Valid: true}, + }) } } _, err = pgxTx.CopyFrom( ctx, pgx.Identifier{"transactions_accounts"}, - []string{"tx_to_id", "account_id"}, + []string{"ledger_created_at", "tx_to_id", "account_id"}, pgx.CopyFromRows(taRows), ) if err != nil { diff --git a/internal/db/migrations/2025-06-10.2-transactions.sql b/internal/db/migrations/2025-06-10.2-transactions.sql index b8b7558e7..bd7c87344 100644 --- a/internal/db/migrations/2025-06-10.2-transactions.sql +++ b/internal/db/migrations/2025-06-10.2-transactions.sql @@ -27,15 +27,21 @@ CREATE INDEX idx_transactions_hash ON transactions(hash); -- Index for to_id lookups (TOID-based queries) CREATE INDEX idx_transactions_to_id ON transactions(to_id); --- Table: transactions_accounts (no FK - hypertable compound PK) +-- Table: transactions_accounts (TimescaleDB hypertable for automatic cleanup with retention) CREATE TABLE transactions_accounts ( + ledger_created_at TIMESTAMPTZ NOT NULL, tx_to_id BIGINT NOT NULL, account_id TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (account_id, tx_to_id) + PRIMARY KEY (ledger_created_at, account_id, tx_to_id) +) WITH ( + tsdb.hypertable, + tsdb.partition_column = 'ledger_created_at', + tsdb.chunk_interval = '1 day', + tsdb.orderby = 'ledger_created_at DESC' ); CREATE INDEX idx_transactions_accounts_tx_to_id ON transactions_accounts(tx_to_id); +CREATE INDEX idx_transactions_accounts_account_id ON transactions_accounts(account_id); -- +migrate Down diff --git a/internal/db/migrations/2025-06-10.3-operations.sql b/internal/db/migrations/2025-06-10.3-operations.sql index 1e6273921..6dae253c6 100644 --- a/internal/db/migrations/2025-06-10.3-operations.sql +++ b/internal/db/migrations/2025-06-10.3-operations.sql @@ -37,15 +37,21 @@ CREATE INDEX idx_operations_ledger_created_at ON operations(ledger_created_at); -- Index for id lookups (TOID-based queries) CREATE INDEX idx_operations_id ON operations(id); --- Table: operations_accounts (no FK - hypertable compound PK) +-- Table: operations_accounts (TimescaleDB hypertable for automatic cleanup with retention) CREATE TABLE operations_accounts ( + ledger_created_at TIMESTAMPTZ NOT NULL, operation_id BIGINT NOT NULL, account_id TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (account_id, operation_id) + PRIMARY KEY (ledger_created_at, account_id, operation_id) +) WITH ( + tsdb.hypertable, + tsdb.partition_column = 'ledger_created_at', + tsdb.chunk_interval = '1 day', + tsdb.orderby = 'ledger_created_at DESC' ); CREATE INDEX idx_operations_accounts_operation_id ON operations_accounts(operation_id); +CREATE INDEX idx_operations_accounts_account_id ON operations_accounts(account_id); -- +migrate Down From f2bff57bebb0d23d26503422278a3e7a359ee8ff Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 2 Feb 2026 15:44:12 -0500 Subject: [PATCH 05/77] fix failing tests --- internal/data/accounts_test.go | 4 ++-- internal/data/operations_test.go | 10 +++++----- internal/data/transactions_test.go | 10 +++++----- internal/db/migrations/2025-06-10.2-transactions.sql | 3 --- internal/db/migrations/2025-06-10.4-statechanges.sql | 3 --- internal/serve/graphql/resolvers/test_utils.go | 8 ++++---- 6 files changed, 16 insertions(+), 22 deletions(-) diff --git a/internal/data/accounts_test.go b/internal/data/accounts_test.go index 97fbb5deb..271bb5dcb 100644 --- a/internal/data/accounts_test.go +++ b/internal/data/accounts_test.go @@ -263,7 +263,7 @@ func TestAccountModelBatchGetByToIDs(t *testing.T) { require.NoError(t, err) // Insert test transactions_accounts links - _, err = m.DB.ExecContext(ctx, "INSERT INTO transactions_accounts (tx_to_id, account_id) VALUES ($1, $2), ($3, $4)", toID1, address1, toID2, address2) + _, err = m.DB.ExecContext(ctx, "INSERT INTO transactions_accounts (ledger_created_at, tx_to_id, account_id) VALUES (NOW(), $1, $2), (NOW(), $3, $4)", toID1, address1, toID2, address2) require.NoError(t, err) // Test BatchGetByToIDs function @@ -317,7 +317,7 @@ func TestAccountModelBatchGetByOperationIDs(t *testing.T) { require.NoError(t, err) // Insert test operations_accounts links - _, err = m.DB.ExecContext(ctx, "INSERT INTO operations_accounts (operation_id, account_id) VALUES ($1, $2), ($3, $4)", operationID1, address1, operationID2, address2) + _, err = m.DB.ExecContext(ctx, "INSERT INTO operations_accounts (ledger_created_at, operation_id, account_id) VALUES (NOW(), $1, $2), (NOW(), $3, $4)", operationID1, address1, operationID2, address2) require.NoError(t, err) // Test BatchGetByOperationID function diff --git a/internal/data/operations_test.go b/internal/data/operations_test.go index a78387951..c96a939f5 100644 --- a/internal/data/operations_test.go +++ b/internal/data/operations_test.go @@ -820,12 +820,12 @@ func TestOperationModel_BatchGetByAccountAddresses(t *testing.T) { // Create test operations_accounts links _, err = dbConnectionPool.ExecContext(ctx, ` - INSERT INTO operations_accounts (operation_id, account_id) + INSERT INTO operations_accounts (ledger_created_at, operation_id, account_id) VALUES - (4097, $1), - (8193, $1), - (12289, $2) - `, address1, address2) + ($1, 4097, $2), + ($1, 8193, $2), + ($1, 12289, $3) + `, now, address1, address2) require.NoError(t, err) // Test BatchGetByAccount diff --git a/internal/data/transactions_test.go b/internal/data/transactions_test.go index f04ffc38b..4140c9349 100644 --- a/internal/data/transactions_test.go +++ b/internal/data/transactions_test.go @@ -584,12 +584,12 @@ func TestTransactionModel_BatchGetByAccountAddress(t *testing.T) { // Create test transactions_accounts links _, err = dbConnectionPool.ExecContext(ctx, ` - INSERT INTO transactions_accounts (tx_to_id, account_id) + INSERT INTO transactions_accounts (ledger_created_at, tx_to_id, account_id) VALUES - (1, $1), - (2, $1), - (3, $2) - `, address1, address2) + ($1, 1, $2), + ($1, 2, $2), + ($1, 3, $3) + `, now, address1, address2) require.NoError(t, err) // Test BatchGetByAccount diff --git a/internal/db/migrations/2025-06-10.2-transactions.sql b/internal/db/migrations/2025-06-10.2-transactions.sql index bd7c87344..44fe923fa 100644 --- a/internal/db/migrations/2025-06-10.2-transactions.sql +++ b/internal/db/migrations/2025-06-10.2-transactions.sql @@ -21,10 +21,7 @@ CREATE TABLE transactions ( tsdb.orderby = 'ledger_created_at DESC' ); --- Index for hash lookups (GetByHash queries) CREATE INDEX idx_transactions_hash ON transactions(hash); - --- Index for to_id lookups (TOID-based queries) CREATE INDEX idx_transactions_to_id ON transactions(to_id); -- Table: transactions_accounts (TimescaleDB hypertable for automatic cleanup with retention) diff --git a/internal/db/migrations/2025-06-10.4-statechanges.sql b/internal/db/migrations/2025-06-10.4-statechanges.sql index 75df06376..ae1ef8caf 100644 --- a/internal/db/migrations/2025-06-10.4-statechanges.sql +++ b/internal/db/migrations/2025-06-10.4-statechanges.sql @@ -52,14 +52,11 @@ CREATE TABLE state_changes ( tsdb.orderby = 'ledger_created_at DESC' ); --- Index for account_id lookups (most common query pattern) CREATE INDEX idx_state_changes_account_id ON state_changes(account_id); CREATE INDEX idx_state_changes_ledger_created_at ON state_changes(ledger_created_at); -- Index for to_id lookups (transaction-based queries) CREATE INDEX idx_state_changes_to_id ON state_changes(to_id); - --- Index for operation_id lookups CREATE INDEX idx_state_changes_operation_id ON state_changes(operation_id) WHERE operation_id > 0; -- +migrate Down diff --git a/internal/serve/graphql/resolvers/test_utils.go b/internal/serve/graphql/resolvers/test_utils.go index 0da392cec..0fea49fc6 100644 --- a/internal/serve/graphql/resolvers/test_utils.go +++ b/internal/serve/graphql/resolvers/test_utils.go @@ -136,8 +136,8 @@ func setupDB(ctx context.Context, t *testing.T, dbConnectionPool db.ConnectionPo require.NoError(t, err) _, err = tx.ExecContext(ctx, - `INSERT INTO transactions_accounts (tx_to_id, account_id) VALUES ($1, $2)`, - txn.ToID, parentAccount.StellarAddress) + `INSERT INTO transactions_accounts (ledger_created_at, tx_to_id, account_id) VALUES ($1, $2, $3)`, + txn.LedgerCreatedAt, txn.ToID, parentAccount.StellarAddress) require.NoError(t, err) } @@ -148,8 +148,8 @@ func setupDB(ctx context.Context, t *testing.T, dbConnectionPool db.ConnectionPo require.NoError(t, err) _, err = tx.ExecContext(ctx, - `INSERT INTO operations_accounts (operation_id, account_id) VALUES ($1, $2)`, - op.ID, parentAccount.StellarAddress) + `INSERT INTO operations_accounts (ledger_created_at, operation_id, account_id) VALUES ($1, $2, $3)`, + op.LedgerCreatedAt, op.ID, parentAccount.StellarAddress) require.NoError(t, err) } From 2bc2180d102ffab2175e60e4b44e61fa672f1051 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 2 Feb 2026 16:53:13 -0500 Subject: [PATCH 06/77] remove comments --- internal/db/migrations/2025-06-10.3-operations.sql | 2 -- internal/db/migrations/2025-06-10.4-statechanges.sql | 2 -- 2 files changed, 4 deletions(-) diff --git a/internal/db/migrations/2025-06-10.3-operations.sql b/internal/db/migrations/2025-06-10.3-operations.sql index 6dae253c6..d9dc2ea6e 100644 --- a/internal/db/migrations/2025-06-10.3-operations.sql +++ b/internal/db/migrations/2025-06-10.3-operations.sql @@ -33,8 +33,6 @@ CREATE TABLE operations ( ); CREATE INDEX idx_operations_ledger_created_at ON operations(ledger_created_at); - --- Index for id lookups (TOID-based queries) CREATE INDEX idx_operations_id ON operations(id); -- Table: operations_accounts (TimescaleDB hypertable for automatic cleanup with retention) diff --git a/internal/db/migrations/2025-06-10.4-statechanges.sql b/internal/db/migrations/2025-06-10.4-statechanges.sql index ae1ef8caf..91fc1c14a 100644 --- a/internal/db/migrations/2025-06-10.4-statechanges.sql +++ b/internal/db/migrations/2025-06-10.4-statechanges.sql @@ -54,8 +54,6 @@ CREATE TABLE state_changes ( CREATE INDEX idx_state_changes_account_id ON state_changes(account_id); CREATE INDEX idx_state_changes_ledger_created_at ON state_changes(ledger_created_at); - --- Index for to_id lookups (transaction-based queries) CREATE INDEX idx_state_changes_to_id ON state_changes(to_id); CREATE INDEX idx_state_changes_operation_id ON state_changes(operation_id) WHERE operation_id > 0; From 34124ee24b26ed7097b23bf5b63447a129503e2f Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 2 Feb 2026 16:55:30 -0500 Subject: [PATCH 07/77] changes to indexes --- internal/db/migrations/2025-06-10.3-operations.sql | 1 - internal/db/migrations/2025-06-10.4-statechanges.sql | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/db/migrations/2025-06-10.3-operations.sql b/internal/db/migrations/2025-06-10.3-operations.sql index d9dc2ea6e..6b5cec85c 100644 --- a/internal/db/migrations/2025-06-10.3-operations.sql +++ b/internal/db/migrations/2025-06-10.3-operations.sql @@ -32,7 +32,6 @@ CREATE TABLE operations ( tsdb.orderby = 'ledger_created_at DESC' ); -CREATE INDEX idx_operations_ledger_created_at ON operations(ledger_created_at); CREATE INDEX idx_operations_id ON operations(id); -- Table: operations_accounts (TimescaleDB hypertable for automatic cleanup with retention) diff --git a/internal/db/migrations/2025-06-10.4-statechanges.sql b/internal/db/migrations/2025-06-10.4-statechanges.sql index 91fc1c14a..d167790c6 100644 --- a/internal/db/migrations/2025-06-10.4-statechanges.sql +++ b/internal/db/migrations/2025-06-10.4-statechanges.sql @@ -53,9 +53,8 @@ CREATE TABLE state_changes ( ); CREATE INDEX idx_state_changes_account_id ON state_changes(account_id); -CREATE INDEX idx_state_changes_ledger_created_at ON state_changes(ledger_created_at); CREATE INDEX idx_state_changes_to_id ON state_changes(to_id); -CREATE INDEX idx_state_changes_operation_id ON state_changes(operation_id) WHERE operation_id > 0; +CREATE INDEX idx_state_changes_operation_id ON state_changes(operation_id); -- +migrate Down From e51e6978a297b5cf0750069c14a76d1d30e3fb1f Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 2 Feb 2026 17:06:47 -0500 Subject: [PATCH 08/77] Update 2025-06-10.2-transactions.sql --- internal/db/migrations/2025-06-10.2-transactions.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/db/migrations/2025-06-10.2-transactions.sql b/internal/db/migrations/2025-06-10.2-transactions.sql index 44fe923fa..eee4fd6d4 100644 --- a/internal/db/migrations/2025-06-10.2-transactions.sql +++ b/internal/db/migrations/2025-06-10.2-transactions.sql @@ -13,7 +13,6 @@ CREATE TABLE transactions ( is_fee_bump BOOLEAN NOT NULL DEFAULT false, ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), PRIMARY KEY (ledger_created_at, to_id), - UNIQUE (ledger_created_at, hash) ) WITH ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', From 3fb3a6c4dc709f505cad13b55baeb8786fa15e85 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 2 Feb 2026 17:22:52 -0500 Subject: [PATCH 09/77] Update 2025-06-10.2-transactions.sql --- internal/db/migrations/2025-06-10.2-transactions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/db/migrations/2025-06-10.2-transactions.sql b/internal/db/migrations/2025-06-10.2-transactions.sql index eee4fd6d4..8330481e8 100644 --- a/internal/db/migrations/2025-06-10.2-transactions.sql +++ b/internal/db/migrations/2025-06-10.2-transactions.sql @@ -12,7 +12,7 @@ CREATE TABLE transactions ( ledger_number INTEGER NOT NULL, is_fee_bump BOOLEAN NOT NULL DEFAULT false, ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (ledger_created_at, to_id), + PRIMARY KEY (ledger_created_at, to_id) ) WITH ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', From 66c39bef5d43e74d5d367063dcc29c925c65f3f7 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 2 Feb 2026 19:13:30 -0500 Subject: [PATCH 10/77] compress chunks at the end --- internal/services/ingest_backfill.go | 77 ++++++++++++++++++++++++---- internal/services/ingest_test.go | 4 +- 2 files changed, 71 insertions(+), 10 deletions(-) diff --git a/internal/services/ingest_backfill.go b/internal/services/ingest_backfill.go index a996d717d..a4b7fd53b 100644 --- a/internal/services/ingest_backfill.go +++ b/internal/services/ingest_backfill.go @@ -48,6 +48,8 @@ type BackfillResult struct { Duration time.Duration Error error BatchChanges *BatchChanges // Only populated for catchup mode + StartTime time.Time // First ledger close time in batch (for compression) + EndTime time.Time // Last ledger close time in batch (for compression) } // BatchChanges holds data collected from a backfill batch for catchup mode. @@ -176,6 +178,24 @@ func (m *ingestService) startBackfilling(ctx context.Context, startLedger, endLe numFailedBatches := analyzeBatchResults(ctx, results) + // Compress backfilled chunks for historical mode (no batches failed) + if mode.isHistorical() && numFailedBatches == 0 { + var minTime, maxTime time.Time + for _, result := range results { + if result.Error == nil { + if minTime.IsZero() || result.StartTime.Before(minTime) { + minTime = result.StartTime + } + if result.EndTime.After(maxTime) { + maxTime = result.EndTime + } + } + } + if !minTime.IsZero() { + m.compressBackfilledChunks(ctx, minTime, maxTime) + } + } + // Update latest ledger cursor and process catchup data for catchup mode if mode.isCatchup() { if numFailedBatches > 0 { @@ -347,9 +367,11 @@ func (m *ingestService) processSingleBatch(ctx context.Context, mode BackfillMod }() // Process all ledgers in batch (cursor is updated atomically with final flush for historical mode) - ledgersCount, batchChanges, err := m.processLedgersInBatch(ctx, backend, batch, mode) + ledgersCount, batchChanges, timeRange, err := m.processLedgersInBatch(ctx, backend, batch, mode) result.LedgersCount = ledgersCount result.BatchChanges = batchChanges + result.StartTime = timeRange.StartTime + result.EndTime = timeRange.EndTime if err != nil { result.Error = err result.Duration = time.Since(start) @@ -455,19 +477,26 @@ func (m *ingestService) flushBatchBufferWithRetry(ctx context.Context, buffer *i return lastErr } +// BatchTimeRange captures the time range of ledgers processed in a batch. +type BatchTimeRange struct { + StartTime time.Time + EndTime time.Time +} + // processLedgersInBatch processes all ledgers in a batch, flushing to DB periodically. // For historical backfill mode, the cursor is updated atomically with the final data flush. // For catchup mode, returns collected batch changes for post-catchup processing. -// Returns the count of ledgers processed and batch changes (nil for historical mode). +// Returns the count of ledgers processed, batch changes (nil for historical mode), and time range. func (m *ingestService) processLedgersInBatch( ctx context.Context, backend ledgerbackend.LedgerBackend, batch BackfillBatch, mode BackfillMode, -) (int, *BatchChanges, error) { +) (int, *BatchChanges, BatchTimeRange, error) { batchBuffer := indexer.NewIndexerBuffer() ledgersInBuffer := uint32(0) ledgersProcessed := 0 + var timeRange BatchTimeRange // Initialize batch changes collector for catchup mode var batchChanges *BatchChanges @@ -485,11 +514,18 @@ func (m *ingestService) processLedgersInBatch( for ledgerSeq := batch.StartLedger; ledgerSeq <= batch.EndLedger; ledgerSeq++ { ledgerMeta, err := m.getLedgerWithRetry(ctx, backend, ledgerSeq) if err != nil { - return ledgersProcessed, nil, fmt.Errorf("getting ledger %d: %w", ledgerSeq, err) + return ledgersProcessed, nil, timeRange, fmt.Errorf("getting ledger %d: %w", ledgerSeq, err) + } + + // Track time range for compression + ledgerTime := ledgerMeta.ClosedAt() + if timeRange.StartTime.IsZero() { + timeRange.StartTime = ledgerTime } + timeRange.EndTime = ledgerTime if err := m.processLedger(ctx, ledgerMeta, batchBuffer); err != nil { - return ledgersProcessed, nil, fmt.Errorf("processing ledger %d: %w", ledgerSeq, err) + return ledgersProcessed, nil, timeRange, fmt.Errorf("processing ledger %d: %w", ledgerSeq, err) } ledgersProcessed++ ledgersInBuffer++ @@ -497,7 +533,7 @@ func (m *ingestService) processLedgersInBatch( // Flush buffer periodically to control memory usage (intermediate flushes, no cursor update) if ledgersInBuffer >= m.backfillDBInsertBatchSize { if err := m.flushBatchBufferWithRetry(ctx, batchBuffer, nil, batchChanges); err != nil { - return ledgersProcessed, batchChanges, err + return ledgersProcessed, batchChanges, timeRange, err } batchBuffer.Clear() ledgersInBuffer = 0 @@ -511,17 +547,17 @@ func (m *ingestService) processLedgersInBatch( cursorUpdate = &batch.StartLedger } if err := m.flushBatchBufferWithRetry(ctx, batchBuffer, cursorUpdate, batchChanges); err != nil { - return ledgersProcessed, batchChanges, err + return ledgersProcessed, batchChanges, timeRange, err } } else if mode.isHistorical() { // All data was flushed in intermediate batches, but we still need to update the cursor // This happens when ledgersInBuffer == 0 (exact multiple of batch size) if err := m.updateOldestCursor(ctx, batch.StartLedger); err != nil { - return ledgersProcessed, nil, err + return ledgersProcessed, nil, timeRange, err } } - return ledgersProcessed, batchChanges, nil + return ledgersProcessed, batchChanges, timeRange, nil } // updateOldestCursor updates the oldest ledger cursor to the given ledger. @@ -584,3 +620,26 @@ func (m *ingestService) processBatchChanges( return nil } + +// compressBackfilledChunks compresses TimescaleDB chunks within the given time range. +// This is called after historical backfill completes to ensure newly created chunks +// are compressed immediately, rather than waiting for the compression policy interval. +func (m *ingestService) compressBackfilledChunks(ctx context.Context, startTime, endTime time.Time) { + tables := []string{"transactions", "transactions_accounts", "operations", "operations_accounts", "state_changes"} + for _, table := range tables { + _, err := m.models.DB.PgxPool().Exec(ctx, ` + DO $$ + DECLARE chunk regclass; + BEGIN + FOR chunk IN SELECT c FROM show_chunks($1::regclass, older_than => $2, newer_than => $3) c + LOOP + CALL convert_to_columnstore(chunk, if_not_columnstore => true); + END LOOP; + END$$; + `, table, endTime, startTime) + if err != nil { + log.Ctx(ctx).Warnf("Failed to compress chunks for %s: %v", table, err) + } + } + log.Ctx(ctx).Infof("Compressed backfilled chunks for time range [%s - %s]", startTime.Format(time.RFC3339), endTime.Format(time.RFC3339)) +} diff --git a/internal/services/ingest_test.go b/internal/services/ingest_test.go index 2d5b12c32..775768345 100644 --- a/internal/services/ingest_test.go +++ b/internal/services/ingest_test.go @@ -2884,10 +2884,12 @@ func Test_ingestService_processLedgersInBatch_catchupMode(t *testing.T) { require.NoError(t, err) batch := BackfillBatch{StartLedger: 4599, EndLedger: 4599} - ledgersProcessed, batchChanges, err := svc.processLedgersInBatch(ctx, mockLedgerBackend, batch, tc.mode) + ledgersProcessed, batchChanges, timeRange, err := svc.processLedgersInBatch(ctx, mockLedgerBackend, batch, tc.mode) require.NoError(t, err) assert.Equal(t, 1, ledgersProcessed) + assert.False(t, timeRange.StartTime.IsZero(), "expected non-zero start time") + assert.False(t, timeRange.EndTime.IsZero(), "expected non-zero end time") if tc.wantBatchChangesNil { assert.Nil(t, batchChanges, "expected nil batch changes for historical mode") From 1bf65801ae5d1d263429879a704d69ef19d925b8 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 2 Feb 2026 19:47:31 -0500 Subject: [PATCH 11/77] parallely compress chunks --- internal/services/ingest_backfill.go | 74 ++++++++++++++++------------ internal/services/ingest_test.go | 6 +-- 2 files changed, 45 insertions(+), 35 deletions(-) diff --git a/internal/services/ingest_backfill.go b/internal/services/ingest_backfill.go index a4b7fd53b..d3234c85d 100644 --- a/internal/services/ingest_backfill.go +++ b/internal/services/ingest_backfill.go @@ -367,11 +367,11 @@ func (m *ingestService) processSingleBatch(ctx context.Context, mode BackfillMod }() // Process all ledgers in batch (cursor is updated atomically with final flush for historical mode) - ledgersCount, batchChanges, timeRange, err := m.processLedgersInBatch(ctx, backend, batch, mode) + ledgersCount, batchChanges, batchStartTime, batchEndTime, err := m.processLedgersInBatch(ctx, backend, batch, mode) result.LedgersCount = ledgersCount result.BatchChanges = batchChanges - result.StartTime = timeRange.StartTime - result.EndTime = timeRange.EndTime + result.StartTime = batchStartTime + result.EndTime = batchEndTime if err != nil { result.Error = err result.Duration = time.Since(start) @@ -477,12 +477,6 @@ func (m *ingestService) flushBatchBufferWithRetry(ctx context.Context, buffer *i return lastErr } -// BatchTimeRange captures the time range of ledgers processed in a batch. -type BatchTimeRange struct { - StartTime time.Time - EndTime time.Time -} - // processLedgersInBatch processes all ledgers in a batch, flushing to DB periodically. // For historical backfill mode, the cursor is updated atomically with the final data flush. // For catchup mode, returns collected batch changes for post-catchup processing. @@ -492,11 +486,11 @@ func (m *ingestService) processLedgersInBatch( backend ledgerbackend.LedgerBackend, batch BackfillBatch, mode BackfillMode, -) (int, *BatchChanges, BatchTimeRange, error) { +) (int, *BatchChanges, time.Time, time.Time, error) { batchBuffer := indexer.NewIndexerBuffer() ledgersInBuffer := uint32(0) ledgersProcessed := 0 - var timeRange BatchTimeRange + var startTime, endTime time.Time // Initialize batch changes collector for catchup mode var batchChanges *BatchChanges @@ -514,18 +508,18 @@ func (m *ingestService) processLedgersInBatch( for ledgerSeq := batch.StartLedger; ledgerSeq <= batch.EndLedger; ledgerSeq++ { ledgerMeta, err := m.getLedgerWithRetry(ctx, backend, ledgerSeq) if err != nil { - return ledgersProcessed, nil, timeRange, fmt.Errorf("getting ledger %d: %w", ledgerSeq, err) + return ledgersProcessed, nil, startTime, endTime, fmt.Errorf("getting ledger %d: %w", ledgerSeq, err) } // Track time range for compression ledgerTime := ledgerMeta.ClosedAt() - if timeRange.StartTime.IsZero() { - timeRange.StartTime = ledgerTime + if startTime.IsZero() { + startTime = ledgerTime } - timeRange.EndTime = ledgerTime + endTime = ledgerTime if err := m.processLedger(ctx, ledgerMeta, batchBuffer); err != nil { - return ledgersProcessed, nil, timeRange, fmt.Errorf("processing ledger %d: %w", ledgerSeq, err) + return ledgersProcessed, nil, startTime, endTime, fmt.Errorf("processing ledger %d: %w", ledgerSeq, err) } ledgersProcessed++ ledgersInBuffer++ @@ -533,7 +527,7 @@ func (m *ingestService) processLedgersInBatch( // Flush buffer periodically to control memory usage (intermediate flushes, no cursor update) if ledgersInBuffer >= m.backfillDBInsertBatchSize { if err := m.flushBatchBufferWithRetry(ctx, batchBuffer, nil, batchChanges); err != nil { - return ledgersProcessed, batchChanges, timeRange, err + return ledgersProcessed, batchChanges, startTime, endTime, err } batchBuffer.Clear() ledgersInBuffer = 0 @@ -547,17 +541,17 @@ func (m *ingestService) processLedgersInBatch( cursorUpdate = &batch.StartLedger } if err := m.flushBatchBufferWithRetry(ctx, batchBuffer, cursorUpdate, batchChanges); err != nil { - return ledgersProcessed, batchChanges, timeRange, err + return ledgersProcessed, batchChanges, startTime, endTime, err } } else if mode.isHistorical() { // All data was flushed in intermediate batches, but we still need to update the cursor // This happens when ledgersInBuffer == 0 (exact multiple of batch size) if err := m.updateOldestCursor(ctx, batch.StartLedger); err != nil { - return ledgersProcessed, nil, timeRange, err + return ledgersProcessed, nil, startTime, endTime, err } } - return ledgersProcessed, batchChanges, timeRange, nil + return ledgersProcessed, batchChanges, startTime, endTime, nil } // updateOldestCursor updates the oldest ledger cursor to the given ledger. @@ -624,22 +618,38 @@ func (m *ingestService) processBatchChanges( // compressBackfilledChunks compresses TimescaleDB chunks within the given time range. // This is called after historical backfill completes to ensure newly created chunks // are compressed immediately, rather than waiting for the compression policy interval. +// Chunks are compressed in parallel using the backfill worker pool. func (m *ingestService) compressBackfilledChunks(ctx context.Context, startTime, endTime time.Time) { tables := []string{"transactions", "transactions_accounts", "operations", "operations_accounts", "state_changes"} + group := m.backfillPool.NewGroupContext(ctx) + for _, table := range tables { - _, err := m.models.DB.PgxPool().Exec(ctx, ` - DO $$ - DECLARE chunk regclass; - BEGIN - FOR chunk IN SELECT c FROM show_chunks($1::regclass, older_than => $2, newer_than => $3) c - LOOP - CALL convert_to_columnstore(chunk, if_not_columnstore => true); - END LOOP; - END$$; - `, table, endTime, startTime) + rows, err := m.models.DB.PgxPool().Query(ctx, + `SELECT c::text FROM show_chunks($1::regclass, older_than => $2, newer_than => $3) c`, + table, endTime, startTime) if err != nil { - log.Ctx(ctx).Warnf("Failed to compress chunks for %s: %v", table, err) + log.Ctx(ctx).Warnf("Failed to get chunks for %s: %v", table, err) + continue } + for rows.Next() { + var chunk string + if err := rows.Scan(&chunk); err != nil { + continue + } + group.Submit(func() { + _, err := m.models.DB.PgxPool().Exec(ctx, + `CALL convert_to_columnstore($1::regclass, if_not_columnstore => true)`, chunk) + if err != nil { + log.Ctx(ctx).Warnf("Failed to compress chunk %s: %v", chunk, err) + } + }) + } + rows.Close() } - log.Ctx(ctx).Infof("Compressed backfilled chunks for time range [%s - %s]", startTime.Format(time.RFC3339), endTime.Format(time.RFC3339)) + if err := group.Wait(); err != nil { + log.Ctx(ctx).Warnf("Chunk compression group wait returned error: %v", err) + } + + log.Ctx(ctx).Infof("Compressed backfilled chunks for time range [%s - %s]", + startTime.Format(time.RFC3339), endTime.Format(time.RFC3339)) } diff --git a/internal/services/ingest_test.go b/internal/services/ingest_test.go index 775768345..60d4dfdc9 100644 --- a/internal/services/ingest_test.go +++ b/internal/services/ingest_test.go @@ -2884,12 +2884,12 @@ func Test_ingestService_processLedgersInBatch_catchupMode(t *testing.T) { require.NoError(t, err) batch := BackfillBatch{StartLedger: 4599, EndLedger: 4599} - ledgersProcessed, batchChanges, timeRange, err := svc.processLedgersInBatch(ctx, mockLedgerBackend, batch, tc.mode) + ledgersProcessed, batchChanges, startTime, endTime, err := svc.processLedgersInBatch(ctx, mockLedgerBackend, batch, tc.mode) require.NoError(t, err) assert.Equal(t, 1, ledgersProcessed) - assert.False(t, timeRange.StartTime.IsZero(), "expected non-zero start time") - assert.False(t, timeRange.EndTime.IsZero(), "expected non-zero end time") + assert.False(t, startTime.IsZero(), "expected non-zero start time") + assert.False(t, endTime.IsZero(), "expected non-zero end time") if tc.wantBatchChangesNil { assert.Nil(t, batchChanges, "expected nil batch changes for historical mode") From 2fffe5df93c028012410a77e353f7b53600f8fcf Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 2 Feb 2026 20:08:14 -0500 Subject: [PATCH 12/77] Update ingest_backfill.go --- internal/services/ingest_backfill.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/services/ingest_backfill.go b/internal/services/ingest_backfill.go index d3234c85d..cd0612fac 100644 --- a/internal/services/ingest_backfill.go +++ b/internal/services/ingest_backfill.go @@ -625,7 +625,7 @@ func (m *ingestService) compressBackfilledChunks(ctx context.Context, startTime, for _, table := range tables { rows, err := m.models.DB.PgxPool().Query(ctx, - `SELECT c::text FROM show_chunks($1::regclass, older_than => $2, newer_than => $3) c`, + `SELECT c::text FROM show_chunks($1::regclass, older_than => $2::timestamptz, newer_than => $3::timestamptz) c`, table, endTime, startTime) if err != nil { log.Ctx(ctx).Warnf("Failed to get chunks for %s: %v", table, err) From fc165bf7f33aa6e88cd2dd109c815fb58bc1ade4 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 2 Feb 2026 20:39:04 -0500 Subject: [PATCH 13/77] Update ingest_backfill.go --- internal/services/ingest_backfill.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/internal/services/ingest_backfill.go b/internal/services/ingest_backfill.go index cd0612fac..6918f721b 100644 --- a/internal/services/ingest_backfill.go +++ b/internal/services/ingest_backfill.go @@ -615,17 +615,18 @@ func (m *ingestService) processBatchChanges( return nil } -// compressBackfilledChunks compresses TimescaleDB chunks within the given time range. -// This is called after historical backfill completes to ensure newly created chunks -// are compressed immediately, rather than waiting for the compression policy interval. -// Chunks are compressed in parallel using the backfill worker pool. +// compressBackfilledChunks compresses uncompressed chunks overlapping the backfill range. +// Skips chunks where range_end >= NOW() to avoid compressing active live ingestion chunks. func (m *ingestService) compressBackfilledChunks(ctx context.Context, startTime, endTime time.Time) { tables := []string{"transactions", "transactions_accounts", "operations", "operations_accounts", "state_changes"} group := m.backfillPool.NewGroupContext(ctx) for _, table := range tables { rows, err := m.models.DB.PgxPool().Query(ctx, - `SELECT c::text FROM show_chunks($1::regclass, older_than => $2::timestamptz, newer_than => $3::timestamptz) c`, + `SELECT chunk_schema || '.' || chunk_name FROM timescaledb_information.chunks + WHERE hypertable_name = $1 AND NOT is_compressed + AND range_start < $2::timestamptz AND range_end > $3::timestamptz + AND range_end < NOW()`, table, endTime, startTime) if err != nil { log.Ctx(ctx).Warnf("Failed to get chunks for %s: %v", table, err) @@ -646,10 +647,6 @@ func (m *ingestService) compressBackfilledChunks(ctx context.Context, startTime, } rows.Close() } - if err := group.Wait(); err != nil { - log.Ctx(ctx).Warnf("Chunk compression group wait returned error: %v", err) - } - - log.Ctx(ctx).Infof("Compressed backfilled chunks for time range [%s - %s]", - startTime.Format(time.RFC3339), endTime.Format(time.RFC3339)) + _ = group.Wait() + log.Ctx(ctx).Infof("Compressed backfilled chunks for time range [%s - %s]", startTime.Format(time.RFC3339), endTime.Format(time.RFC3339)) } From 38d08c1ec793fd4cad09e672dbd069abdd38b029 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 3 Feb 2026 09:07:30 -0500 Subject: [PATCH 14/77] Update ingest_backfill.go --- internal/services/ingest_backfill.go | 37 ++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/internal/services/ingest_backfill.go b/internal/services/ingest_backfill.go index 6918f721b..00ec1eb46 100644 --- a/internal/services/ingest_backfill.go +++ b/internal/services/ingest_backfill.go @@ -616,10 +616,11 @@ func (m *ingestService) processBatchChanges( } // compressBackfilledChunks compresses uncompressed chunks overlapping the backfill range. +// Chunks are compressed sequentially to avoid OOM errors during large backfills. // Skips chunks where range_end >= NOW() to avoid compressing active live ingestion chunks. func (m *ingestService) compressBackfilledChunks(ctx context.Context, startTime, endTime time.Time) { tables := []string{"transactions", "transactions_accounts", "operations", "operations_accounts", "state_changes"} - group := m.backfillPool.NewGroupContext(ctx) + totalCompressed := 0 for _, table := range tables { rows, err := m.models.DB.PgxPool().Query(ctx, @@ -632,21 +633,37 @@ func (m *ingestService) compressBackfilledChunks(ctx context.Context, startTime, log.Ctx(ctx).Warnf("Failed to get chunks for %s: %v", table, err) continue } + + var chunks []string for rows.Next() { var chunk string if err := rows.Scan(&chunk); err != nil { continue } - group.Submit(func() { - _, err := m.models.DB.PgxPool().Exec(ctx, - `CALL convert_to_columnstore($1::regclass, if_not_columnstore => true)`, chunk) - if err != nil { - log.Ctx(ctx).Warnf("Failed to compress chunk %s: %v", chunk, err) - } - }) + chunks = append(chunks, chunk) } rows.Close() + + // Compress chunks sequentially to avoid OOM + for i, chunk := range chunks { + select { + case <-ctx.Done(): + log.Ctx(ctx).Warnf("Compression cancelled after %d chunks", totalCompressed) + return + default: + } + + _, err := m.models.DB.PgxPool().Exec(ctx, + `CALL convert_to_columnstore($1::regclass, if_not_columnstore => true)`, chunk) + if err != nil { + log.Ctx(ctx).Warnf("Failed to compress chunk %s: %v", chunk, err) + continue + } + totalCompressed++ + log.Ctx(ctx).Debugf("Compressed chunk %d/%d for %s: %s", i+1, len(chunks), table, chunk) + } + log.Ctx(ctx).Infof("Compressed %d chunks for table %s", len(chunks), table) } - _ = group.Wait() - log.Ctx(ctx).Infof("Compressed backfilled chunks for time range [%s - %s]", startTime.Format(time.RFC3339), endTime.Format(time.RFC3339)) + log.Ctx(ctx).Infof("Compressed %d total chunks for time range [%s - %s]", + totalCompressed, startTime.Format(time.RFC3339), endTime.Format(time.RFC3339)) } From 369e79a0ba9e29c1f7ebb44df5013a204d8b1f29 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 10 Feb 2026 14:41:44 -0500 Subject: [PATCH 15/77] Cherry-pick opxdr-bytea-2 files (types, indexer, graphql, processors, tests) Bring in BYTEA type definitions, indexer changes, GraphQL schema/resolver updates, processor changes, and test files from opxdr-bytea-2 branch. These files had no modifications on the timescale branch. --- internal/data/accounts.go | 67 +++- internal/data/query_utils.go | 21 +- .../db/migrations/2024-04-29.1-accounts.sql | 2 +- internal/indexer/indexer.go | 4 +- internal/indexer/indexer_buffer.go | 6 +- internal/indexer/indexer_buffer_test.go | 86 +++--- internal/indexer/indexer_test.go | 10 +- .../processors/contracts/test_utils.go | 2 +- .../processors/contracts_test_utils.go | 4 +- internal/indexer/processors/effects_test.go | 36 +-- .../processors/processors_test_utils.go | 2 +- .../processors/state_change_builder.go | 20 +- .../indexer/processors/token_transfer_test.go | 2 +- internal/indexer/processors/utils.go | 6 +- internal/indexer/processors/utils_test.go | 7 +- internal/indexer/types/types.go | 187 ++++++++++- internal/indexer/types/types_test.go | 291 ++++++++++++++++++ .../infrastructure/backfill_helpers.go | 8 +- internal/serve/graphql/generated/generated.go | 100 +++++- .../graphql/resolvers/account.resolvers.go | 8 +- .../resolvers/account_resolvers_test.go | 98 +++--- .../graphql/resolvers/mutations.resolvers.go | 2 +- .../resolvers/mutations_resolvers_test.go | 2 +- .../graphql/resolvers/operation.resolvers.go | 6 + .../resolvers/operation_resolvers_test.go | 4 +- .../graphql/resolvers/queries.resolvers.go | 2 +- .../resolvers/queries_resolvers_test.go | 85 ++--- internal/serve/graphql/resolvers/resolver.go | 10 + .../resolvers/statechange.resolvers.go | 8 +- .../resolvers/statechange_resolvers_test.go | 4 +- .../resolvers/transaction.resolvers.go | 5 + .../resolvers/transaction_resolvers_test.go | 33 +- .../serve/graphql/schema/operation.graphqls | 2 +- .../serve/graphql/schema/transaction.graphqls | 2 +- internal/services/account_service_test.go | 9 +- internal/services/ingest.go | 2 +- internal/utils/sql.go | 9 + 37 files changed, 889 insertions(+), 263 deletions(-) diff --git a/internal/data/accounts.go b/internal/data/accounts.go index 89187fee7..d2450c242 100644 --- a/internal/data/accounts.go +++ b/internal/data/accounts.go @@ -36,7 +36,7 @@ func (m *AccountModel) Get(ctx context.Context, address string) (*types.Account, const query = `SELECT * FROM accounts WHERE stellar_address = $1` var account types.Account start := time.Now() - err := m.DB.GetContext(ctx, &account, query, address) + err := m.DB.GetContext(ctx, &account, query, types.AddressBytea(address)) duration := time.Since(start).Seconds() m.MetricsService.ObserveDBQueryDuration("Get", "accounts", duration) if err != nil { @@ -50,8 +50,8 @@ func (m *AccountModel) Get(ctx context.Context, address string) (*types.Account, func (m *AccountModel) GetAll(ctx context.Context) ([]string, error) { const query = `SELECT stellar_address FROM accounts` start := time.Now() - accounts := []string{} - err := m.DB.SelectContext(ctx, &accounts, query) + var addresses []types.AddressBytea + err := m.DB.SelectContext(ctx, &addresses, query) duration := time.Since(start).Seconds() m.MetricsService.ObserveDBQueryDuration("GetAll", "accounts", duration) if err != nil { @@ -59,13 +59,18 @@ func (m *AccountModel) GetAll(ctx context.Context) ([]string, error) { return nil, fmt.Errorf("getting all accounts: %w", err) } m.MetricsService.IncDBQuery("GetAll", "accounts") - return accounts, nil + // Convert []AddressBytea to []string + result := make([]string, len(addresses)) + for i, addr := range addresses { + result[i] = string(addr) + } + return result, nil } func (m *AccountModel) Insert(ctx context.Context, address string) error { const query = `INSERT INTO accounts (stellar_address) VALUES ($1)` start := time.Now() - _, err := m.DB.ExecContext(ctx, query, address) + _, err := m.DB.ExecContext(ctx, query, types.AddressBytea(address)) duration := time.Since(start).Seconds() m.MetricsService.ObserveDBQueryDuration("Insert", "accounts", duration) if err != nil { @@ -82,7 +87,7 @@ func (m *AccountModel) Insert(ctx context.Context, address string) error { func (m *AccountModel) Delete(ctx context.Context, address string) error { const query = `DELETE FROM accounts WHERE stellar_address = $1` start := time.Now() - result, err := m.DB.ExecContext(ctx, query, address) + result, err := m.DB.ExecContext(ctx, query, types.AddressBytea(address)) duration := time.Since(start).Seconds() m.MetricsService.ObserveDBQueryDuration("Delete", "accounts", duration) if err != nil { @@ -104,25 +109,53 @@ func (m *AccountModel) Delete(ctx context.Context, address string) error { return nil } +// BatchGetByIDs returns the subset of provided account IDs that exist in the accounts table. // BatchGetByIDs returns the subset of provided account IDs that exist in the accounts table. func (m *AccountModel) BatchGetByIDs(ctx context.Context, dbTx pgx.Tx, accountIDs []string) ([]string, error) { if len(accountIDs) == 0 { return []string{}, nil } + // Convert string addresses to [][]byte for BYTEA array comparison + byteAddresses := make([][]byte, len(accountIDs)) + for i, addr := range accountIDs { + addrBytes, err := types.AddressBytea(addr).Value() + if err != nil { + return nil, fmt.Errorf("converting address %s to bytes: %w", addr, err) + } + if addrBytes == nil { + return nil, fmt.Errorf("address %s converted to nil", addr) + } + byteAddresses[i] = addrBytes.([]byte) + } + const query = `SELECT stellar_address FROM accounts WHERE stellar_address = ANY($1)` start := time.Now() - var existingAccounts []string - rows, err := dbTx.Query(ctx, query, accountIDs) + rows, err := dbTx.Query(ctx, query, byteAddresses) if err != nil { m.MetricsService.IncDBQueryError("BatchGetByIDs", "accounts", utils.GetDBErrorType(err)) return nil, fmt.Errorf("querying accounts by IDs: %w", err) } - existingAccounts, err = pgx.CollectRows(rows, pgx.RowTo[string]) - if err != nil { + defer rows.Close() + + var existingAccounts []string + for rows.Next() { + var addrBytes []byte + if err := rows.Scan(&addrBytes); err != nil { + m.MetricsService.IncDBQueryError("BatchGetByIDs", "accounts", utils.GetDBErrorType(err)) + return nil, fmt.Errorf("scanning address: %w", err) + } + var addr types.AddressBytea + if err := addr.Scan(addrBytes); err != nil { + return nil, fmt.Errorf("converting address bytes: %w", err) + } + existingAccounts = append(existingAccounts, string(addr)) + } + if err := rows.Err(); err != nil { m.MetricsService.IncDBQueryError("BatchGetByIDs", "accounts", utils.GetDBErrorType(err)) - return nil, fmt.Errorf("collecting rows: %w", err) + return nil, fmt.Errorf("iterating rows: %w", err) } + duration := time.Since(start).Seconds() m.MetricsService.ObserveDBQueryDuration("BatchGetByIDs", "accounts", duration) m.MetricsService.ObserveDBBatchSize("BatchGetByIDs", "accounts", len(accountIDs)) @@ -133,17 +166,17 @@ func (m *AccountModel) BatchGetByIDs(ctx context.Context, dbTx pgx.Tx, accountID // IsAccountFeeBumpEligible checks whether an account is eligible to have its transaction fee-bumped. Channel Accounts should be // eligible because some of the transactions will have the channel accounts as the source account (i. e. create account sponsorship). func (m *AccountModel) IsAccountFeeBumpEligible(ctx context.Context, address string) (bool, error) { + // accounts.stellar_address is BYTEA, channel_accounts.public_key is VARCHAR + // Use separate EXISTS checks to avoid type mismatch in UNION const query = ` SELECT - EXISTS( - SELECT stellar_address FROM accounts WHERE stellar_address = $1 - UNION - SELECT public_key FROM channel_accounts WHERE public_key = $1 - ) + EXISTS(SELECT 1 FROM accounts WHERE stellar_address = $1) + OR + EXISTS(SELECT 1 FROM channel_accounts WHERE public_key = $2) ` var exists bool start := time.Now() - err := m.DB.GetContext(ctx, &exists, query, address) + err := m.DB.GetContext(ctx, &exists, query, types.AddressBytea(address), address) duration := time.Since(start).Seconds() m.MetricsService.ObserveDBQueryDuration("IsAccountFeeBumpEligible", "accounts", duration) if err != nil { diff --git a/internal/data/query_utils.go b/internal/data/query_utils.go index c458ba13d..d2304136a 100644 --- a/internal/data/query_utils.go +++ b/internal/data/query_utils.go @@ -65,6 +65,21 @@ func pgtypeInt2FromNullInt16(ni sql.NullInt16) pgtype.Int2 { return pgtype.Int2{Int16: ni.Int16, Valid: ni.Valid} } +// pgtypeBytesFromNullAddressBytea converts NullAddressBytea to bytes for BYTEA insert. +func pgtypeBytesFromNullAddressBytea(na types.NullAddressBytea) ([]byte, error) { + if !na.Valid { + return nil, nil + } + val, err := na.Value() + if err != nil { + return nil, fmt.Errorf("converting address to bytes: %w", err) + } + if val == nil { + return nil, nil + } + return val.([]byte), nil +} + // BuildPaginatedQuery constructs a paginated SQL query with cursor-based pagination func buildGetByAccountAddressQuery(config paginatedQueryConfig) (string, []any) { var queryBuilder strings.Builder @@ -75,8 +90,8 @@ func buildGetByAccountAddressQuery(config paginatedQueryConfig) (string, []any) queryBuilder.WriteString(fmt.Sprintf(` SELECT %s, %s.%s as cursor FROM %s - INNER JOIN %s - ON %s + INNER JOIN %s + ON %s WHERE %s.account_id = $%d`, config.Columns, config.TableName, @@ -86,7 +101,7 @@ func buildGetByAccountAddressQuery(config paginatedQueryConfig) (string, []any) config.JoinCondition, config.JoinTable, argIndex)) - args = append(args, config.AccountAddress) + args = append(args, types.AddressBytea(config.AccountAddress)) argIndex++ // Add cursor condition if provided diff --git a/internal/db/migrations/2024-04-29.1-accounts.sql b/internal/db/migrations/2024-04-29.1-accounts.sql index 5d2f81e11..7c861b598 100644 --- a/internal/db/migrations/2024-04-29.1-accounts.sql +++ b/internal/db/migrations/2024-04-29.1-accounts.sql @@ -1,7 +1,7 @@ -- +migrate Up CREATE TABLE accounts ( - stellar_address text NOT NULL, + stellar_address BYTEA NOT NULL, created_at timestamp with time zone NOT NULL DEFAULT NOW(), PRIMARY KEY (stellar_address) ); diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index c435e3c81..4adad122e 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -184,7 +184,7 @@ func (i *Indexer) processTransaction(ctx context.Context, tx ingest.LedgerTransa allParticipants = allParticipants.Union(opParticipants.Participants) } for _, stateChange := range stateChanges { - allParticipants.Add(stateChange.AccountID) + allParticipants.Add(string(stateChange.AccountID)) } // Insert transaction participants @@ -251,7 +251,7 @@ func (i *Indexer) processTransaction(ctx context.Context, tx ingest.LedgerTransa // Only store contract changes when contract token is SEP41 if stateChange.ContractType == types.ContractTypeSEP41 { contractChange := types.ContractChange{ - AccountID: stateChange.AccountID, + AccountID: string(stateChange.AccountID), OperationID: stateChange.OperationID, ContractID: stateChange.TokenID.String, LedgerNumber: tx.Ledger.LedgerSequence(), diff --git a/internal/indexer/indexer_buffer.go b/internal/indexer/indexer_buffer.go index 6594e8fa7..9cae55f09 100644 --- a/internal/indexer/indexer_buffer.go +++ b/internal/indexer/indexer_buffer.go @@ -114,7 +114,7 @@ func (b *IndexerBuffer) PushTransaction(participant string, transaction types.Tr // // Caller must hold write lock. func (b *IndexerBuffer) pushTransactionUnsafe(participant string, transaction *types.Transaction) { - txHash := transaction.Hash + txHash := transaction.Hash.String() if _, exists := b.txByHash[txHash]; !exists { b.txByHash[txHash] = transaction } @@ -382,10 +382,10 @@ func (b *IndexerBuffer) PushStateChange(transaction types.Transaction, operation defer b.mu.Unlock() b.stateChanges = append(b.stateChanges, stateChange) - b.pushTransactionUnsafe(stateChange.AccountID, &transaction) + b.pushTransactionUnsafe(string(stateChange.AccountID), &transaction) // Fee changes dont have an operation ID associated with them if stateChange.OperationID != 0 { - b.pushOperationUnsafe(stateChange.AccountID, &operation) + b.pushOperationUnsafe(string(stateChange.AccountID), &operation) } } diff --git a/internal/indexer/indexer_buffer_test.go b/internal/indexer/indexer_buffer_test.go index f2088480d..c868b5335 100644 --- a/internal/indexer/indexer_buffer_test.go +++ b/internal/indexer/indexer_buffer_test.go @@ -17,7 +17,7 @@ func buildStateChange(toID int64, reason types.StateChangeReason, accountID stri ToID: toID, StateChangeCategory: types.StateChangeCategoryBalance, StateChangeReason: &reason, - AccountID: accountID, + AccountID: types.AddressBytea(accountID), OperationID: operationID, SortKey: fmt.Sprintf("%d:%s:%s", toID, types.StateChangeCategoryBalance, accountID), } @@ -27,8 +27,8 @@ func TestIndexerBuffer_PushTransaction(t *testing.T) { t.Run("🟢 sequential pushes", func(t *testing.T) { indexerBuffer := NewIndexerBuffer() - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} - tx2 := types.Transaction{Hash: "tx_hash_2", ToID: 2} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} + tx2 := types.Transaction{Hash: "a76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48761", ToID: 2} indexerBuffer.PushTransaction("alice", tx1) indexerBuffer.PushTransaction("alice", tx2) @@ -50,8 +50,8 @@ func TestIndexerBuffer_PushTransaction(t *testing.T) { t.Run("🟢 concurrent pushes", func(t *testing.T) { indexerBuffer := NewIndexerBuffer() - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} - tx2 := types.Transaction{Hash: "tx_hash_2", ToID: 2} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} + tx2 := types.Transaction{Hash: "a76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48761", ToID: 2} wg := sync.WaitGroup{} wg.Add(4) @@ -87,8 +87,8 @@ func TestIndexerBuffer_PushOperation(t *testing.T) { t.Run("🟢 sequential pushes", func(t *testing.T) { indexerBuffer := NewIndexerBuffer() - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} - tx2 := types.Transaction{Hash: "tx_hash_2", ToID: 2} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} + tx2 := types.Transaction{Hash: "a76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48761", ToID: 2} op1 := types.Operation{ID: 1} op2 := types.Operation{ID: 2} @@ -111,8 +111,8 @@ func TestIndexerBuffer_PushOperation(t *testing.T) { t.Run("🟢 concurrent pushes", func(t *testing.T) { indexerBuffer := NewIndexerBuffer() - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} - tx2 := types.Transaction{Hash: "tx_hash_2", ToID: 2} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} + tx2 := types.Transaction{Hash: "a76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48761", ToID: 2} op1 := types.Operation{ID: 1} op2 := types.Operation{ID: 2} @@ -147,7 +147,7 @@ func TestIndexerBuffer_PushStateChange(t *testing.T) { t.Run("🟢 sequential pushes", func(t *testing.T) { indexerBuffer := NewIndexerBuffer() - tx := types.Transaction{Hash: "test_tx_hash", ToID: 1} + tx := types.Transaction{Hash: "c76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48763", ToID: 1} op := types.Operation{ID: 1} sc1 := types.StateChange{ToID: 1, StateChangeOrder: 1} @@ -165,7 +165,7 @@ func TestIndexerBuffer_PushStateChange(t *testing.T) { t.Run("🟢 concurrent pushes", func(t *testing.T) { indexerBuffer := NewIndexerBuffer() - tx := types.Transaction{Hash: "test_tx_hash", ToID: 1} + tx := types.Transaction{Hash: "c76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48763", ToID: 1} op := types.Operation{ID: 1} sc1 := types.StateChange{ToID: 1, StateChangeOrder: 1} @@ -196,8 +196,8 @@ func TestIndexerBuffer_PushStateChange(t *testing.T) { t.Run("🟢 with operations and transactions", func(t *testing.T) { indexerBuffer := NewIndexerBuffer() - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} - tx2 := types.Transaction{Hash: "tx_hash_2", ToID: 2} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} + tx2 := types.Transaction{Hash: "a76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48761", ToID: 2} op1 := types.Operation{ID: 3} op2 := types.Operation{ID: 4} op3 := types.Operation{ID: 5} @@ -239,8 +239,8 @@ func TestIndexerBuffer_GetNumberOfTransactions(t *testing.T) { assert.Equal(t, 0, indexerBuffer.GetNumberOfTransactions()) - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} - tx2 := types.Transaction{Hash: "tx_hash_2", ToID: 2} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} + tx2 := types.Transaction{Hash: "a76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48761", ToID: 2} indexerBuffer.PushTransaction("alice", tx1) assert.Equal(t, 1, indexerBuffer.GetNumberOfTransactions()) @@ -258,8 +258,8 @@ func TestIndexerBuffer_GetAllTransactions(t *testing.T) { t.Run("🟢 returns all unique transactions", func(t *testing.T) { indexerBuffer := NewIndexerBuffer() - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1, LedgerNumber: 100} - tx2 := types.Transaction{Hash: "tx_hash_2", ToID: 2, LedgerNumber: 101} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1, LedgerNumber: 100} + tx2 := types.Transaction{Hash: "a76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48761", ToID: 2, LedgerNumber: 101} indexerBuffer.PushTransaction("alice", tx1) indexerBuffer.PushTransaction("bob", tx2) @@ -275,8 +275,8 @@ func TestIndexerBuffer_GetAllTransactionsParticipants(t *testing.T) { t.Run("🟢 returns correct participants mapping", func(t *testing.T) { indexerBuffer := NewIndexerBuffer() - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} - tx2 := types.Transaction{Hash: "tx_hash_2", ToID: 2} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} + tx2 := types.Transaction{Hash: "a76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48761", ToID: 2} indexerBuffer.PushTransaction("alice", tx1) indexerBuffer.PushTransaction("bob", tx1) @@ -292,7 +292,7 @@ func TestIndexerBuffer_GetAllOperations(t *testing.T) { t.Run("🟢 returns all unique operations", func(t *testing.T) { indexerBuffer := NewIndexerBuffer() - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} op1 := types.Operation{ID: 1} op2 := types.Operation{ID: 2} @@ -310,7 +310,7 @@ func TestIndexerBuffer_GetAllOperationsParticipants(t *testing.T) { t.Run("🟢 returns correct participants mapping", func(t *testing.T) { indexerBuffer := NewIndexerBuffer() - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} op1 := types.Operation{ID: 1} op2 := types.Operation{ID: 2} @@ -328,7 +328,7 @@ func TestIndexerBuffer_GetAllStateChanges(t *testing.T) { t.Run("🟢 returns all state changes in order", func(t *testing.T) { indexerBuffer := NewIndexerBuffer() - tx := types.Transaction{Hash: "test_tx_hash", ToID: 1} + tx := types.Transaction{Hash: "c76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48763", ToID: 1} op := types.Operation{ID: 1} sc1 := types.StateChange{ToID: 1, StateChangeOrder: 1, AccountID: "alice"} @@ -354,8 +354,8 @@ func TestIndexerBuffer_GetAllParticipants(t *testing.T) { t.Run("🟢 collects participants from transactions", func(t *testing.T) { indexerBuffer := NewIndexerBuffer() - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} - tx2 := types.Transaction{Hash: "tx_hash_2", ToID: 2} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} + tx2 := types.Transaction{Hash: "a76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48761", ToID: 2} indexerBuffer.PushTransaction("alice", tx1) indexerBuffer.PushTransaction("bob", tx2) @@ -368,7 +368,7 @@ func TestIndexerBuffer_GetAllParticipants(t *testing.T) { t.Run("🟢 collects participants from operations", func(t *testing.T) { indexerBuffer := NewIndexerBuffer() - tx := types.Transaction{Hash: "tx_hash_1", ToID: 1} + tx := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} op1 := types.Operation{ID: 1} op2 := types.Operation{ID: 2} @@ -383,7 +383,7 @@ func TestIndexerBuffer_GetAllParticipants(t *testing.T) { t.Run("🟢 collects participants from state changes", func(t *testing.T) { indexerBuffer := NewIndexerBuffer() - tx := types.Transaction{Hash: "tx_hash_1", ToID: 1} + tx := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} op := types.Operation{ID: 1} sc1 := types.StateChange{ToID: 1, StateChangeOrder: 1, AccountID: "alice", OperationID: 1} @@ -401,7 +401,7 @@ func TestIndexerBuffer_GetAllParticipants(t *testing.T) { t.Run("🟢 collects unique participants from all sources", func(t *testing.T) { indexerBuffer := NewIndexerBuffer() - tx := types.Transaction{Hash: "tx_hash_1", ToID: 1} + tx := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} op := types.Operation{ID: 1} sc := types.StateChange{ToID: 1, StateChangeOrder: 1, AccountID: "dave", OperationID: 1} @@ -418,7 +418,7 @@ func TestIndexerBuffer_GetAllParticipants(t *testing.T) { t.Run("🟢 ignores empty participants", func(t *testing.T) { indexerBuffer := NewIndexerBuffer() - tx := types.Transaction{Hash: "tx_hash_1", ToID: 1} + tx := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} indexerBuffer.PushTransaction("", tx) // empty participant indexerBuffer.PushTransaction("alice", tx) @@ -441,8 +441,8 @@ func TestIndexerBuffer_Merge(t *testing.T) { buffer1 := NewIndexerBuffer() buffer2 := NewIndexerBuffer() - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} - tx2 := types.Transaction{Hash: "tx_hash_2", ToID: 2} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} + tx2 := types.Transaction{Hash: "a76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48761", ToID: 2} buffer1.PushTransaction("alice", tx1) buffer2.PushTransaction("bob", tx2) @@ -464,7 +464,7 @@ func TestIndexerBuffer_Merge(t *testing.T) { buffer1 := NewIndexerBuffer() buffer2 := NewIndexerBuffer() - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} op1 := types.Operation{ID: 1} op2 := types.Operation{ID: 2} @@ -488,7 +488,7 @@ func TestIndexerBuffer_Merge(t *testing.T) { buffer1 := NewIndexerBuffer() buffer2 := NewIndexerBuffer() - tx := types.Transaction{Hash: "test_tx_hash", ToID: 1} + tx := types.Transaction{Hash: "c76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48763", ToID: 1} op := types.Operation{ID: 1} sc1 := types.StateChange{ToID: 1, StateChangeOrder: 1, AccountID: "alice"} @@ -510,8 +510,8 @@ func TestIndexerBuffer_Merge(t *testing.T) { buffer1 := NewIndexerBuffer() buffer2 := NewIndexerBuffer() - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} - tx2 := types.Transaction{Hash: "tx_hash_2", ToID: 2} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} + tx2 := types.Transaction{Hash: "a76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48761", ToID: 2} op1 := types.Operation{ID: 1} // Buffer1 has tx1 with alice @@ -543,7 +543,7 @@ func TestIndexerBuffer_Merge(t *testing.T) { buffer1 := NewIndexerBuffer() buffer2 := NewIndexerBuffer() - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} op1 := types.Operation{ID: 1} sc1 := types.StateChange{ToID: 1, StateChangeOrder: 1, AccountID: "alice"} @@ -562,7 +562,7 @@ func TestIndexerBuffer_Merge(t *testing.T) { buffer1 := NewIndexerBuffer() buffer2 := NewIndexerBuffer() - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} buffer1.PushTransaction("alice", tx1) buffer1.Merge(buffer2) @@ -575,9 +575,9 @@ func TestIndexerBuffer_Merge(t *testing.T) { buffer2 := NewIndexerBuffer() buffer3 := NewIndexerBuffer() - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} - tx2 := types.Transaction{Hash: "tx_hash_2", ToID: 2} - tx3 := types.Transaction{Hash: "tx_hash_3", ToID: 3} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} + tx2 := types.Transaction{Hash: "a76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48761", ToID: 2} + tx3 := types.Transaction{Hash: "b76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48762", ToID: 3} buffer1.PushTransaction("alice", tx1) buffer2.PushTransaction("bob", tx2) @@ -606,8 +606,8 @@ func TestIndexerBuffer_Merge(t *testing.T) { buffer1 := NewIndexerBuffer() buffer2 := NewIndexerBuffer() - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} - tx2 := types.Transaction{Hash: "tx_hash_2", ToID: 2} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} + tx2 := types.Transaction{Hash: "a76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48761", ToID: 2} op1 := types.Operation{ID: 1} op2 := types.Operation{ID: 2} sc1 := types.StateChange{ToID: 1, StateChangeOrder: 1, AccountID: "alice", OperationID: 1} @@ -653,8 +653,8 @@ func TestIndexerBuffer_Merge(t *testing.T) { buffer1 := NewIndexerBuffer() buffer2 := NewIndexerBuffer() - tx1 := types.Transaction{Hash: "tx_hash_1", ToID: 1} - tx2 := types.Transaction{Hash: "tx_hash_2", ToID: 2} + tx1 := types.Transaction{Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1} + tx2 := types.Transaction{Hash: "a76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48761", ToID: 2} buffer1.PushTransaction("alice", tx1) buffer1.PushTransaction("bob", tx1) diff --git a/internal/indexer/indexer_test.go b/internal/indexer/indexer_test.go index e54c312de..7725f5b8e 100644 --- a/internal/indexer/indexer_test.go +++ b/internal/indexer/indexer_test.go @@ -647,19 +647,19 @@ func TestIndexer_ProcessLedgerTransactions(t *testing.T) { require.Len(t, stateChanges, 3, "should have 3 state changes") // Verify first state change - assert.Equal(t, "alice", stateChanges[0].AccountID) + assert.Equal(t, "alice", stateChanges[0].AccountID.String()) assert.Equal(t, int64(1), stateChanges[0].ToID) assert.Equal(t, int64(1), stateChanges[0].OperationID) assert.Equal(t, int64(1), stateChanges[0].StateChangeOrder, "first state change should have order 1") // Verify second state change - assert.Equal(t, "alice", stateChanges[1].AccountID) + assert.Equal(t, "alice", stateChanges[1].AccountID.String()) assert.Equal(t, int64(2), stateChanges[1].ToID) assert.Equal(t, int64(1), stateChanges[1].OperationID) assert.Equal(t, int64(2), stateChanges[1].StateChangeOrder, "second state change should have order 2") // Verify third state change - assert.Equal(t, "alice", stateChanges[2].AccountID) + assert.Equal(t, "alice", stateChanges[2].AccountID.String()) assert.Equal(t, int64(3), stateChanges[2].ToID) assert.Equal(t, int64(1), stateChanges[2].OperationID) assert.Equal(t, int64(3), stateChanges[2].StateChangeOrder, "third state change should have order 3") @@ -728,7 +728,7 @@ func TestIndexer_getTransactionStateChanges(t *testing.T) { foundBob := false foundCharlie := false for _, sc := range stateChanges { - switch sc.AccountID { + switch sc.AccountID.String() { case "alice": assert.Equal(t, int64(1), sc.ToID) assert.Equal(t, int64(1), sc.OperationID) @@ -914,7 +914,7 @@ func TestIndexer_getTransactionStateChanges(t *testing.T) { // Verify it's the correct state change sc := stateChanges[0] - assert.Equal(t, "alice", sc.AccountID) + assert.Equal(t, "alice", sc.AccountID.String()) assert.Equal(t, int64(1), sc.ToID) assert.Equal(t, int64(1), sc.OperationID) diff --git a/internal/indexer/processors/contracts/test_utils.go b/internal/indexer/processors/contracts/test_utils.go index 66e68d0e7..d6ca9c03e 100644 --- a/internal/indexer/processors/contracts/test_utils.go +++ b/internal/indexer/processors/contracts/test_utils.go @@ -625,7 +625,7 @@ func createInvalidBalanceMapTx(contractAccount, admin string, asset xdr.Asset, i func assertContractEvent(t *testing.T, change types.StateChange, reason types.StateChangeReason, expectedAccount string, expectedContractID string) { t.Helper() require.Equal(t, types.StateChangeCategoryBalanceAuthorization, change.StateChangeCategory) - require.Equal(t, expectedAccount, change.AccountID) + require.Equal(t, expectedAccount, change.AccountID.String()) if expectedContractID != "" { require.NotNil(t, change.TokenID) require.Equal(t, expectedContractID, change.TokenID.String) diff --git a/internal/indexer/processors/contracts_test_utils.go b/internal/indexer/processors/contracts_test_utils.go index d544c3ef7..db4f284f7 100644 --- a/internal/indexer/processors/contracts_test_utils.go +++ b/internal/indexer/processors/contracts_test_utils.go @@ -170,11 +170,11 @@ func assertStateChangesElementsMatch(t *testing.T, want []types.StateChange, got wantMap := make(map[string]types.StateChange) for _, w := range want { - wantMap[fmt.Sprintf("%d-%s-%s", w.ToID, w.AccountID, w.DeployerAccountID.String)] = w + wantMap[fmt.Sprintf("%d-%s-%s", w.ToID, w.AccountID, w.DeployerAccountID.String())] = w } for _, g := range got { - key := fmt.Sprintf("%d-%s-%s", g.ToID, g.AccountID, g.DeployerAccountID.String) + key := fmt.Sprintf("%d-%s-%s", g.ToID, g.AccountID, g.DeployerAccountID.String()) if _, ok := wantMap[key]; !ok { assert.Fail(t, "state change not found", "state change id: %s", key) } diff --git a/internal/indexer/processors/effects_test.go b/internal/indexer/processors/effects_test.go index 623ee254e..335b79033 100644 --- a/internal/indexer/processors/effects_test.go +++ b/internal/indexer/processors/effects_test.go @@ -85,12 +85,12 @@ func TestEffects_ProcessTransaction(t *testing.T) { switch *change.StateChangeReason { case types.StateChangeReasonUpdate: assert.True(t, change.SignerAccountID.Valid) - assert.Equal(t, "GC4XF7RE3R4P77GY5XNGICM56IOKUURWAAANPXHFC7G5H6FCNQVVH3OH", change.SignerAccountID.String) + assert.Equal(t, "GC4XF7RE3R4P77GY5XNGICM56IOKUURWAAANPXHFC7G5H6FCNQVVH3OH", change.SignerAccountID.String()) assert.Equal(t, int16(1), change.SignerWeightOld.Int16) assert.Equal(t, int16(3), change.SignerWeightNew.Int16) case types.StateChangeReasonAdd: assert.True(t, change.SignerAccountID.Valid) - assert.Equal(t, "GAQHWQYBBW272OOXNQMMLCA5WY2XAZPODGB7Q3S5OKKIXVESKO55ZQ7C", change.SignerAccountID.String) + assert.Equal(t, "GAQHWQYBBW272OOXNQMMLCA5WY2XAZPODGB7Q3S5OKKIXVESKO55ZQ7C", change.SignerAccountID.String()) assert.False(t, change.SignerWeightOld.Valid) // New signer has no old weight assert.Equal(t, int16(2), change.SignerWeightNew.Int16) } @@ -295,45 +295,45 @@ func TestEffects_ProcessTransaction(t *testing.T) { // TxHash removed - lookup via to_id instead assert.Equal(t, types.StateChangeCategoryReserves, changes[1].StateChangeCategory) assert.Equal(t, types.StateChangeReasonUnsponsor, *changes[1].StateChangeReason) - assert.Equal(t, "GACMZD5VJXTRLKVET72CETCYKELPNCOTTBDC6DHFEUPLG5DHEK534JQX", changes[1].AccountID) - assert.Equal(t, "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", changes[1].SponsoredAccountID.String) + assert.Equal(t, "GACMZD5VJXTRLKVET72CETCYKELPNCOTTBDC6DHFEUPLG5DHEK534JQX", changes[1].AccountID.String()) + assert.Equal(t, "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", changes[1].SponsoredAccountID.String()) assert.Equal(t, types.StateChangeCategoryReserves, changes[2].StateChangeCategory) assert.Equal(t, types.StateChangeReasonUnsponsor, *changes[2].StateChangeReason) - assert.Equal(t, "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", changes[2].AccountID) - assert.Equal(t, "GACMZD5VJXTRLKVET72CETCYKELPNCOTTBDC6DHFEUPLG5DHEK534JQX", changes[2].SponsorAccountID.String) + assert.Equal(t, "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", changes[2].AccountID.String()) + assert.Equal(t, "GACMZD5VJXTRLKVET72CETCYKELPNCOTTBDC6DHFEUPLG5DHEK534JQX", changes[2].SponsorAccountID.String()) // Updating sponsorship creates 4 state changes - one for the new sponsor, one for the former sponsor, and two for the target account assert.Equal(t, types.StateChangeCategoryReserves, changes[3].StateChangeCategory) assert.Equal(t, types.StateChangeReasonSponsor, *changes[3].StateChangeReason) - assert.Equal(t, "GACMZD5VJXTRLKVET72CETCYKELPNCOTTBDC6DHFEUPLG5DHEK534JQX", changes[3].AccountID) - assert.Equal(t, "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", changes[3].SponsoredAccountID.String) + assert.Equal(t, "GACMZD5VJXTRLKVET72CETCYKELPNCOTTBDC6DHFEUPLG5DHEK534JQX", changes[3].AccountID.String()) + assert.Equal(t, "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", changes[3].SponsoredAccountID.String()) assert.Equal(t, types.StateChangeCategoryReserves, changes[4].StateChangeCategory) assert.Equal(t, types.StateChangeReasonUnsponsor, *changes[4].StateChangeReason) - assert.Equal(t, "GAHK7EEG2WWHVKDNT4CEQFZGKF2LGDSW2IVM4S5DP42RBW3K6BTODB4A", changes[4].AccountID) - assert.Equal(t, "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", changes[4].SponsoredAccountID.String) + assert.Equal(t, "GAHK7EEG2WWHVKDNT4CEQFZGKF2LGDSW2IVM4S5DP42RBW3K6BTODB4A", changes[4].AccountID.String()) + assert.Equal(t, "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", changes[4].SponsoredAccountID.String()) assert.Equal(t, types.StateChangeCategoryReserves, changes[5].StateChangeCategory) assert.Equal(t, types.StateChangeReasonSponsor, *changes[5].StateChangeReason) - assert.Equal(t, "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", changes[5].AccountID) - assert.Equal(t, "GACMZD5VJXTRLKVET72CETCYKELPNCOTTBDC6DHFEUPLG5DHEK534JQX", changes[5].SponsorAccountID.String) + assert.Equal(t, "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", changes[5].AccountID.String()) + assert.Equal(t, "GACMZD5VJXTRLKVET72CETCYKELPNCOTTBDC6DHFEUPLG5DHEK534JQX", changes[5].SponsorAccountID.String()) assert.Equal(t, types.StateChangeCategoryReserves, changes[6].StateChangeCategory) assert.Equal(t, types.StateChangeReasonUnsponsor, *changes[6].StateChangeReason) - assert.Equal(t, "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", changes[6].AccountID) - assert.Equal(t, "GAHK7EEG2WWHVKDNT4CEQFZGKF2LGDSW2IVM4S5DP42RBW3K6BTODB4A", changes[6].SponsorAccountID.String) + assert.Equal(t, "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", changes[6].AccountID.String()) + assert.Equal(t, "GAHK7EEG2WWHVKDNT4CEQFZGKF2LGDSW2IVM4S5DP42RBW3K6BTODB4A", changes[6].SponsorAccountID.String()) // Sponsorship created creates two state changes - one for the sponsor and one for the target account assert.Equal(t, types.StateChangeCategoryReserves, changes[7].StateChangeCategory) assert.Equal(t, types.StateChangeReasonSponsor, *changes[7].StateChangeReason) - assert.Equal(t, "GAHK7EEG2WWHVKDNT4CEQFZGKF2LGDSW2IVM4S5DP42RBW3K6BTODB4A", changes[7].AccountID) - assert.Equal(t, "GCQZP3IU7XU6EJ63JZXKCQOYT2RNXN3HB5CNHENNUEUHSMA4VUJJJSEN", changes[7].SponsoredAccountID.String) + assert.Equal(t, "GAHK7EEG2WWHVKDNT4CEQFZGKF2LGDSW2IVM4S5DP42RBW3K6BTODB4A", changes[7].AccountID.String()) + assert.Equal(t, "GCQZP3IU7XU6EJ63JZXKCQOYT2RNXN3HB5CNHENNUEUHSMA4VUJJJSEN", changes[7].SponsoredAccountID.String()) assert.Equal(t, types.StateChangeCategoryReserves, changes[8].StateChangeCategory) assert.Equal(t, types.StateChangeReasonSponsor, *changes[8].StateChangeReason) - assert.Equal(t, "GCQZP3IU7XU6EJ63JZXKCQOYT2RNXN3HB5CNHENNUEUHSMA4VUJJJSEN", changes[8].AccountID) - assert.Equal(t, "GAHK7EEG2WWHVKDNT4CEQFZGKF2LGDSW2IVM4S5DP42RBW3K6BTODB4A", changes[8].SponsorAccountID.String) + assert.Equal(t, "GCQZP3IU7XU6EJ63JZXKCQOYT2RNXN3HB5CNHENNUEUHSMA4VUJJJSEN", changes[8].AccountID.String()) + assert.Equal(t, "GAHK7EEG2WWHVKDNT4CEQFZGKF2LGDSW2IVM4S5DP42RBW3K6BTODB4A", changes[8].SponsorAccountID.String()) }) t.Run("ChangeTrust - trustline created", func(t *testing.T) { envelopeXDR := "AAAAAgAAAAAf1miSBZ7jc0TxIHULMUqdj+dibtkh1JEEwITVtQ05ZgAAAGQAB1eLAAAAAwAAAAEAAAAAAAAAAAAAAABowwQqAAAAAAAAAAEAAAAAAAAABgAAAAFURVNUAAAAAFrnJwiWP46hSSjcYc6wY93h556Qpe47SA8bIQGXMJTlf/////////8AAAAAAAAAAbUNOWYAAABAzWelNCrF4Q+iSKX30xHrBm76FMa2h89pPauijrWAVlcj/swEyYZqjU94SYU+8XEWUuvg2rpjCIHGPHHyzSXlAw==" diff --git a/internal/indexer/processors/processors_test_utils.go b/internal/indexer/processors/processors_test_utils.go index 9f98f7a37..a7e7c014b 100644 --- a/internal/indexer/processors/processors_test_utils.go +++ b/internal/indexer/processors/processors_test_utils.go @@ -806,7 +806,7 @@ func requireEventCount(t *testing.T, changes []types.StateChange, expectedCount func assertStateChangeBase(t *testing.T, change types.StateChange, category types.StateChangeCategory, expectedAccount string, expectedAmount string, expectedToken string) { t.Helper() require.Equal(t, category, change.StateChangeCategory) - require.Equal(t, expectedAccount, change.AccountID) + require.Equal(t, expectedAccount, change.AccountID.String()) if expectedAmount != "" { require.Equal(t, utils.SQLNullString(expectedAmount), change.Amount) } diff --git a/internal/indexer/processors/state_change_builder.go b/internal/indexer/processors/state_change_builder.go index b50436e71..a6b6e26d1 100644 --- a/internal/indexer/processors/state_change_builder.go +++ b/internal/indexer/processors/state_change_builder.go @@ -76,13 +76,13 @@ func (b *StateChangeBuilder) WithFlags(flags []string) *StateChangeBuilder { // WithAccount sets the account ID func (b *StateChangeBuilder) WithAccount(accountID string) *StateChangeBuilder { - b.base.AccountID = accountID + b.base.AccountID = types.AddressBytea(accountID) return b } // WithSigner sets the signer account ID and the weights directly func (b *StateChangeBuilder) WithSigner(signer string, oldWeight, newWeight *int16) *StateChangeBuilder { - b.base.SignerAccountID = utils.SQLNullString(signer) + b.base.SignerAccountID = utils.NullAddressBytea(signer) if oldWeight != nil { b.base.SignerWeightOld = sql.NullInt16{Int16: *oldWeight, Valid: true} } @@ -94,19 +94,19 @@ func (b *StateChangeBuilder) WithSigner(signer string, oldWeight, newWeight *int // WithDeployer sets the deployer account ID, usually associated with a contract deployment. func (b *StateChangeBuilder) WithDeployer(deployer string) *StateChangeBuilder { - b.base.DeployerAccountID = utils.SQLNullString(deployer) + b.base.DeployerAccountID = utils.NullAddressBytea(deployer) return b } // WithFunder sets the funder account ID func (b *StateChangeBuilder) WithFunder(funder string) *StateChangeBuilder { - b.base.FunderAccountID = utils.SQLNullString(funder) + b.base.FunderAccountID = utils.NullAddressBytea(funder) return b } // WithSponsor sets the sponsor func (b *StateChangeBuilder) WithSponsor(sponsor string) *StateChangeBuilder { - b.base.SponsorAccountID = utils.SQLNullString(sponsor) + b.base.SponsorAccountID = utils.NullAddressBytea(sponsor) return b } @@ -136,7 +136,7 @@ func (b *StateChangeBuilder) WithTokenType(tokenType types.ContractType) *StateC // WithSponsoredAccountID sets the sponsored account ID for a sponsorship state change func (b *StateChangeBuilder) WithSponsoredAccountID(sponsoredAccountID string) *StateChangeBuilder { - b.base.SponsoredAccountID = utils.SQLNullString(sponsoredAccountID) + b.base.SponsoredAccountID = utils.NullAddressBytea(sponsoredAccountID) return b } @@ -192,10 +192,10 @@ func (b *StateChangeBuilder) generateSortKey() string { b.base.AccountID, b.base.TokenID.String, b.base.Amount.String, - b.base.SignerAccountID.String, - b.base.SpenderAccountID.String, - b.base.SponsoredAccountID.String, - b.base.SponsorAccountID.String, + b.base.SignerAccountID.String(), + b.base.SpenderAccountID.String(), + b.base.SponsoredAccountID.String(), + b.base.SponsorAccountID.String(), b.base.SignerWeightOld.Int16, b.base.SignerWeightNew.Int16, b.base.ThresholdOld.Int16, diff --git a/internal/indexer/processors/token_transfer_test.go b/internal/indexer/processors/token_transfer_test.go index c1e85cac7..35cb46b88 100644 --- a/internal/indexer/processors/token_transfer_test.go +++ b/internal/indexer/processors/token_transfer_test.go @@ -75,7 +75,7 @@ func TestTokenTransferProcessor_Process(t *testing.T) { assertFeeEvent(t, changes[0], "100") assertStateChangeBase(t, changes[1], types.StateChangeCategoryAccount, accountB.ToAccountId().Address(), "", "") require.Equal(t, types.StateChangeReasonCreate, *changes[1].StateChangeReason) - require.Equal(t, accountA.ToAccountId().Address(), changes[1].FunderAccountID.String) + require.Equal(t, accountA.ToAccountId().Address(), changes[1].FunderAccountID.String()) assertDebitEvent(t, changes[2], accountA.ToAccountId().Address(), "1000000000", nativeContractAddress) assertCreditEvent(t, changes[3], accountB.ToAccountId().Address(), "1000000000", nativeContractAddress) }) diff --git a/internal/indexer/processors/utils.go b/internal/indexer/processors/utils.go index 4176d098d..c4cfb614d 100644 --- a/internal/indexer/processors/utils.go +++ b/internal/indexer/processors/utils.go @@ -309,7 +309,7 @@ func ConvertTransaction(transaction *ingest.LedgerTransaction, skipTxMeta bool, return &types.Transaction{ ToID: transactionID, - Hash: transaction.Hash.HexString(), + Hash: types.HashBytea(transaction.Hash.HexString()), LedgerCreatedAt: transaction.Ledger.ClosedAt(), EnvelopeXDR: envelopeXDR, FeeCharged: feeCharged, @@ -328,7 +328,7 @@ func ConvertOperation( opIndex uint32, opResults []xdr.OperationResult, ) (*types.Operation, error) { - xdrOpStr, err := xdr.MarshalBase64(op) + xdrBytes, err := op.MarshalBinary() if err != nil { return nil, fmt.Errorf("marshalling operation %d: %w", opID, err) } @@ -350,7 +350,7 @@ func ConvertOperation( return &types.Operation{ ID: opID, OperationType: types.OperationTypeFromXDR(op.Body.Type), - OperationXDR: xdrOpStr, + OperationXDR: types.XDRBytea(xdrBytes), ResultCode: resultCode, Successful: successful, LedgerCreatedAt: transaction.Ledger.ClosedAt(), diff --git a/internal/indexer/processors/utils_test.go b/internal/indexer/processors/utils_test.go index 0f68c1292..483e1e970 100644 --- a/internal/indexer/processors/utils_test.go +++ b/internal/indexer/processors/utils_test.go @@ -1,6 +1,7 @@ package processors import ( + "encoding/base64" "testing" "time" @@ -101,10 +102,14 @@ func Test_ConvertOperation(t *testing.T) { gotDataOp, err := ConvertOperation(&ingestTx, &op, opID, opIndex, opResults) require.NoError(t, err) + // Decode expected base64 XDR to raw bytes for comparison + expectedXDRBytes, err := base64.StdEncoding.DecodeString(opXDRStr) + require.NoError(t, err) + wantDataOp := &types.Operation{ ID: opID, OperationType: types.OperationTypeFromXDR(op.Body.Type), - OperationXDR: opXDRStr, + OperationXDR: types.XDRBytea(expectedXDRBytes), ResultCode: OpSuccess, Successful: true, LedgerCreatedAt: time.Date(2025, time.June, 19, 0, 3, 16, 0, time.UTC), diff --git a/internal/indexer/types/types.go b/internal/indexer/types/types.go index d1458c486..ae6d2344b 100644 --- a/internal/indexer/types/types.go +++ b/internal/indexer/types/types.go @@ -34,13 +34,172 @@ package types import ( "database/sql" "database/sql/driver" + "encoding/base64" + "encoding/hex" "encoding/json" "fmt" "time" + "github.com/stellar/go-stellar-sdk/strkey" "github.com/stellar/go-stellar-sdk/xdr" ) +// AddressBytea represents a Stellar address stored as BYTEA in the database. +// Storage format: 33 bytes (1 version byte + 32 raw key bytes) +// Go representation: StrKey string (G.../C...) +type AddressBytea string + +// Scan implements sql.Scanner - converts BYTEA (33 bytes) to StrKey string +func (a *AddressBytea) Scan(value any) error { + if value == nil { + *a = "" + return nil + } + bytes, ok := value.([]byte) + if !ok { + return fmt.Errorf("expected []byte, got %T", value) + } + if len(bytes) != 33 { + return fmt.Errorf("expected 33 bytes, got %d", len(bytes)) + } + versionByte := strkey.VersionByte(bytes[0]) + rawKey := bytes[1:33] + encoded, err := strkey.Encode(versionByte, rawKey) + if err != nil { + return fmt.Errorf("encoding stellar address: %w", err) + } + *a = AddressBytea(encoded) + return nil +} + +// Value implements driver.Valuer - converts StrKey string to 33-byte []byte +func (a AddressBytea) Value() (driver.Value, error) { + if a == "" { + return nil, nil + } + versionByte, rawBytes, err := strkey.DecodeAny(string(a)) + if err != nil { + return nil, fmt.Errorf("decoding stellar address %s: %w", a, err) + } + result := make([]byte, 33) + result[0] = byte(versionByte) + copy(result[1:], rawBytes) + return result, nil +} + +// String returns the Stellar address as a string. +func (a AddressBytea) String() string { + return string(a) +} + +// NullAddressBytea represents a nullable Stellar address stored as BYTEA in the database. +// Similar to sql.NullString but handles BYTEA encoding/decoding for Stellar addresses. +type NullAddressBytea struct { + AddressBytea AddressBytea // The Stellar address (G.../C...) + Valid bool // Valid is true if AddressBytea is not NULL +} + +// Scan implements sql.Scanner - converts nullable BYTEA (33 bytes) to StrKey string +func (n *NullAddressBytea) Scan(value any) error { + if value == nil { + n.AddressBytea, n.Valid = "", false + return nil + } + if err := n.AddressBytea.Scan(value); err != nil { + return err + } + n.Valid = true + return nil +} + +// Value implements driver.Valuer - converts StrKey string to 33-byte []byte or nil +func (n NullAddressBytea) Value() (driver.Value, error) { + if !n.Valid { + return nil, nil + } + return n.AddressBytea.Value() +} + +// String returns the Stellar address as a string (convenience accessor). +func (n NullAddressBytea) String() string { + return string(n.AddressBytea) +} + +// HashBytea represents a transaction hash stored as BYTEA in the database. +// Storage format: 32 bytes (raw SHA-256 hash) +// Go representation: hex string (64 characters) +type HashBytea string + +// Scan implements sql.Scanner - converts BYTEA (32 bytes) to hex string +func (h *HashBytea) Scan(value any) error { + if value == nil { + *h = "" + return nil + } + bytes, ok := value.([]byte) + if !ok { + return fmt.Errorf("expected []byte, got %T", value) + } + if len(bytes) != 32 { + return fmt.Errorf("expected 32 bytes, got %d", len(bytes)) + } + *h = HashBytea(hex.EncodeToString(bytes)) + return nil +} + +// Value implements driver.Valuer - converts hex string to 32-byte []byte +func (h HashBytea) Value() (driver.Value, error) { + if h == "" { + return nil, nil + } + bytes, err := hex.DecodeString(string(h)) + if err != nil { + return nil, fmt.Errorf("decoding hex hash %s: %w", h, err) + } + if len(bytes) != 32 { + return nil, fmt.Errorf("invalid hash length: expected 32 bytes, got %d", len(bytes)) + } + return bytes, nil +} + +// String returns the hash as a hex string. +func (h HashBytea) String() string { + return string(h) +} + +// XDRBytea represents XDR data stored as BYTEA in the database. +// Storage format: raw XDR bytes (variable length) +// Go representation: raw bytes internally, base64 string via String() +type XDRBytea []byte + +// Scan implements sql.Scanner - reads raw bytes from BYTEA column +func (x *XDRBytea) Scan(value any) error { + if value == nil { + *x = nil + return nil + } + bytes, ok := value.([]byte) + if !ok { + return fmt.Errorf("expected []byte, got %T", value) + } + *x = make([]byte, len(bytes)) + copy(*x, bytes) + return nil +} + +// Value implements driver.Valuer - returns raw bytes for BYTEA storage +func (x XDRBytea) Value() (driver.Value, error) { + if len(x) == 0 { + return nil, nil + } + return []byte(x), nil +} + +// String returns the XDR as a base64 string. +func (x XDRBytea) String() string { + return base64.StdEncoding.EncodeToString(x) +} + type ContractType string const ( @@ -118,8 +277,8 @@ const ( ) type Account struct { - StellarAddress string `json:"address,omitempty" db:"stellar_address"` - CreatedAt time.Time `json:"createdAt,omitempty" db:"created_at"` + StellarAddress AddressBytea `json:"address,omitempty" db:"stellar_address"` + CreatedAt time.Time `json:"createdAt,omitempty" db:"created_at"` } type AccountWithToID struct { @@ -133,7 +292,7 @@ type AccountWithOperationID struct { } type Transaction struct { - Hash string `json:"hash,omitempty" db:"hash"` + Hash HashBytea `json:"hash,omitempty" db:"hash"` ToID int64 `json:"toId,omitempty" db:"to_id"` EnvelopeXDR *string `json:"envelopeXdr,omitempty" db:"envelope_xdr"` FeeCharged int64 `json:"feeCharged,omitempty" db:"fee_charged"` @@ -247,7 +406,7 @@ type Operation struct { // The parent transaction's to_id can be derived: ID &^ 0xFFF ID int64 `json:"id,omitempty" db:"id"` OperationType OperationType `json:"operationType,omitempty" db:"operation_type"` - OperationXDR string `json:"operationXdr,omitempty" db:"operation_xdr"` + OperationXDR XDRBytea `json:"operationXdr,omitempty" db:"operation_xdr"` ResultCode string `json:"resultCode,omitempty" db:"result_code"` Successful bool `json:"successful,omitempty" db:"successful"` LedgerNumber uint32 `json:"ledgerNumber,omitempty" db:"ledger_number"` @@ -405,14 +564,16 @@ type StateChange struct { LedgerNumber uint32 `json:"ledgerNumber,omitempty" db:"ledger_number"` // Nullable string fields: - TokenID sql.NullString `json:"tokenId,omitempty" db:"token_id"` - Amount sql.NullString `json:"amount,omitempty" db:"amount"` - SignerAccountID sql.NullString `json:"signerAccountId,omitempty" db:"signer_account_id"` - SpenderAccountID sql.NullString `json:"spenderAccountId,omitempty" db:"spender_account_id"` - SponsoredAccountID sql.NullString `json:"sponsoredAccountId,omitempty" db:"sponsored_account_id"` - SponsorAccountID sql.NullString `json:"sponsorAccountId,omitempty" db:"sponsor_account_id"` - DeployerAccountID sql.NullString `json:"deployerAccountId,omitempty" db:"deployer_account_id"` - FunderAccountID sql.NullString `json:"funderAccountId,omitempty" db:"funder_account_id"` + TokenID sql.NullString `json:"tokenId,omitempty" db:"token_id"` + Amount sql.NullString `json:"amount,omitempty" db:"amount"` + + // Nullable address fields (stored as BYTEA in database): + SignerAccountID NullAddressBytea `json:"signerAccountId,omitempty" db:"signer_account_id"` + SpenderAccountID NullAddressBytea `json:"spenderAccountId,omitempty" db:"spender_account_id"` + SponsoredAccountID NullAddressBytea `json:"sponsoredAccountId,omitempty" db:"sponsored_account_id"` + SponsorAccountID NullAddressBytea `json:"sponsorAccountId,omitempty" db:"sponsor_account_id"` + DeployerAccountID NullAddressBytea `json:"deployerAccountId,omitempty" db:"deployer_account_id"` + FunderAccountID NullAddressBytea `json:"funderAccountId,omitempty" db:"funder_account_id"` // Entity identifiers (moved from key_value JSONB): ClaimableBalanceID sql.NullString `json:"claimableBalanceId,omitempty" db:"claimable_balance_id"` @@ -438,7 +599,7 @@ type StateChange struct { KeyValue NullableJSONB `json:"keyValue,omitempty" db:"key_value"` // Relationships: - AccountID string `json:"accountId,omitempty" db:"account_id"` + AccountID AddressBytea `json:"accountId,omitempty" db:"account_id"` Account *Account `json:"account,omitempty"` OperationID int64 `json:"operationId,omitempty" db:"operation_id"` Operation *Operation `json:"operation,omitempty"` diff --git a/internal/indexer/types/types_test.go b/internal/indexer/types/types_test.go index 78a71a93c..c8422f890 100644 --- a/internal/indexer/types/types_test.go +++ b/internal/indexer/types/types_test.go @@ -2,9 +2,13 @@ package types import ( "database/sql/driver" + "encoding/hex" "testing" + "github.com/stellar/go-stellar-sdk/keypair" + "github.com/stellar/go-stellar-sdk/strkey" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestNullableJSONB_Scan(t *testing.T) { @@ -51,6 +55,156 @@ func TestNullableJSONB_Scan(t *testing.T) { } } +func TestAddressBytea_Scan(t *testing.T) { + // Generate a valid G... address for testing + kp := keypair.MustRandom() + validAddress := kp.Address() + + // Build expected 33-byte representation + _, rawBytes, err := strkey.DecodeAny(validAddress) + require.NoError(t, err) + validBytes := make([]byte, 33) + validBytes[0] = byte(strkey.VersionByteAccountID) + copy(validBytes[1:], rawBytes) + + testCases := []struct { + name string + input any + want AddressBytea + wantErrContains string + }{ + { + name: "🟢nil value", + input: nil, + want: "", + }, + { + name: "🟢valid 33-byte address", + input: validBytes, + want: AddressBytea(validAddress), + }, + { + name: "🔴wrong type", + input: 12345, + wantErrContains: "expected []byte", + }, + { + name: "🔴wrong length", + input: []byte{1, 2, 3}, + wantErrContains: "expected 33 bytes", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var s AddressBytea + err := s.Scan(tc.input) + if tc.wantErrContains != "" { + assert.ErrorContains(t, err, tc.wantErrContains) + } else { + assert.NoError(t, err) + assert.Equal(t, tc.want, s) + } + }) + } +} + +func TestAddressBytea_Value(t *testing.T) { + // Generate a valid G... address for testing + kp := keypair.MustRandom() + validAddress := kp.Address() + + // Build expected 33-byte representation + _, rawBytes, err := strkey.DecodeAny(validAddress) + require.NoError(t, err) + expectedBytes := make([]byte, 33) + expectedBytes[0] = byte(strkey.VersionByteAccountID) + copy(expectedBytes[1:], rawBytes) + + testCases := []struct { + name string + input AddressBytea + want driver.Value + wantErrContains string + }{ + { + name: "🟢empty string", + input: "", + want: nil, + }, + { + name: "🟢valid address", + input: AddressBytea(validAddress), + want: expectedBytes, + }, + { + name: "🔴invalid address", + input: "not-a-valid-address", + wantErrContains: "decoding stellar address", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := tc.input.Value() + if tc.wantErrContains != "" { + assert.ErrorContains(t, err, tc.wantErrContains) + } else { + assert.NoError(t, err) + assert.Equal(t, tc.want, got) + } + }) + } +} + +func TestAddressBytea_Roundtrip(t *testing.T) { + // Test that Value -> Scan produces the original address + kp := keypair.MustRandom() + original := AddressBytea(kp.Address()) + + // Convert to bytes + bytes, err := original.Value() + require.NoError(t, err) + + // Convert back to address + var restored AddressBytea + err = restored.Scan(bytes) + require.NoError(t, err) + + assert.Equal(t, original, restored) +} + +func TestAddressBytea_String(t *testing.T) { + testCases := []struct { + name string + input AddressBytea + want string + }{ + { + name: "🟢empty string", + input: "", + want: "", + }, + { + name: "🟢valid G address", + input: AddressBytea("GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H"), + want: "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", + }, + { + name: "🟢valid C address", + input: AddressBytea("CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC"), + want: "CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := tc.input.String() + assert.Equal(t, tc.want, got) + }) + } +} + func TestNullableJSONB_Value(t *testing.T) { testCases := []struct { name string @@ -88,3 +242,140 @@ func TestNullableJSONB_Value(t *testing.T) { }) } } + +func TestHashBytea_Scan(t *testing.T) { + // Valid 32-byte hash + validHex := "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760" + validBytes, err := hex.DecodeString(validHex) + require.NoError(t, err) + + testCases := []struct { + name string + input any + want HashBytea + wantErrContains string + }{ + { + name: "🟢nil value", + input: nil, + want: "", + }, + { + name: "🟢valid 32-byte hash", + input: validBytes, + want: HashBytea(validHex), + }, + { + name: "🔴wrong type", + input: 12345, + wantErrContains: "expected []byte", + }, + { + name: "🔴wrong length", + input: []byte{1, 2, 3}, + wantErrContains: "expected 32 bytes", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var h HashBytea + err := h.Scan(tc.input) + if tc.wantErrContains != "" { + assert.ErrorContains(t, err, tc.wantErrContains) + } else { + assert.NoError(t, err) + assert.Equal(t, tc.want, h) + } + }) + } +} + +func TestHashBytea_Value(t *testing.T) { + // Valid 32-byte hash + validHex := "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760" + expectedBytes, err := hex.DecodeString(validHex) + require.NoError(t, err) + + testCases := []struct { + name string + input HashBytea + want driver.Value + wantErrContains string + }{ + { + name: "🟢empty string", + input: "", + want: nil, + }, + { + name: "🟢valid hex hash", + input: HashBytea(validHex), + want: expectedBytes, + }, + { + name: "🔴invalid hex", + input: "not-a-valid-hex", + wantErrContains: "decoding hex hash", + }, + { + name: "🔴wrong length (short)", + input: "abcd", + wantErrContains: "invalid hash length", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := tc.input.Value() + if tc.wantErrContains != "" { + assert.ErrorContains(t, err, tc.wantErrContains) + } else { + assert.NoError(t, err) + assert.Equal(t, tc.want, got) + } + }) + } +} + +func TestHashBytea_Roundtrip(t *testing.T) { + // Test that Value -> Scan produces the original hash + original := HashBytea("e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760") + + // Convert to bytes + bytes, err := original.Value() + require.NoError(t, err) + + // Convert back to hash + var restored HashBytea + err = restored.Scan(bytes) + require.NoError(t, err) + + assert.Equal(t, original, restored) +} + +func TestHashBytea_String(t *testing.T) { + testCases := []struct { + name string + input HashBytea + want string + }{ + { + name: "🟢empty string", + input: "", + want: "", + }, + { + name: "🟢valid hex hash", + input: HashBytea("e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760"), + want: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := tc.input.String() + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/internal/integrationtests/infrastructure/backfill_helpers.go b/internal/integrationtests/infrastructure/backfill_helpers.go index 9448175ad..526e584f6 100644 --- a/internal/integrationtests/infrastructure/backfill_helpers.go +++ b/internal/integrationtests/infrastructure/backfill_helpers.go @@ -10,6 +10,8 @@ import ( "time" "github.com/stellar/go-stellar-sdk/support/log" + + "github.com/stellar/wallet-backend/internal/indexer/types" ) // GetIngestCursor retrieves a cursor value from the ingest_store table. @@ -59,7 +61,7 @@ func (s *SharedContainers) GetTransactionCountForAccount(ctx context.Context, ac WHERE ta.account_id = $1 AND t.ledger_number BETWEEN $2 AND $3 ` - err = db.QueryRowContext(ctx, query, accountAddr, startLedger, endLedger).Scan(&count) + err = db.QueryRowContext(ctx, query, types.AddressBytea(accountAddr), startLedger, endLedger).Scan(&count) if err != nil { return 0, fmt.Errorf("counting transactions for account %s: %w", accountAddr, err) } @@ -89,7 +91,7 @@ func (s *SharedContainers) HasOperationForAccount(ctx context.Context, accountAd AND o.ledger_number BETWEEN $3 AND $4 ) ` - err = db.QueryRowContext(ctx, query, accountAddr, opType, startLedger, endLedger).Scan(&exists) + err = db.QueryRowContext(ctx, query, types.AddressBytea(accountAddr), opType, startLedger, endLedger).Scan(&exists) if err != nil { return false, fmt.Errorf("checking operation for account %s: %w", accountAddr, err) } @@ -117,7 +119,7 @@ func (s *SharedContainers) GetTransactionAccountLinkCount(ctx context.Context, a WHERE ta.account_id = $1 AND t.ledger_number BETWEEN $2 AND $3 ` - err = db.QueryRowContext(ctx, query, accountAddr, startLedger, endLedger).Scan(&count) + err = db.QueryRowContext(ctx, query, types.AddressBytea(accountAddr), startLedger, endLedger).Scan(&count) if err != nil { return 0, fmt.Errorf("counting transaction-account links for %s: %w", accountAddr, err) } diff --git a/internal/serve/graphql/generated/generated.go b/internal/serve/graphql/generated/generated.go index 88c82a13c..836693bd5 100644 --- a/internal/serve/graphql/generated/generated.go +++ b/internal/serve/graphql/generated/generated.go @@ -164,7 +164,7 @@ type ComplexityRoot struct { LedgerCreatedAt func(childComplexity int) int LedgerNumber func(childComplexity int) int OperationType func(childComplexity int) int - OperationXDR func(childComplexity int) int + OperationXdr func(childComplexity int) int ResultCode func(childComplexity int) int StateChanges func(childComplexity int, first *int32, after *string, last *int32, before *string) int Successful func(childComplexity int) int @@ -395,6 +395,8 @@ type MutationResolver interface { CreateFeeBumpTransaction(ctx context.Context, input CreateFeeBumpTransactionInput) (*CreateFeeBumpTransactionPayload, error) } type OperationResolver interface { + OperationXdr(ctx context.Context, obj *types.Operation) (string, error) + Transaction(ctx context.Context, obj *types.Operation) (*types.Transaction, error) Accounts(ctx context.Context, obj *types.Operation) ([]*types.Account, error) StateChanges(ctx context.Context, obj *types.Operation, first *int32, after *string, last *int32, before *string) (*StateChangeConnection, error) @@ -453,6 +455,8 @@ type StandardBalanceChangeResolver interface { Amount(ctx context.Context, obj *types.StandardBalanceStateChangeModel) (string, error) } type TransactionResolver interface { + Hash(ctx context.Context, obj *types.Transaction) (string, error) + Operations(ctx context.Context, obj *types.Transaction, first *int32, after *string, last *int32, before *string) (*OperationConnection, error) Accounts(ctx context.Context, obj *types.Transaction) ([]*types.Account, error) StateChanges(ctx context.Context, obj *types.Transaction, first *int32, after *string, last *int32, before *string) (*StateChangeConnection, error) @@ -1007,11 +1011,11 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return e.complexity.Operation.OperationType(childComplexity), true case "Operation.operationXdr": - if e.complexity.Operation.OperationXDR == nil { + if e.complexity.Operation.OperationXdr == nil { break } - return e.complexity.Operation.OperationXDR(childComplexity), true + return e.complexity.Operation.OperationXdr(childComplexity), true case "Operation.resultCode": if e.complexity.Operation.ResultCode == nil { @@ -2314,7 +2318,7 @@ type CreateFeeBumpTransactionPayload { type Operation{ id: Int64! operationType: OperationType! - operationXdr: String! + operationXdr: String! @goField(forceResolver: true) resultCode: String! successful: Boolean! ledgerNumber: UInt32! @@ -2551,7 +2555,7 @@ type BalanceAuthorizationChange implements BaseStateChange{ {Name: "../schema/transaction.graphqls", Input: `# GraphQL Transaction type - represents a blockchain transaction # gqlgen generates Go structs from this schema definition type Transaction{ - hash: String! + hash: String! @goField(forceResolver: true) envelopeXdr: String feeCharged: Int64! resultCode: String! @@ -6820,7 +6824,7 @@ func (ec *executionContext) _Operation_operationXdr(ctx context.Context, field g }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { ctx = rctx // use context from middleware stack in children - return obj.OperationXDR, nil + return ec.resolvers.Operation().OperationXdr(rctx, obj) }) if err != nil { ec.Error(ctx, err) @@ -6841,8 +6845,8 @@ func (ec *executionContext) fieldContext_Operation_operationXdr(_ context.Contex fc = &graphql.FieldContext{ Object: "Operation", Field: field, - IsMethod: false, - IsResolver: false, + IsMethod: true, + IsResolver: true, Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { return nil, errors.New("field of type String does not have child fields") }, @@ -11278,7 +11282,7 @@ func (ec *executionContext) _Transaction_hash(ctx context.Context, field graphql }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { ctx = rctx // use context from middleware stack in children - return obj.Hash, nil + return ec.resolvers.Transaction().Hash(rctx, obj) }) if err != nil { ec.Error(ctx, err) @@ -11299,8 +11303,8 @@ func (ec *executionContext) fieldContext_Transaction_hash(_ context.Context, fie fc = &graphql.FieldContext{ Object: "Transaction", Field: field, - IsMethod: false, - IsResolver: false, + IsMethod: true, + IsResolver: true, Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { return nil, errors.New("field of type String does not have child fields") }, @@ -17010,10 +17014,41 @@ func (ec *executionContext) _Operation(ctx context.Context, sel ast.SelectionSet atomic.AddUint32(&out.Invalids, 1) } case "operationXdr": - out.Values[i] = ec._Operation_operationXdr(ctx, field, obj) - if out.Values[i] == graphql.Null { - atomic.AddUint32(&out.Invalids, 1) + field := field + + innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + } + }() + res = ec._Operation_operationXdr(ctx, field, obj) + if res == graphql.Null { + atomic.AddUint32(&fs.Invalids, 1) + } + return res + } + + if field.Deferrable != nil { + dfs, ok := deferred[field.Deferrable.Label] + di := 0 + if ok { + dfs.AddField(field) + di = len(dfs.Values) - 1 + } else { + dfs = graphql.NewFieldSet([]graphql.CollectedField{field}) + deferred[field.Deferrable.Label] = dfs + } + dfs.Concurrently(di, func(ctx context.Context) graphql.Marshaler { + return innerFunc(ctx, dfs) + }) + + // don't run the out.Concurrently() call below + out.Values[i] = graphql.Null + continue } + + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) case "resultCode": out.Values[i] = ec._Operation_resultCode(ctx, field, obj) if out.Values[i] == graphql.Null { @@ -19054,10 +19089,41 @@ func (ec *executionContext) _Transaction(ctx context.Context, sel ast.SelectionS case "__typename": out.Values[i] = graphql.MarshalString("Transaction") case "hash": - out.Values[i] = ec._Transaction_hash(ctx, field, obj) - if out.Values[i] == graphql.Null { - atomic.AddUint32(&out.Invalids, 1) + field := field + + innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + } + }() + res = ec._Transaction_hash(ctx, field, obj) + if res == graphql.Null { + atomic.AddUint32(&fs.Invalids, 1) + } + return res + } + + if field.Deferrable != nil { + dfs, ok := deferred[field.Deferrable.Label] + di := 0 + if ok { + dfs.AddField(field) + di = len(dfs.Values) - 1 + } else { + dfs = graphql.NewFieldSet([]graphql.CollectedField{field}) + deferred[field.Deferrable.Label] = dfs + } + dfs.Concurrently(di, func(ctx context.Context) graphql.Marshaler { + return innerFunc(ctx, dfs) + }) + + // don't run the out.Concurrently() call below + out.Values[i] = graphql.Null + continue } + + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) case "envelopeXdr": out.Values[i] = ec._Transaction_envelopeXdr(ctx, field, obj) case "feeCharged": diff --git a/internal/serve/graphql/resolvers/account.resolvers.go b/internal/serve/graphql/resolvers/account.resolvers.go index 85e78a2c7..42cce7eb3 100644 --- a/internal/serve/graphql/resolvers/account.resolvers.go +++ b/internal/serve/graphql/resolvers/account.resolvers.go @@ -15,7 +15,7 @@ import ( // Address is the resolver for the address field. func (r *accountResolver) Address(ctx context.Context, obj *types.Account) (string, error) { - return obj.StellarAddress, nil + return string(obj.StellarAddress), nil } // Transactions is the resolver for the transactions field. @@ -30,7 +30,7 @@ func (r *accountResolver) Transactions(ctx context.Context, obj *types.Account, queryLimit := *params.Limit + 1 // +1 to check if there is a next page dbColumns := GetDBColumnsForFields(ctx, types.Transaction{}) - transactions, err := r.models.Transactions.BatchGetByAccountAddress(ctx, obj.StellarAddress, strings.Join(dbColumns, ", "), &queryLimit, params.Cursor, params.SortOrder) + transactions, err := r.models.Transactions.BatchGetByAccountAddress(ctx, string(obj.StellarAddress), strings.Join(dbColumns, ", "), &queryLimit, params.Cursor, params.SortOrder) if err != nil { return nil, fmt.Errorf("getting transactions from db for account %s: %w", obj.StellarAddress, err) } @@ -63,7 +63,7 @@ func (r *accountResolver) Operations(ctx context.Context, obj *types.Account, fi queryLimit := *params.Limit + 1 // +1 to check if there is a next page dbColumns := GetDBColumnsForFields(ctx, types.Operation{}) - operations, err := r.models.Operations.BatchGetByAccountAddress(ctx, obj.StellarAddress, strings.Join(dbColumns, ", "), &queryLimit, params.Cursor, params.SortOrder) + operations, err := r.models.Operations.BatchGetByAccountAddress(ctx, string(obj.StellarAddress), strings.Join(dbColumns, ", "), &queryLimit, params.Cursor, params.SortOrder) if err != nil { return nil, fmt.Errorf("getting operations from db for account %s: %w", obj.StellarAddress, err) } @@ -115,7 +115,7 @@ func (r *accountResolver) StateChanges(ctx context.Context, obj *types.Account, } dbColumns := GetDBColumnsForFields(ctx, types.StateChange{}) - stateChanges, err := r.models.StateChanges.BatchGetByAccountAddress(ctx, obj.StellarAddress, txHash, operationID, category, reason, strings.Join(dbColumns, ", "), &queryLimit, params.StateChangeCursor, params.SortOrder) + stateChanges, err := r.models.StateChanges.BatchGetByAccountAddress(ctx, string(obj.StellarAddress), txHash, operationID, category, reason, strings.Join(dbColumns, ", "), &queryLimit, params.StateChangeCursor, params.SortOrder) if err != nil { return nil, fmt.Errorf("getting state changes from db for account %s: %w", obj.StellarAddress, err) } diff --git a/internal/serve/graphql/resolvers/account_resolvers_test.go b/internal/serve/graphql/resolvers/account_resolvers_test.go index a70e67977..609786c51 100644 --- a/internal/serve/graphql/resolvers/account_resolvers_test.go +++ b/internal/serve/graphql/resolvers/account_resolvers_test.go @@ -1,6 +1,8 @@ package resolvers import ( + "encoding/base64" + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -15,8 +17,13 @@ import ( graphql1 "github.com/stellar/wallet-backend/internal/serve/graphql/generated" ) +// testOpXDRAcc returns the expected base64-encoded XDR for test operation N +func testOpXDRAcc(n int) string { + return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("opxdr%d", n))) +} + func TestAccountResolver_Transactions(t *testing.T) { - parentAccount := &types.Account{StellarAddress: "test-account"} + parentAccount := &types.Account{StellarAddress: types.AddressBytea(sharedTestAccountAddress)} mockMetricsService := &metrics.MockMetricsService{} mockMetricsService.On("IncDBQuery", "BatchGetByAccountAddress", "transactions").Return() @@ -39,10 +46,10 @@ func TestAccountResolver_Transactions(t *testing.T) { require.NoError(t, err) require.Len(t, transactions.Edges, 4) - assert.Equal(t, "tx1", transactions.Edges[0].Node.Hash) - assert.Equal(t, "tx2", transactions.Edges[1].Node.Hash) - assert.Equal(t, "tx3", transactions.Edges[2].Node.Hash) - assert.Equal(t, "tx4", transactions.Edges[3].Node.Hash) + assert.Equal(t, testTxHash1, transactions.Edges[0].Node.Hash.String()) + assert.Equal(t, testTxHash2, transactions.Edges[1].Node.Hash.String()) + assert.Equal(t, testTxHash3, transactions.Edges[2].Node.Hash.String()) + assert.Equal(t, testTxHash4, transactions.Edges[3].Node.Hash.String()) mockMetricsService.AssertExpectations(t) }) @@ -52,8 +59,8 @@ func TestAccountResolver_Transactions(t *testing.T) { txs, err := resolver.Transactions(ctx, parentAccount, &first, nil, nil, nil) require.NoError(t, err) assert.Len(t, txs.Edges, 2) - assert.Equal(t, "tx1", txs.Edges[0].Node.Hash) - assert.Equal(t, "tx2", txs.Edges[1].Node.Hash) + assert.Equal(t, testTxHash1, txs.Edges[0].Node.Hash.String()) + assert.Equal(t, testTxHash2, txs.Edges[1].Node.Hash.String()) assert.True(t, txs.PageInfo.HasNextPage) assert.False(t, txs.PageInfo.HasPreviousPage) @@ -63,8 +70,8 @@ func TestAccountResolver_Transactions(t *testing.T) { txs, err = resolver.Transactions(ctx, parentAccount, &first, nextCursor, nil, nil) require.NoError(t, err) assert.Len(t, txs.Edges, 2) - assert.Equal(t, "tx3", txs.Edges[0].Node.Hash) - assert.Equal(t, "tx4", txs.Edges[1].Node.Hash) + assert.Equal(t, testTxHash3, txs.Edges[0].Node.Hash.String()) + assert.Equal(t, testTxHash4, txs.Edges[1].Node.Hash.String()) assert.False(t, txs.PageInfo.HasNextPage) assert.True(t, txs.PageInfo.HasPreviousPage) mockMetricsService.AssertExpectations(t) @@ -76,8 +83,8 @@ func TestAccountResolver_Transactions(t *testing.T) { txs, err := resolver.Transactions(ctx, parentAccount, nil, nil, &last, nil) require.NoError(t, err) assert.Len(t, txs.Edges, 2) - assert.Equal(t, "tx3", txs.Edges[0].Node.Hash) - assert.Equal(t, "tx4", txs.Edges[1].Node.Hash) + assert.Equal(t, testTxHash3, txs.Edges[0].Node.Hash.String()) + assert.Equal(t, testTxHash4, txs.Edges[1].Node.Hash.String()) assert.False(t, txs.PageInfo.HasNextPage) assert.True(t, txs.PageInfo.HasPreviousPage) @@ -88,7 +95,7 @@ func TestAccountResolver_Transactions(t *testing.T) { txs, err = resolver.Transactions(ctx, parentAccount, nil, nil, &last, nextCursor) require.NoError(t, err) assert.Len(t, txs.Edges, 1) - assert.Equal(t, "tx2", txs.Edges[0].Node.Hash) + assert.Equal(t, testTxHash2, txs.Edges[0].Node.Hash.String()) assert.True(t, txs.PageInfo.HasNextPage) assert.True(t, txs.PageInfo.HasPreviousPage) @@ -98,14 +105,14 @@ func TestAccountResolver_Transactions(t *testing.T) { txs, err = resolver.Transactions(ctx, parentAccount, nil, nil, &last, nextCursor) require.NoError(t, err) assert.Len(t, txs.Edges, 1) - assert.Equal(t, "tx1", txs.Edges[0].Node.Hash) + assert.Equal(t, testTxHash1, txs.Edges[0].Node.Hash.String()) assert.True(t, txs.PageInfo.HasNextPage) assert.False(t, txs.PageInfo.HasPreviousPage) mockMetricsService.AssertExpectations(t) }) t.Run("account with no transactions", func(t *testing.T) { - nonExistentAccount := &types.Account{StellarAddress: "non-existent-account"} + nonExistentAccount := &types.Account{StellarAddress: types.AddressBytea(sharedNonExistentAccountAddress)} ctx := getTestCtx("transactions", []string{"hash"}) transactions, err := resolver.Transactions(ctx, nonExistentAccount, nil, nil, nil, nil) @@ -144,7 +151,7 @@ func TestAccountResolver_Transactions(t *testing.T) { } func TestAccountResolver_Operations(t *testing.T) { - parentAccount := &types.Account{StellarAddress: "test-account"} + parentAccount := &types.Account{StellarAddress: types.AddressBytea(sharedTestAccountAddress)} mockMetricsService := &metrics.MockMetricsService{} mockMetricsService.On("IncDBQuery", "BatchGetByAccountAddress", "operations").Return() @@ -166,10 +173,10 @@ func TestAccountResolver_Operations(t *testing.T) { require.NoError(t, err) require.Len(t, operations.Edges, 8) - assert.Equal(t, "opxdr1", operations.Edges[0].Node.OperationXDR) - assert.Equal(t, "opxdr2", operations.Edges[1].Node.OperationXDR) - assert.Equal(t, "opxdr3", operations.Edges[2].Node.OperationXDR) - assert.Equal(t, "opxdr4", operations.Edges[3].Node.OperationXDR) + assert.Equal(t, testOpXDRAcc(1), operations.Edges[0].Node.OperationXDR.String()) + assert.Equal(t, testOpXDRAcc(2), operations.Edges[1].Node.OperationXDR.String()) + assert.Equal(t, testOpXDRAcc(3), operations.Edges[2].Node.OperationXDR.String()) + assert.Equal(t, testOpXDRAcc(4), operations.Edges[3].Node.OperationXDR.String()) }) t.Run("get operations with first/after limit and cursor", func(t *testing.T) { @@ -178,8 +185,8 @@ func TestAccountResolver_Operations(t *testing.T) { ops, err := resolver.Operations(ctx, parentAccount, &first, nil, nil, nil) require.NoError(t, err) assert.Len(t, ops.Edges, 2) - assert.Equal(t, "opxdr1", ops.Edges[0].Node.OperationXDR) - assert.Equal(t, "opxdr2", ops.Edges[1].Node.OperationXDR) + assert.Equal(t, testOpXDRAcc(1), ops.Edges[0].Node.OperationXDR.String()) + assert.Equal(t, testOpXDRAcc(2), ops.Edges[1].Node.OperationXDR.String()) assert.True(t, ops.PageInfo.HasNextPage) assert.False(t, ops.PageInfo.HasPreviousPage) @@ -189,8 +196,8 @@ func TestAccountResolver_Operations(t *testing.T) { ops, err = resolver.Operations(ctx, parentAccount, &first, nextCursor, nil, nil) require.NoError(t, err) assert.Len(t, ops.Edges, 2) - assert.Equal(t, "opxdr3", ops.Edges[0].Node.OperationXDR) - assert.Equal(t, "opxdr4", ops.Edges[1].Node.OperationXDR) + assert.Equal(t, testOpXDRAcc(3), ops.Edges[0].Node.OperationXDR.String()) + assert.Equal(t, testOpXDRAcc(4), ops.Edges[1].Node.OperationXDR.String()) assert.True(t, ops.PageInfo.HasNextPage) assert.True(t, ops.PageInfo.HasPreviousPage) @@ -200,10 +207,10 @@ func TestAccountResolver_Operations(t *testing.T) { ops, err = resolver.Operations(ctx, parentAccount, &first, nextCursor, nil, nil) require.NoError(t, err) assert.Len(t, ops.Edges, 4) - assert.Equal(t, "opxdr5", ops.Edges[0].Node.OperationXDR) - assert.Equal(t, "opxdr6", ops.Edges[1].Node.OperationXDR) - assert.Equal(t, "opxdr7", ops.Edges[2].Node.OperationXDR) - assert.Equal(t, "opxdr8", ops.Edges[3].Node.OperationXDR) + assert.Equal(t, testOpXDRAcc(5), ops.Edges[0].Node.OperationXDR.String()) + assert.Equal(t, testOpXDRAcc(6), ops.Edges[1].Node.OperationXDR.String()) + assert.Equal(t, testOpXDRAcc(7), ops.Edges[2].Node.OperationXDR.String()) + assert.Equal(t, testOpXDRAcc(8), ops.Edges[3].Node.OperationXDR.String()) assert.False(t, ops.PageInfo.HasNextPage) assert.True(t, ops.PageInfo.HasPreviousPage) }) @@ -214,8 +221,8 @@ func TestAccountResolver_Operations(t *testing.T) { ops, err := resolver.Operations(ctx, parentAccount, nil, nil, &last, nil) require.NoError(t, err) assert.Len(t, ops.Edges, 2) - assert.Equal(t, "opxdr7", ops.Edges[0].Node.OperationXDR) - assert.Equal(t, "opxdr8", ops.Edges[1].Node.OperationXDR) + assert.Equal(t, testOpXDRAcc(7), ops.Edges[0].Node.OperationXDR.String()) + assert.Equal(t, testOpXDRAcc(8), ops.Edges[1].Node.OperationXDR.String()) assert.True(t, ops.PageInfo.HasPreviousPage) assert.False(t, ops.PageInfo.HasNextPage) @@ -225,8 +232,8 @@ func TestAccountResolver_Operations(t *testing.T) { ops, err = resolver.Operations(ctx, parentAccount, nil, nil, &last, nextCursor) require.NoError(t, err) assert.Len(t, ops.Edges, 2) - assert.Equal(t, "opxdr5", ops.Edges[0].Node.OperationXDR) - assert.Equal(t, "opxdr6", ops.Edges[1].Node.OperationXDR) + assert.Equal(t, testOpXDRAcc(5), ops.Edges[0].Node.OperationXDR.String()) + assert.Equal(t, testOpXDRAcc(6), ops.Edges[1].Node.OperationXDR.String()) assert.True(t, ops.PageInfo.HasNextPage) assert.True(t, ops.PageInfo.HasPreviousPage) @@ -236,16 +243,16 @@ func TestAccountResolver_Operations(t *testing.T) { ops, err = resolver.Operations(ctx, parentAccount, nil, nil, &last, nextCursor) require.NoError(t, err) assert.Len(t, ops.Edges, 4) - assert.Equal(t, "opxdr1", ops.Edges[0].Node.OperationXDR) - assert.Equal(t, "opxdr2", ops.Edges[1].Node.OperationXDR) - assert.Equal(t, "opxdr3", ops.Edges[2].Node.OperationXDR) - assert.Equal(t, "opxdr4", ops.Edges[3].Node.OperationXDR) + assert.Equal(t, testOpXDRAcc(1), ops.Edges[0].Node.OperationXDR.String()) + assert.Equal(t, testOpXDRAcc(2), ops.Edges[1].Node.OperationXDR.String()) + assert.Equal(t, testOpXDRAcc(3), ops.Edges[2].Node.OperationXDR.String()) + assert.Equal(t, testOpXDRAcc(4), ops.Edges[3].Node.OperationXDR.String()) assert.True(t, ops.PageInfo.HasNextPage) assert.False(t, ops.PageInfo.HasPreviousPage) }) t.Run("account with no operations", func(t *testing.T) { - nonExistentAccount := &types.Account{StellarAddress: "non-existent-account"} + nonExistentAccount := &types.Account{StellarAddress: types.AddressBytea(sharedNonExistentAccountAddress)} ctx := getTestCtx("operations", []string{"id"}) operations, err := resolver.Operations(ctx, nonExistentAccount, nil, nil, nil, nil) @@ -255,7 +262,7 @@ func TestAccountResolver_Operations(t *testing.T) { } func TestAccountResolver_StateChanges(t *testing.T) { - parentAccount := &types.Account{StellarAddress: "test-account"} + parentAccount := &types.Account{StellarAddress: types.AddressBytea(sharedTestAccountAddress)} mockMetricsService := &metrics.MockMetricsService{} mockMetricsService.On("IncDBQuery", "BatchGetByAccountAddress", "state_changes").Return() @@ -490,7 +497,7 @@ func TestAccountResolver_StateChanges(t *testing.T) { }) t.Run("account with no state changes", func(t *testing.T) { - nonExistentAccount := &types.Account{StellarAddress: "non-existent-account"} + nonExistentAccount := &types.Account{StellarAddress: types.AddressBytea(sharedNonExistentAccountAddress)} ctx := getTestCtx("state_changes", []string{"to_id", "state_change_order"}) stateChanges, err := resolver.StateChanges(ctx, nonExistentAccount, nil, nil, nil, nil, nil) @@ -500,7 +507,7 @@ func TestAccountResolver_StateChanges(t *testing.T) { } func TestAccountResolver_StateChanges_WithFilters(t *testing.T) { - parentAccount := &types.Account{StellarAddress: "test-account"} + parentAccount := &types.Account{StellarAddress: types.AddressBytea(sharedTestAccountAddress)} mockMetricsService := &metrics.MockMetricsService{} mockMetricsService.On("IncDBQuery", "BatchGetByAccountAddress", "state_changes").Return() @@ -518,7 +525,7 @@ func TestAccountResolver_StateChanges_WithFilters(t *testing.T) { t.Run("filter by transaction hash only", func(t *testing.T) { ctx := getTestCtx("state_changes", []string{""}) - txHash := "tx1" + txHash := testTxHash1 filter := &graphql1.AccountStateChangeFilterInput{ TransactionHash: &txHash, } @@ -591,7 +598,7 @@ func TestAccountResolver_StateChanges_WithFilters(t *testing.T) { t.Run("filter by both transaction hash and operation ID", func(t *testing.T) { ctx := getTestCtx("state_changes", []string{""}) - txHash := "tx2" + txHash := testTxHash2 opID := toid.New(1000, 2, 1).ToInt64() txToID := opID &^ 0xFFF // Derive transaction to_id from operation_id filter := &graphql1.AccountStateChangeFilterInput{ @@ -618,7 +625,8 @@ func TestAccountResolver_StateChanges_WithFilters(t *testing.T) { t.Run("filter with no matching results", func(t *testing.T) { ctx := getTestCtx("state_changes", []string{""}) - txHash := "non-existent-tx" + // Use a valid 64-char hex hash that doesn't exist in the test DB + txHash := "0000000000000000000000000000000000000000000000000000000000000000" filter := &graphql1.AccountStateChangeFilterInput{ TransactionHash: &txHash, } @@ -632,7 +640,7 @@ func TestAccountResolver_StateChanges_WithFilters(t *testing.T) { t.Run("filter with pagination", func(t *testing.T) { ctx := getTestCtx("state_changes", []string{""}) - txHash := "tx1" + txHash := testTxHash1 filter := &graphql1.AccountStateChangeFilterInput{ TransactionHash: &txHash, } @@ -701,7 +709,7 @@ func TestAccountResolver_StateChanges_WithFilters(t *testing.T) { } func TestAccountResolver_StateChanges_WithCategoryReasonFilters(t *testing.T) { - parentAccount := &types.Account{StellarAddress: "test-account"} + parentAccount := &types.Account{StellarAddress: types.AddressBytea(sharedTestAccountAddress)} mockMetricsService := &metrics.MockMetricsService{} mockMetricsService.On("IncDBQuery", "BatchGetByAccountAddress", "state_changes").Return() @@ -766,7 +774,7 @@ func TestAccountResolver_StateChanges_WithCategoryReasonFilters(t *testing.T) { t.Run("filter with all filters - txHash, operationID, category, reason", func(t *testing.T) { ctx := getTestCtx("state_changes", []string{""}) - txHash := "tx1" + txHash := testTxHash1 opID := toid.New(1000, 1, 1).ToInt64() txToID := opID &^ 0xFFF // Derive transaction to_id from operation_id category := "BALANCE" diff --git a/internal/serve/graphql/resolvers/mutations.resolvers.go b/internal/serve/graphql/resolvers/mutations.resolvers.go index c2e8a2851..64396a7e7 100644 --- a/internal/serve/graphql/resolvers/mutations.resolvers.go +++ b/internal/serve/graphql/resolvers/mutations.resolvers.go @@ -53,7 +53,7 @@ func (r *mutationResolver) RegisterAccount(ctx context.Context, input graphql1.R // Return the account data directly since we know the address account := &types.Account{ - StellarAddress: input.Address, + StellarAddress: types.AddressBytea(input.Address), CreatedAt: time.Now(), } diff --git a/internal/serve/graphql/resolvers/mutations_resolvers_test.go b/internal/serve/graphql/resolvers/mutations_resolvers_test.go index 924aa3afe..48c76bcef 100644 --- a/internal/serve/graphql/resolvers/mutations_resolvers_test.go +++ b/internal/serve/graphql/resolvers/mutations_resolvers_test.go @@ -86,7 +86,7 @@ func TestMutationResolver_RegisterAccount(t *testing.T) { assert.NotNil(t, result) assert.True(t, result.Success) assert.NotNil(t, result.Account) - assert.Equal(t, input.Address, result.Account.StellarAddress) + assert.Equal(t, input.Address, string(result.Account.StellarAddress)) mockService.AssertExpectations(t) }) diff --git a/internal/serve/graphql/resolvers/operation.resolvers.go b/internal/serve/graphql/resolvers/operation.resolvers.go index ca23c400c..4097f59c9 100644 --- a/internal/serve/graphql/resolvers/operation.resolvers.go +++ b/internal/serve/graphql/resolvers/operation.resolvers.go @@ -15,6 +15,12 @@ import ( "github.com/stellar/wallet-backend/internal/serve/middleware" ) +// OperationXdr is the resolver for the operationXdr field. +// Returns the operation XDR as a base64-encoded string. +func (r *operationResolver) OperationXdr(ctx context.Context, obj *types.Operation) (string, error) { + return obj.OperationXDR.String(), nil +} + // Transaction is the resolver for the transaction field. // This is a field resolver - it resolves the "transaction" field on an Operation object // gqlgen calls this when a GraphQL query requests the transaction field on an Operation diff --git a/internal/serve/graphql/resolvers/operation_resolvers_test.go b/internal/serve/graphql/resolvers/operation_resolvers_test.go index 0c1ebc4e1..fcb384c4a 100644 --- a/internal/serve/graphql/resolvers/operation_resolvers_test.go +++ b/internal/serve/graphql/resolvers/operation_resolvers_test.go @@ -42,7 +42,7 @@ func TestOperationResolver_Transaction(t *testing.T) { require.NoError(t, err) require.NotNil(t, transaction) - assert.Equal(t, "tx1", transaction.Hash) + assert.Equal(t, testTxHash1, transaction.Hash.String()) }) t.Run("nil operation panics", func(t *testing.T) { @@ -92,7 +92,7 @@ func TestOperationResolver_Accounts(t *testing.T) { require.NoError(t, err) require.Len(t, accounts, 1) - assert.Equal(t, "test-account", accounts[0].StellarAddress) + assert.Equal(t, sharedTestAccountAddress, string(accounts[0].StellarAddress)) }) t.Run("nil operation panics", func(t *testing.T) { diff --git a/internal/serve/graphql/resolvers/queries.resolvers.go b/internal/serve/graphql/resolvers/queries.resolvers.go index 1fdb97d3c..3677303ab 100644 --- a/internal/serve/graphql/resolvers/queries.resolvers.go +++ b/internal/serve/graphql/resolvers/queries.resolvers.go @@ -78,7 +78,7 @@ func (r *queryResolver) AccountByAddress(ctx context.Context, address string) (* } // When participant filtering is disabled, we return the account object so that the resolver can return a valid object. - return &types.Account{StellarAddress: address}, nil + return &types.Account{StellarAddress: types.AddressBytea(address)}, nil } return nil, err } diff --git a/internal/serve/graphql/resolvers/queries_resolvers_test.go b/internal/serve/graphql/resolvers/queries_resolvers_test.go index 7a3b7b104..2be5511cb 100644 --- a/internal/serve/graphql/resolvers/queries_resolvers_test.go +++ b/internal/serve/graphql/resolvers/queries_resolvers_test.go @@ -1,6 +1,8 @@ package resolvers import ( + "encoding/base64" + "fmt" "testing" "github.com/stellar/go-stellar-sdk/toid" @@ -12,6 +14,11 @@ import ( "github.com/stellar/wallet-backend/internal/metrics" ) +// testOpXDR returns the expected base64-encoded XDR for test operation N +func testOpXDR(n int) string { + return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("opxdr%d", n))) +} + func TestQueryResolver_TransactionByHash(t *testing.T) { mockMetricsService := &metrics.MockMetricsService{} mockMetricsService.On("ObserveDBQueryDuration", "GetByHash", "transactions", mock.Anything).Return() @@ -32,10 +39,10 @@ func TestQueryResolver_TransactionByHash(t *testing.T) { t.Run("success", func(t *testing.T) { ctx := getTestCtx("transactions", []string{"hash", "toId", "envelopeXdr", "feeCharged", "resultCode", "metaXdr", "ledgerNumber", "ledgerCreatedAt", "isFeeBump"}) - tx, err := resolver.TransactionByHash(ctx, "tx1") + tx, err := resolver.TransactionByHash(ctx, testTxHash1) require.NoError(t, err) - assert.Equal(t, "tx1", tx.Hash) + assert.Equal(t, testTxHash1, tx.Hash.String()) assert.Equal(t, toid.New(1000, 1, 0).ToInt64(), tx.ToID) require.NotNil(t, tx.EnvelopeXDR) assert.Equal(t, "envelope1", *tx.EnvelopeXDR) @@ -86,10 +93,10 @@ func TestQueryResolver_Transactions(t *testing.T) { require.NoError(t, err) require.Len(t, transactions.Edges, 4) - assert.Equal(t, "tx1", transactions.Edges[0].Node.Hash) - assert.Equal(t, "tx2", transactions.Edges[1].Node.Hash) - assert.Equal(t, "tx3", transactions.Edges[2].Node.Hash) - assert.Equal(t, "tx4", transactions.Edges[3].Node.Hash) + assert.Equal(t, testTxHash1, transactions.Edges[0].Node.Hash.String()) + assert.Equal(t, testTxHash2, transactions.Edges[1].Node.Hash.String()) + assert.Equal(t, testTxHash3, transactions.Edges[2].Node.Hash.String()) + assert.Equal(t, testTxHash4, transactions.Edges[3].Node.Hash.String()) }) t.Run("get transactions with first/after limit and cursor", func(t *testing.T) { @@ -98,8 +105,8 @@ func TestQueryResolver_Transactions(t *testing.T) { txs, err := resolver.Transactions(ctx, &first, nil, nil, nil) require.NoError(t, err) assert.Len(t, txs.Edges, 2) - assert.Equal(t, "tx1", txs.Edges[0].Node.Hash) - assert.Equal(t, "tx2", txs.Edges[1].Node.Hash) + assert.Equal(t, testTxHash1, txs.Edges[0].Node.Hash.String()) + assert.Equal(t, testTxHash2, txs.Edges[1].Node.Hash.String()) assert.True(t, txs.PageInfo.HasNextPage) assert.False(t, txs.PageInfo.HasPreviousPage) @@ -110,7 +117,7 @@ func TestQueryResolver_Transactions(t *testing.T) { txs, err = resolver.Transactions(ctx, &first, nextCursor, nil, nil) require.NoError(t, err) assert.Len(t, txs.Edges, 1) - assert.Equal(t, "tx3", txs.Edges[0].Node.Hash) + assert.Equal(t, testTxHash3, txs.Edges[0].Node.Hash.String()) assert.True(t, txs.PageInfo.HasNextPage) assert.True(t, txs.PageInfo.HasPreviousPage) @@ -121,7 +128,7 @@ func TestQueryResolver_Transactions(t *testing.T) { txs, err = resolver.Transactions(ctx, &first, nextCursor, nil, nil) require.NoError(t, err) assert.Len(t, txs.Edges, 1) - assert.Equal(t, "tx4", txs.Edges[0].Node.Hash) + assert.Equal(t, testTxHash4, txs.Edges[0].Node.Hash.String()) assert.False(t, txs.PageInfo.HasNextPage) assert.True(t, txs.PageInfo.HasPreviousPage) }) @@ -132,8 +139,8 @@ func TestQueryResolver_Transactions(t *testing.T) { txs, err := resolver.Transactions(ctx, nil, nil, &last, nil) require.NoError(t, err) assert.Len(t, txs.Edges, 2) - assert.Equal(t, "tx3", txs.Edges[0].Node.Hash) - assert.Equal(t, "tx4", txs.Edges[1].Node.Hash) + assert.Equal(t, testTxHash3, txs.Edges[0].Node.Hash.String()) + assert.Equal(t, testTxHash4, txs.Edges[1].Node.Hash.String()) assert.False(t, txs.PageInfo.HasNextPage) assert.True(t, txs.PageInfo.HasPreviousPage) @@ -144,7 +151,7 @@ func TestQueryResolver_Transactions(t *testing.T) { txs, err = resolver.Transactions(ctx, nil, nil, &last, nextCursor) require.NoError(t, err) assert.Len(t, txs.Edges, 1) - assert.Equal(t, "tx2", txs.Edges[0].Node.Hash) + assert.Equal(t, testTxHash2, txs.Edges[0].Node.Hash.String()) assert.True(t, txs.PageInfo.HasNextPage) assert.True(t, txs.PageInfo.HasPreviousPage) @@ -154,7 +161,7 @@ func TestQueryResolver_Transactions(t *testing.T) { txs, err = resolver.Transactions(ctx, nil, nil, &last, nextCursor) require.NoError(t, err) assert.Len(t, txs.Edges, 1) - assert.Equal(t, "tx1", txs.Edges[0].Node.Hash) + assert.Equal(t, testTxHash1, txs.Edges[0].Node.Hash.String()) assert.True(t, txs.PageInfo.HasNextPage) assert.False(t, txs.PageInfo.HasPreviousPage) }) @@ -235,16 +242,16 @@ func TestQueryResolver_Account(t *testing.T) { } t.Run("success", func(t *testing.T) { - acc, err := resolver.AccountByAddress(testCtx, "test-account") + acc, err := resolver.AccountByAddress(testCtx, sharedTestAccountAddress) require.NoError(t, err) - assert.Equal(t, "test-account", acc.StellarAddress) + assert.Equal(t, sharedTestAccountAddress, string(acc.StellarAddress)) }) t.Run("non-existent account", func(t *testing.T) { - acc, err := resolver.AccountByAddress(testCtx, "non-existent-account") + acc, err := resolver.AccountByAddress(testCtx, sharedNonExistentAccountAddress) require.NoError(t, err) assert.NotNil(t, acc) - assert.Equal(t, "non-existent-account", acc.StellarAddress) + assert.Equal(t, sharedNonExistentAccountAddress, string(acc.StellarAddress)) }) t.Run("empty address", func(t *testing.T) { @@ -278,10 +285,10 @@ func TestQueryResolver_Operations(t *testing.T) { require.NoError(t, err) require.Len(t, operations.Edges, 8) // Operations are ordered by ID ascending - assert.Equal(t, "opxdr1", operations.Edges[0].Node.OperationXDR) - assert.Equal(t, "opxdr2", operations.Edges[1].Node.OperationXDR) - assert.Equal(t, "opxdr3", operations.Edges[2].Node.OperationXDR) - assert.Equal(t, "opxdr4", operations.Edges[3].Node.OperationXDR) + assert.Equal(t, testOpXDR(1), operations.Edges[0].Node.OperationXDR.String()) + assert.Equal(t, testOpXDR(2), operations.Edges[1].Node.OperationXDR.String()) + assert.Equal(t, testOpXDR(3), operations.Edges[2].Node.OperationXDR.String()) + assert.Equal(t, testOpXDR(4), operations.Edges[3].Node.OperationXDR.String()) }) t.Run("get operations with first/after limit and cursor", func(t *testing.T) { @@ -290,8 +297,8 @@ func TestQueryResolver_Operations(t *testing.T) { ops, err := resolver.Operations(ctx, &first, nil, nil, nil) require.NoError(t, err) assert.Len(t, ops.Edges, 2) - assert.Equal(t, "opxdr1", ops.Edges[0].Node.OperationXDR) - assert.Equal(t, "opxdr2", ops.Edges[1].Node.OperationXDR) + assert.Equal(t, testOpXDR(1), ops.Edges[0].Node.OperationXDR.String()) + assert.Equal(t, testOpXDR(2), ops.Edges[1].Node.OperationXDR.String()) assert.True(t, ops.PageInfo.HasNextPage) assert.False(t, ops.PageInfo.HasPreviousPage) @@ -302,7 +309,7 @@ func TestQueryResolver_Operations(t *testing.T) { ops, err = resolver.Operations(ctx, &first, nextCursor, nil, nil) require.NoError(t, err) assert.Len(t, ops.Edges, 1) - assert.Equal(t, "opxdr3", ops.Edges[0].Node.OperationXDR) + assert.Equal(t, testOpXDR(3), ops.Edges[0].Node.OperationXDR.String()) assert.True(t, ops.PageInfo.HasNextPage) assert.True(t, ops.PageInfo.HasPreviousPage) @@ -313,11 +320,11 @@ func TestQueryResolver_Operations(t *testing.T) { ops, err = resolver.Operations(ctx, &first, nextCursor, nil, nil) require.NoError(t, err) assert.Len(t, ops.Edges, 5) - assert.Equal(t, "opxdr4", ops.Edges[0].Node.OperationXDR) - assert.Equal(t, "opxdr5", ops.Edges[1].Node.OperationXDR) - assert.Equal(t, "opxdr6", ops.Edges[2].Node.OperationXDR) - assert.Equal(t, "opxdr7", ops.Edges[3].Node.OperationXDR) - assert.Equal(t, "opxdr8", ops.Edges[4].Node.OperationXDR) + assert.Equal(t, testOpXDR(4), ops.Edges[0].Node.OperationXDR.String()) + assert.Equal(t, testOpXDR(5), ops.Edges[1].Node.OperationXDR.String()) + assert.Equal(t, testOpXDR(6), ops.Edges[2].Node.OperationXDR.String()) + assert.Equal(t, testOpXDR(7), ops.Edges[3].Node.OperationXDR.String()) + assert.Equal(t, testOpXDR(8), ops.Edges[4].Node.OperationXDR.String()) assert.False(t, ops.PageInfo.HasNextPage) assert.True(t, ops.PageInfo.HasPreviousPage) }) @@ -329,8 +336,8 @@ func TestQueryResolver_Operations(t *testing.T) { require.NoError(t, err) assert.Len(t, ops.Edges, 2) // With backward pagination, we get the last 2 items - assert.Equal(t, "opxdr7", ops.Edges[0].Node.OperationXDR) - assert.Equal(t, "opxdr8", ops.Edges[1].Node.OperationXDR) + assert.Equal(t, testOpXDR(7), ops.Edges[0].Node.OperationXDR.String()) + assert.Equal(t, testOpXDR(8), ops.Edges[1].Node.OperationXDR.String()) assert.False(t, ops.PageInfo.HasNextPage) assert.True(t, ops.PageInfo.HasPreviousPage) @@ -341,7 +348,7 @@ func TestQueryResolver_Operations(t *testing.T) { ops, err = resolver.Operations(ctx, nil, nil, &last, prevCursor) require.NoError(t, err) assert.Len(t, ops.Edges, 1) - assert.Equal(t, "opxdr6", ops.Edges[0].Node.OperationXDR) + assert.Equal(t, testOpXDR(6), ops.Edges[0].Node.OperationXDR.String()) assert.True(t, ops.PageInfo.HasNextPage) assert.True(t, ops.PageInfo.HasPreviousPage) @@ -352,11 +359,11 @@ func TestQueryResolver_Operations(t *testing.T) { require.NoError(t, err) // There are 5 operations before (2,1): (2,2), (3,1), (3,2), (4,1), (4,2) assert.Len(t, ops.Edges, 5) - assert.Equal(t, "opxdr1", ops.Edges[0].Node.OperationXDR) - assert.Equal(t, "opxdr2", ops.Edges[1].Node.OperationXDR) - assert.Equal(t, "opxdr3", ops.Edges[2].Node.OperationXDR) - assert.Equal(t, "opxdr4", ops.Edges[3].Node.OperationXDR) - assert.Equal(t, "opxdr5", ops.Edges[4].Node.OperationXDR) + assert.Equal(t, testOpXDR(1), ops.Edges[0].Node.OperationXDR.String()) + assert.Equal(t, testOpXDR(2), ops.Edges[1].Node.OperationXDR.String()) + assert.Equal(t, testOpXDR(3), ops.Edges[2].Node.OperationXDR.String()) + assert.Equal(t, testOpXDR(4), ops.Edges[3].Node.OperationXDR.String()) + assert.Equal(t, testOpXDR(5), ops.Edges[4].Node.OperationXDR.String()) assert.True(t, ops.PageInfo.HasNextPage) assert.False(t, ops.PageInfo.HasPreviousPage) }) @@ -442,7 +449,7 @@ func TestQueryResolver_OperationByID(t *testing.T) { require.NoError(t, err) assert.Equal(t, toid.New(1000, 1, 1).ToInt64(), op.ID) - assert.Equal(t, "opxdr1", op.OperationXDR) + assert.Equal(t, testOpXDR(1), op.OperationXDR.String()) assert.Equal(t, uint32(1), op.LedgerNumber) }) diff --git a/internal/serve/graphql/resolvers/resolver.go b/internal/serve/graphql/resolvers/resolver.go index 15a4d4beb..b853bc102 100644 --- a/internal/serve/graphql/resolvers/resolver.go +++ b/internal/serve/graphql/resolvers/resolver.go @@ -109,6 +109,16 @@ func (r *Resolver) resolveNullableString(field sql.NullString) *string { return nil } +// resolveNullableAddress resolves nullable address fields from the database +// Returns pointer to string if valid, nil if null +func (r *Resolver) resolveNullableAddress(field types.NullAddressBytea) *string { + if field.Valid { + s := field.String() + return &s + } + return nil +} + // resolveRequiredString resolves required string fields from the database // Returns empty string if null to satisfy non-nullable GraphQL fields func (r *Resolver) resolveRequiredString(field sql.NullString) string { diff --git a/internal/serve/graphql/resolvers/statechange.resolvers.go b/internal/serve/graphql/resolvers/statechange.resolvers.go index f70a83050..9583c2b93 100644 --- a/internal/serve/graphql/resolvers/statechange.resolvers.go +++ b/internal/serve/graphql/resolvers/statechange.resolvers.go @@ -39,7 +39,7 @@ func (r *accountChangeResolver) Transaction(ctx context.Context, obj *types.Acco // FunderAddress is the resolver for the funderAddress field. func (r *accountChangeResolver) FunderAddress(ctx context.Context, obj *types.AccountStateChangeModel) (*string, error) { - return r.resolveNullableString(obj.FunderAccountID), nil + return r.resolveNullableAddress(obj.FunderAccountID), nil } // Type is the resolver for the type field. @@ -177,12 +177,12 @@ func (r *reservesChangeResolver) Transaction(ctx context.Context, obj *types.Res // SponsoredAddress is the resolver for the sponsoredAddress field. func (r *reservesChangeResolver) SponsoredAddress(ctx context.Context, obj *types.ReservesStateChangeModel) (*string, error) { - return r.resolveNullableString(obj.SponsoredAccountID), nil + return r.resolveNullableAddress(obj.SponsoredAccountID), nil } // SponsorAddress is the resolver for the sponsorAddress field. func (r *reservesChangeResolver) SponsorAddress(ctx context.Context, obj *types.ReservesStateChangeModel) (*string, error) { - return r.resolveNullableString(obj.SponsorAccountID), nil + return r.resolveNullableAddress(obj.SponsorAccountID), nil } // LiquidityPoolID is the resolver for the liquidityPoolID field. @@ -232,7 +232,7 @@ func (r *signerChangeResolver) Transaction(ctx context.Context, obj *types.Signe // SignerAddress is the resolver for the signerAddress field. func (r *signerChangeResolver) SignerAddress(ctx context.Context, obj *types.SignerStateChangeModel) (*string, error) { - return r.resolveNullableString(obj.SignerAccountID), nil + return r.resolveNullableAddress(obj.SignerAccountID), nil } // SignerWeights is the resolver for the signerWeights field. diff --git a/internal/serve/graphql/resolvers/statechange_resolvers_test.go b/internal/serve/graphql/resolvers/statechange_resolvers_test.go index 8330b767c..8076e5ace 100644 --- a/internal/serve/graphql/resolvers/statechange_resolvers_test.go +++ b/internal/serve/graphql/resolvers/statechange_resolvers_test.go @@ -255,7 +255,7 @@ func TestStateChangeResolver_Account(t *testing.T) { account, err := resolver.Account(ctx, &parentSC) require.NoError(t, err) - assert.Equal(t, "test-account", account.StellarAddress) + assert.Equal(t, sharedTestAccountAddress, string(account.StellarAddress)) }) t.Run("nil state change panics", func(t *testing.T) { @@ -376,7 +376,7 @@ func TestStateChangeResolver_Transaction(t *testing.T) { tx, err := resolver.Transaction(ctx, &parentSC) require.NoError(t, err) - assert.Equal(t, "tx1", tx.Hash) + assert.Equal(t, testTxHash1, tx.Hash.String()) }) t.Run("nil state change panics", func(t *testing.T) { diff --git a/internal/serve/graphql/resolvers/transaction.resolvers.go b/internal/serve/graphql/resolvers/transaction.resolvers.go index 7d83bf8ff..eeb7dec2b 100644 --- a/internal/serve/graphql/resolvers/transaction.resolvers.go +++ b/internal/serve/graphql/resolvers/transaction.resolvers.go @@ -15,6 +15,11 @@ import ( "github.com/stellar/wallet-backend/internal/serve/middleware" ) +// Hash is the resolver for the hash field. +func (r *transactionResolver) Hash(ctx context.Context, obj *types.Transaction) (string, error) { + return obj.Hash.String(), nil +} + // Operations is the resolver for the operations field. // This is a field resolver for the "operations" field on a Transaction object // It's called when a GraphQL query requests the operations within a transaction diff --git a/internal/serve/graphql/resolvers/transaction_resolvers_test.go b/internal/serve/graphql/resolvers/transaction_resolvers_test.go index cac40c729..275cbefcb 100644 --- a/internal/serve/graphql/resolvers/transaction_resolvers_test.go +++ b/internal/serve/graphql/resolvers/transaction_resolvers_test.go @@ -2,6 +2,8 @@ package resolvers import ( "context" + "encoding/base64" + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -17,6 +19,11 @@ import ( "github.com/stellar/wallet-backend/internal/serve/middleware" ) +// testOpXDR returns the expected base64-encoded XDR for test operation N +func testOpXDRTx(n int) string { + return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("opxdr%d", n))) +} + func TestTransactionResolver_Operations(t *testing.T) { mockMetricsService := &metrics.MockMetricsService{} mockMetricsService.On("IncDBQuery", "BatchGetByToID", "operations").Return() @@ -32,7 +39,7 @@ func TestTransactionResolver_Operations(t *testing.T) { }, }} // ToID=toid.New(1000, 1, 0) matches the test data setup in test_utils.go (testLedger=1000, i=0) - parentTx := &types.Transaction{Hash: "tx1", ToID: toid.New(1000, 1, 0).ToInt64()} + parentTx := &types.Transaction{Hash: "1376b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", ToID: toid.New(1000, 1, 0).ToInt64()} t.Run("success", func(t *testing.T) { loaders := dataloaders.NewDataloaders(resolver.models) @@ -42,8 +49,8 @@ func TestTransactionResolver_Operations(t *testing.T) { require.NoError(t, err) require.Len(t, operations.Edges, 2) - assert.Equal(t, "opxdr1", operations.Edges[0].Node.OperationXDR) - assert.Equal(t, "opxdr2", operations.Edges[1].Node.OperationXDR) + assert.Equal(t, testOpXDRTx(1), operations.Edges[0].Node.OperationXDR.String()) + assert.Equal(t, testOpXDRTx(2), operations.Edges[1].Node.OperationXDR.String()) }) t.Run("nil transaction panics", func(t *testing.T) { @@ -56,7 +63,7 @@ func TestTransactionResolver_Operations(t *testing.T) { }) t.Run("transaction with no operations", func(t *testing.T) { - nonExistentTx := &types.Transaction{Hash: "non-existent-tx", ToID: 999999} + nonExistentTx := &types.Transaction{Hash: "2376b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", ToID: 999999} loaders := dataloaders.NewDataloaders(resolver.models) ctx := context.WithValue(getTestCtx("operations", []string{"id"}), middleware.LoadersKey, loaders) @@ -73,7 +80,7 @@ func TestTransactionResolver_Operations(t *testing.T) { ops, err := resolver.Operations(ctx, parentTx, &first, nil, nil, nil) require.NoError(t, err) assert.Len(t, ops.Edges, 1) - assert.Equal(t, "opxdr1", ops.Edges[0].Node.OperationXDR) + assert.Equal(t, testOpXDRTx(1), ops.Edges[0].Node.OperationXDR.String()) assert.True(t, ops.PageInfo.HasNextPage) assert.False(t, ops.PageInfo.HasPreviousPage) @@ -83,7 +90,7 @@ func TestTransactionResolver_Operations(t *testing.T) { ops, err = resolver.Operations(ctx, parentTx, &first, nextCursor, nil, nil) require.NoError(t, err) assert.Len(t, ops.Edges, 1) - assert.Equal(t, "opxdr2", ops.Edges[0].Node.OperationXDR) + assert.Equal(t, testOpXDRTx(2), ops.Edges[0].Node.OperationXDR.String()) assert.False(t, ops.PageInfo.HasNextPage) assert.True(t, ops.PageInfo.HasPreviousPage) }) @@ -95,7 +102,7 @@ func TestTransactionResolver_Operations(t *testing.T) { ops, err := resolver.Operations(ctx, parentTx, nil, nil, &last, nil) require.NoError(t, err) assert.Len(t, ops.Edges, 1) - assert.Equal(t, "opxdr2", ops.Edges[0].Node.OperationXDR) + assert.Equal(t, testOpXDRTx(2), ops.Edges[0].Node.OperationXDR.String()) assert.False(t, ops.PageInfo.HasNextPage) assert.True(t, ops.PageInfo.HasPreviousPage) @@ -105,7 +112,7 @@ func TestTransactionResolver_Operations(t *testing.T) { ops, err = resolver.Operations(ctx, parentTx, nil, nil, &last, prevCursor) require.NoError(t, err) assert.Len(t, ops.Edges, 1) - assert.Equal(t, "opxdr1", ops.Edges[0].Node.OperationXDR) + assert.Equal(t, testOpXDRTx(1), ops.Edges[0].Node.OperationXDR.String()) assert.True(t, ops.PageInfo.HasNextPage) assert.False(t, ops.PageInfo.HasPreviousPage) }) @@ -167,7 +174,7 @@ func TestTransactionResolver_Accounts(t *testing.T) { }, }, }} - parentTx := &types.Transaction{ToID: toid.New(1000, 1, 0).ToInt64(), Hash: "tx1"} + parentTx := &types.Transaction{ToID: toid.New(1000, 1, 0).ToInt64(), Hash: "1376b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877"} t.Run("success", func(t *testing.T) { loaders := dataloaders.NewDataloaders(resolver.models) @@ -177,7 +184,7 @@ func TestTransactionResolver_Accounts(t *testing.T) { require.NoError(t, err) require.Len(t, accounts, 1) - assert.Equal(t, "test-account", accounts[0].StellarAddress) + assert.Equal(t, sharedTestAccountAddress, string(accounts[0].StellarAddress)) }) t.Run("nil transaction panics", func(t *testing.T) { @@ -190,7 +197,7 @@ func TestTransactionResolver_Accounts(t *testing.T) { }) t.Run("transaction with no associated accounts", func(t *testing.T) { - nonExistentTx := &types.Transaction{Hash: "non-existent-tx"} + nonExistentTx := &types.Transaction{Hash: "2376b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877"} loaders := dataloaders.NewDataloaders(resolver.models) ctx := context.WithValue(getTestCtx("accounts", []string{"address"}), middleware.LoadersKey, loaders) @@ -219,8 +226,8 @@ func TestTransactionResolver_StateChanges(t *testing.T) { }, }, }} - parentTx := &types.Transaction{Hash: "tx1", ToID: toid.New(1000, 1, 0).ToInt64()} - nonExistentTx := &types.Transaction{Hash: "non-existent-tx", ToID: 0} + parentTx := &types.Transaction{Hash: "1376b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", ToID: toid.New(1000, 1, 0).ToInt64()} + nonExistentTx := &types.Transaction{Hash: "2376b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", ToID: 0} t.Run("success without pagination", func(t *testing.T) { loaders := dataloaders.NewDataloaders(resolver.models) diff --git a/internal/serve/graphql/schema/operation.graphqls b/internal/serve/graphql/schema/operation.graphqls index 0755fc382..dcec1b379 100644 --- a/internal/serve/graphql/schema/operation.graphqls +++ b/internal/serve/graphql/schema/operation.graphqls @@ -3,7 +3,7 @@ type Operation{ id: Int64! operationType: OperationType! - operationXdr: String! + operationXdr: String! @goField(forceResolver: true) resultCode: String! successful: Boolean! ledgerNumber: UInt32! diff --git a/internal/serve/graphql/schema/transaction.graphqls b/internal/serve/graphql/schema/transaction.graphqls index 6a2bd4585..a5c74d6a0 100644 --- a/internal/serve/graphql/schema/transaction.graphqls +++ b/internal/serve/graphql/schema/transaction.graphqls @@ -1,7 +1,7 @@ # GraphQL Transaction type - represents a blockchain transaction # gqlgen generates Go structs from this schema definition type Transaction{ - hash: String! + hash: String! @goField(forceResolver: true) envelopeXdr: String feeCharged: Int64! resultCode: String! diff --git a/internal/services/account_service_test.go b/internal/services/account_service_test.go index 5bbebc576..252ac6209 100644 --- a/internal/services/account_service_test.go +++ b/internal/services/account_service_test.go @@ -14,6 +14,7 @@ import ( "github.com/stellar/wallet-backend/internal/data" "github.com/stellar/wallet-backend/internal/db" "github.com/stellar/wallet-backend/internal/db/dbtest" + "github.com/stellar/wallet-backend/internal/indexer/types" "github.com/stellar/wallet-backend/internal/metrics" ) @@ -42,12 +43,12 @@ func TestAccountRegister(t *testing.T) { err = accountService.RegisterAccount(ctx, address) require.NoError(t, err) - var dbAddress sql.NullString - err = dbConnectionPool.GetContext(ctx, &dbAddress, "SELECT stellar_address FROM accounts WHERE stellar_address = $1", address) + var dbAddress types.NullAddressBytea + err = dbConnectionPool.GetContext(ctx, &dbAddress, "SELECT stellar_address FROM accounts WHERE stellar_address = $1", types.AddressBytea(address)) require.NoError(t, err) assert.True(t, dbAddress.Valid) - assert.Equal(t, address, dbAddress.String) + assert.Equal(t, address, dbAddress.String()) }) t.Run("duplicate registration fails", func(t *testing.T) { @@ -122,7 +123,7 @@ func TestAccountDeregister(t *testing.T) { ctx := context.Background() address := keypair.MustRandom().Address() - result, err := dbConnectionPool.ExecContext(ctx, "Insert INTO accounts (stellar_address) VALUES ($1)", address) + result, err := dbConnectionPool.ExecContext(ctx, "Insert INTO accounts (stellar_address) VALUES ($1)", types.AddressBytea(address)) require.NoError(t, err) rowAffected, err := result.RowsAffected() require.NoError(t, err) diff --git a/internal/services/ingest.go b/internal/services/ingest.go index 1dcf4c412..1b6bb4f83 100644 --- a/internal/services/ingest.go +++ b/internal/services/ingest.go @@ -300,7 +300,7 @@ func (m *ingestService) filterByRegisteredAccounts( // Filter state changes: include if account is registered filteredSC := make([]types.StateChange, 0) for _, sc := range stateChanges { - if registeredAccounts.Contains(sc.AccountID) { + if registeredAccounts.Contains(string(sc.AccountID)) { filteredSC = append(filteredSC, sc) } } diff --git a/internal/utils/sql.go b/internal/utils/sql.go index a965b2e03..824dc7d61 100644 --- a/internal/utils/sql.go +++ b/internal/utils/sql.go @@ -3,6 +3,8 @@ package utils import ( "database/sql" "time" + + "github.com/stellar/wallet-backend/internal/indexer/types" ) func SQLNullString(s string) sql.NullString { @@ -18,3 +20,10 @@ func SQLNullTime(t time.Time) sql.NullTime { Valid: !t.IsZero(), } } + +func NullAddressBytea(s string) types.NullAddressBytea { + return types.NullAddressBytea{ + AddressBytea: types.AddressBytea(s), + Valid: s != "", + } +} From 1224c17b6c1e8e020cae1da3a781e05a75943f66 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 10 Feb 2026 14:42:17 -0500 Subject: [PATCH 16/77] Migrate TEXT columns to BYTEA in transaction/operation/statechange schemas Change hash, account_id, operation_xdr, and address columns from TEXT to BYTEA type while preserving all TimescaleDB hypertable configuration, composite primary keys, and chunk settings. --- .../db/migrations/2025-06-10.2-transactions.sql | 4 ++-- internal/db/migrations/2025-06-10.3-operations.sql | 4 ++-- .../db/migrations/2025-06-10.4-statechanges.sql | 14 +++++++------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/db/migrations/2025-06-10.2-transactions.sql b/internal/db/migrations/2025-06-10.2-transactions.sql index 8330481e8..09c16f764 100644 --- a/internal/db/migrations/2025-06-10.2-transactions.sql +++ b/internal/db/migrations/2025-06-10.2-transactions.sql @@ -4,7 +4,7 @@ CREATE TABLE transactions ( ledger_created_at TIMESTAMPTZ NOT NULL, to_id BIGINT NOT NULL, - hash TEXT NOT NULL, + hash BYTEA NOT NULL, envelope_xdr TEXT, fee_charged BIGINT NOT NULL, result_code TEXT NOT NULL, @@ -27,7 +27,7 @@ CREATE INDEX idx_transactions_to_id ON transactions(to_id); CREATE TABLE transactions_accounts ( ledger_created_at TIMESTAMPTZ NOT NULL, tx_to_id BIGINT NOT NULL, - account_id TEXT NOT NULL, + account_id BYTEA NOT NULL, PRIMARY KEY (ledger_created_at, account_id, tx_to_id) ) WITH ( tsdb.hypertable, diff --git a/internal/db/migrations/2025-06-10.3-operations.sql b/internal/db/migrations/2025-06-10.3-operations.sql index 6b5cec85c..b4ccd1800 100644 --- a/internal/db/migrations/2025-06-10.3-operations.sql +++ b/internal/db/migrations/2025-06-10.3-operations.sql @@ -18,7 +18,7 @@ CREATE TABLE operations ( 'INVOKE_HOST_FUNCTION', 'EXTEND_FOOTPRINT_TTL', 'RESTORE_FOOTPRINT' ) ), - operation_xdr TEXT, + operation_xdr BYTEA, result_code TEXT NOT NULL, successful BOOLEAN NOT NULL, ledger_number INTEGER NOT NULL, @@ -38,7 +38,7 @@ CREATE INDEX idx_operations_id ON operations(id); CREATE TABLE operations_accounts ( ledger_created_at TIMESTAMPTZ NOT NULL, operation_id BIGINT NOT NULL, - account_id TEXT NOT NULL, + account_id BYTEA NOT NULL, PRIMARY KEY (ledger_created_at, account_id, operation_id) ) WITH ( tsdb.hypertable, diff --git a/internal/db/migrations/2025-06-10.4-statechanges.sql b/internal/db/migrations/2025-06-10.4-statechanges.sql index d167790c6..509564b28 100644 --- a/internal/db/migrations/2025-06-10.4-statechanges.sql +++ b/internal/db/migrations/2025-06-10.4-statechanges.sql @@ -23,15 +23,15 @@ CREATE TABLE state_changes ( ), ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), ledger_number INTEGER NOT NULL, - account_id TEXT NOT NULL, + account_id BYTEA NOT NULL, token_id TEXT, amount TEXT, - signer_account_id TEXT, - spender_account_id TEXT, - sponsored_account_id TEXT, - sponsor_account_id TEXT, - deployer_account_id TEXT, - funder_account_id TEXT, + signer_account_id BYTEA, + spender_account_id BYTEA, + sponsored_account_id BYTEA, + sponsor_account_id BYTEA, + deployer_account_id BYTEA, + funder_account_id BYTEA, claimable_balance_id TEXT, liquidity_pool_id TEXT, sponsored_data TEXT, From 9ef4af806b17cfd79e93d59349d38e295d2966d3 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 10 Feb 2026 14:46:06 -0500 Subject: [PATCH 17/77] Apply BYTEA type conversions to data layer (transactions, operations, statechanges) Convert hash, account_id, operation_xdr, and address columns from TEXT to BYTEA in BatchInsert, BatchCopy, and query methods. Uses HashBytea, AddressBytea, and pgtypeBytesFromNullAddressBytea for type conversions. Preserves TimescaleDB junction table patterns (ledger_created_at columns, composite primary keys, parameter numbering). --- internal/data/operations.go | 28 ++++--- internal/data/statechanges.go | 133 +++++++++++++++++++++++----------- internal/data/transactions.go | 49 +++++++++---- 3 files changed, 144 insertions(+), 66 deletions(-) diff --git a/internal/data/operations.go b/internal/data/operations.go index eeec16b26..313963035 100644 --- a/internal/data/operations.go +++ b/internal/data/operations.go @@ -284,7 +284,7 @@ func (m *OperationModel) BatchInsert( // 1. Flatten the operations into parallel slices ids := make([]int64, len(operations)) operationTypes := make([]string, len(operations)) - operationXDRs := make([]string, len(operations)) + operationXDRs := make([][]byte, len(operations)) resultCodes := make([]string, len(operations)) successfulFlags := make([]bool, len(operations)) ledgerNumbers := make([]uint32, len(operations)) @@ -293,7 +293,7 @@ func (m *OperationModel) BatchInsert( for i, op := range operations { ids[i] = op.ID operationTypes[i] = string(op.OperationType) - operationXDRs[i] = op.OperationXDR + operationXDRs[i] = []byte(op.OperationXDR) resultCodes[i] = op.ResultCode successfulFlags[i] = op.Successful ledgerNumbers[i] = op.LedgerNumber @@ -306,15 +306,19 @@ func (m *OperationModel) BatchInsert( ledgerCreatedAtByOpID[op.ID] = op.LedgerCreatedAt } - // 3. Flatten the stellarAddressesByOpID into parallel slices + // 3. Flatten the stellarAddressesByOpID into parallel slices, converting to BYTEA var opIDs []int64 - var stellarAddresses []string + var stellarAddressBytes [][]byte var oaLedgerCreatedAts []time.Time for opID, addresses := range stellarAddressesByOpID { ledgerCreatedAt := ledgerCreatedAtByOpID[opID] for address := range addresses.Iter() { opIDs = append(opIDs, opID) - stellarAddresses = append(stellarAddresses, address) + addrBytes, err := types.AddressBytea(address).Value() + if err != nil { + return nil, fmt.Errorf("converting address %s to bytes: %w", address, err) + } + stellarAddressBytes = append(stellarAddressBytes, addrBytes.([]byte)) oaLedgerCreatedAts = append(oaLedgerCreatedAts, ledgerCreatedAt) } } @@ -332,7 +336,7 @@ func (m *OperationModel) BatchInsert( SELECT UNNEST($1::bigint[]) AS id, UNNEST($2::text[]) AS operation_type, - UNNEST($3::text[]) AS operation_xdr, + UNNEST($3::bytea[]) AS operation_xdr, UNNEST($4::text[]) AS result_code, UNNEST($5::boolean[]) AS successful, UNNEST($6::bigint[]) AS ledger_number, @@ -352,7 +356,7 @@ func (m *OperationModel) BatchInsert( SELECT UNNEST($8::timestamptz[]) AS ledger_created_at, UNNEST($9::bigint[]) AS op_id, - UNNEST($10::text[]) AS account_id + UNNEST($10::bytea[]) AS account_id ) oa ON CONFLICT DO NOTHING ) @@ -373,7 +377,7 @@ func (m *OperationModel) BatchInsert( pq.Array(ledgerCreatedAts), pq.Array(oaLedgerCreatedAts), pq.Array(opIDs), - pq.Array(stellarAddresses), + pq.Array(stellarAddressBytes), ) duration := time.Since(start).Seconds() for _, dbTableName := range []string{"operations", "operations_accounts"} { @@ -425,7 +429,7 @@ func (m *OperationModel) BatchCopy( return []any{ pgtype.Int8{Int64: op.ID, Valid: true}, pgtype.Text{String: string(op.OperationType), Valid: true}, - pgtype.Text{String: op.OperationXDR, Valid: true}, + []byte(op.OperationXDR), pgtype.Text{String: op.ResultCode, Valid: true}, pgtype.Bool{Bool: op.Successful, Valid: true}, pgtype.Int4{Int32: int32(op.LedgerNumber), Valid: true}, @@ -455,10 +459,14 @@ func (m *OperationModel) BatchCopy( ledgerCreatedAtPgtype := pgtype.Timestamptz{Time: ledgerCreatedAt, Valid: true} opIDPgtype := pgtype.Int8{Int64: opID, Valid: true} for _, addr := range addresses.ToSlice() { + addrBytes, err := types.AddressBytea(addr).Value() + if err != nil { + return 0, fmt.Errorf("converting address %s to bytes: %w", addr, err) + } oaRows = append(oaRows, []any{ ledgerCreatedAtPgtype, opIDPgtype, - pgtype.Text{String: addr, Valid: true}, + addrBytes, }) } } diff --git a/internal/data/statechanges.go b/internal/data/statechanges.go index a181dd8db..a4a10a4bf 100644 --- a/internal/data/statechanges.go +++ b/internal/data/statechanges.go @@ -26,7 +26,7 @@ type StateChangeModel struct { func (m *StateChangeModel) BatchGetByAccountAddress(ctx context.Context, accountAddress string, txHash *string, operationID *int64, category *string, reason *string, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_order") var queryBuilder strings.Builder - args := []interface{}{accountAddress} + args := []interface{}{types.AddressBytea(accountAddress)} argIndex := 2 queryBuilder.WriteString(fmt.Sprintf(` @@ -38,7 +38,7 @@ func (m *StateChangeModel) BatchGetByAccountAddress(ctx context.Context, account // Add transaction hash filter if provided (uses subquery to find to_id by hash) if txHash != nil { queryBuilder.WriteString(fmt.Sprintf(" AND to_id = (SELECT to_id FROM transactions WHERE hash = $%d)", argIndex)) - args = append(args, *txHash) + args = append(args, types.HashBytea(*txHash)) argIndex++ } @@ -184,16 +184,16 @@ func (m *StateChangeModel) BatchInsert( reasons := make([]*string, len(stateChanges)) ledgerCreatedAts := make([]time.Time, len(stateChanges)) ledgerNumbers := make([]int, len(stateChanges)) - accountIDs := make([]string, len(stateChanges)) + accountIDBytes := make([][]byte, len(stateChanges)) operationIDs := make([]int64, len(stateChanges)) tokenIDs := make([]*string, len(stateChanges)) amounts := make([]*string, len(stateChanges)) - signerAccountIDs := make([]*string, len(stateChanges)) - spenderAccountIDs := make([]*string, len(stateChanges)) - sponsoredAccountIDs := make([]*string, len(stateChanges)) - sponsorAccountIDs := make([]*string, len(stateChanges)) - deployerAccountIDs := make([]*string, len(stateChanges)) - funderAccountIDs := make([]*string, len(stateChanges)) + signerAccountIDBytes := make([][]byte, len(stateChanges)) + spenderAccountIDBytes := make([][]byte, len(stateChanges)) + sponsoredAccountIDBytes := make([][]byte, len(stateChanges)) + sponsorAccountIDBytes := make([][]byte, len(stateChanges)) + deployerAccountIDBytes := make([][]byte, len(stateChanges)) + funderAccountIDBytes := make([][]byte, len(stateChanges)) claimableBalanceIDs := make([]*string, len(stateChanges)) liquidityPoolIDs := make([]*string, len(stateChanges)) sponsoredDataValues := make([]*string, len(stateChanges)) @@ -212,9 +212,15 @@ func (m *StateChangeModel) BatchInsert( categories[i] = string(sc.StateChangeCategory) ledgerCreatedAts[i] = sc.LedgerCreatedAt ledgerNumbers[i] = int(sc.LedgerNumber) - accountIDs[i] = sc.AccountID operationIDs[i] = sc.OperationID + // Convert account_id to BYTEA (required field) + addrBytes, err := sc.AccountID.Value() + if err != nil { + return nil, fmt.Errorf("converting account_id: %w", err) + } + accountIDBytes[i] = addrBytes.([]byte) + // Nullable fields if sc.StateChangeReason != nil { reason := string(*sc.StateChangeReason) @@ -226,23 +232,31 @@ func (m *StateChangeModel) BatchInsert( if sc.Amount.Valid { amounts[i] = &sc.Amount.String } - if sc.SignerAccountID.Valid { - signerAccountIDs[i] = &sc.SignerAccountID.String + + // Convert nullable account_id fields to BYTEA + signerAccountIDBytes[i], err = pgtypeBytesFromNullAddressBytea(sc.SignerAccountID) + if err != nil { + return nil, fmt.Errorf("converting signer_account_id: %w", err) } - if sc.SpenderAccountID.Valid { - spenderAccountIDs[i] = &sc.SpenderAccountID.String + spenderAccountIDBytes[i], err = pgtypeBytesFromNullAddressBytea(sc.SpenderAccountID) + if err != nil { + return nil, fmt.Errorf("converting spender_account_id: %w", err) } - if sc.SponsoredAccountID.Valid { - sponsoredAccountIDs[i] = &sc.SponsoredAccountID.String + sponsoredAccountIDBytes[i], err = pgtypeBytesFromNullAddressBytea(sc.SponsoredAccountID) + if err != nil { + return nil, fmt.Errorf("converting sponsored_account_id: %w", err) } - if sc.SponsorAccountID.Valid { - sponsorAccountIDs[i] = &sc.SponsorAccountID.String + sponsorAccountIDBytes[i], err = pgtypeBytesFromNullAddressBytea(sc.SponsorAccountID) + if err != nil { + return nil, fmt.Errorf("converting sponsor_account_id: %w", err) } - if sc.DeployerAccountID.Valid { - deployerAccountIDs[i] = &sc.DeployerAccountID.String + deployerAccountIDBytes[i], err = pgtypeBytesFromNullAddressBytea(sc.DeployerAccountID) + if err != nil { + return nil, fmt.Errorf("converting deployer_account_id: %w", err) } - if sc.FunderAccountID.Valid { - funderAccountIDs[i] = &sc.FunderAccountID.String + funderAccountIDBytes[i], err = pgtypeBytesFromNullAddressBytea(sc.FunderAccountID) + if err != nil { + return nil, fmt.Errorf("converting funder_account_id: %w", err) } if sc.ClaimableBalanceID.Valid { claimableBalanceIDs[i] = &sc.ClaimableBalanceID.String @@ -289,16 +303,16 @@ func (m *StateChangeModel) BatchInsert( UNNEST($4::text[]) AS state_change_reason, UNNEST($5::timestamptz[]) AS ledger_created_at, UNNEST($6::integer[]) AS ledger_number, - UNNEST($7::text[]) AS account_id, + UNNEST($7::bytea[]) AS account_id, UNNEST($8::bigint[]) AS operation_id, UNNEST($9::text[]) AS token_id, UNNEST($10::text[]) AS amount, - UNNEST($11::text[]) AS signer_account_id, - UNNEST($12::text[]) AS spender_account_id, - UNNEST($13::text[]) AS sponsored_account_id, - UNNEST($14::text[]) AS sponsor_account_id, - UNNEST($15::text[]) AS deployer_account_id, - UNNEST($16::text[]) AS funder_account_id, + UNNEST($11::bytea[]) AS signer_account_id, + UNNEST($12::bytea[]) AS spender_account_id, + UNNEST($13::bytea[]) AS sponsored_account_id, + UNNEST($14::bytea[]) AS sponsor_account_id, + UNNEST($15::bytea[]) AS deployer_account_id, + UNNEST($16::bytea[]) AS funder_account_id, UNNEST($17::text[]) AS claimable_balance_id, UNNEST($18::text[]) AS liquidity_pool_id, UNNEST($19::text[]) AS sponsored_data, @@ -342,16 +356,16 @@ func (m *StateChangeModel) BatchInsert( pq.Array(reasons), pq.Array(ledgerCreatedAts), pq.Array(ledgerNumbers), - pq.Array(accountIDs), + pq.Array(accountIDBytes), pq.Array(operationIDs), pq.Array(tokenIDs), pq.Array(amounts), - pq.Array(signerAccountIDs), - pq.Array(spenderAccountIDs), - pq.Array(sponsoredAccountIDs), - pq.Array(sponsorAccountIDs), - pq.Array(deployerAccountIDs), - pq.Array(funderAccountIDs), + pq.Array(signerAccountIDBytes), + pq.Array(spenderAccountIDBytes), + pq.Array(sponsoredAccountIDBytes), + pq.Array(sponsorAccountIDBytes), + pq.Array(deployerAccountIDBytes), + pq.Array(funderAccountIDBytes), pq.Array(claimableBalanceIDs), pq.Array(liquidityPoolIDs), pq.Array(sponsoredDataValues), @@ -410,6 +424,39 @@ func (m *StateChangeModel) BatchCopy( }, pgx.CopyFromSlice(len(stateChanges), func(i int) ([]any, error) { sc := stateChanges[i] + + // Convert account_id to BYTEA (required field) + accountBytes, err := sc.AccountID.Value() + if err != nil { + return nil, fmt.Errorf("converting account_id: %w", err) + } + + // Convert nullable account_id fields to BYTEA + signerBytes, err := pgtypeBytesFromNullAddressBytea(sc.SignerAccountID) + if err != nil { + return nil, fmt.Errorf("converting signer_account_id: %w", err) + } + spenderBytes, err := pgtypeBytesFromNullAddressBytea(sc.SpenderAccountID) + if err != nil { + return nil, fmt.Errorf("converting spender_account_id: %w", err) + } + sponsoredBytes, err := pgtypeBytesFromNullAddressBytea(sc.SponsoredAccountID) + if err != nil { + return nil, fmt.Errorf("converting sponsored_account_id: %w", err) + } + sponsorBytes, err := pgtypeBytesFromNullAddressBytea(sc.SponsorAccountID) + if err != nil { + return nil, fmt.Errorf("converting sponsor_account_id: %w", err) + } + deployerBytes, err := pgtypeBytesFromNullAddressBytea(sc.DeployerAccountID) + if err != nil { + return nil, fmt.Errorf("converting deployer_account_id: %w", err) + } + funderBytes, err := pgtypeBytesFromNullAddressBytea(sc.FunderAccountID) + if err != nil { + return nil, fmt.Errorf("converting funder_account_id: %w", err) + } + return []any{ pgtype.Int8{Int64: sc.ToID, Valid: true}, pgtype.Int8{Int64: sc.StateChangeOrder, Valid: true}, @@ -417,16 +464,16 @@ func (m *StateChangeModel) BatchCopy( pgtypeTextFromReasonPtr(sc.StateChangeReason), pgtype.Timestamptz{Time: sc.LedgerCreatedAt, Valid: true}, pgtype.Int4{Int32: int32(sc.LedgerNumber), Valid: true}, - pgtype.Text{String: sc.AccountID, Valid: true}, + accountBytes, pgtype.Int8{Int64: sc.OperationID, Valid: true}, pgtypeTextFromNullString(sc.TokenID), pgtypeTextFromNullString(sc.Amount), - pgtypeTextFromNullString(sc.SignerAccountID), - pgtypeTextFromNullString(sc.SpenderAccountID), - pgtypeTextFromNullString(sc.SponsoredAccountID), - pgtypeTextFromNullString(sc.SponsorAccountID), - pgtypeTextFromNullString(sc.DeployerAccountID), - pgtypeTextFromNullString(sc.FunderAccountID), + signerBytes, + spenderBytes, + sponsoredBytes, + sponsorBytes, + deployerBytes, + funderBytes, pgtypeTextFromNullString(sc.ClaimableBalanceID), pgtypeTextFromNullString(sc.LiquidityPoolID), pgtypeTextFromNullString(sc.SponsoredData), diff --git a/internal/data/transactions.go b/internal/data/transactions.go index 51a6a88d9..758dc9a4a 100644 --- a/internal/data/transactions.go +++ b/internal/data/transactions.go @@ -27,7 +27,8 @@ func (m *TransactionModel) GetByHash(ctx context.Context, hash string, columns s query := fmt.Sprintf(`SELECT %s FROM transactions WHERE hash = $1`, columns) var transaction types.Transaction start := time.Now() - err := m.DB.GetContext(ctx, &transaction, query, hash) + hashBytea := types.HashBytea(hash) + err := m.DB.GetContext(ctx, &transaction, query, hashBytea) duration := time.Since(start).Seconds() m.MetricsService.ObserveDBQueryDuration("GetByHash", "transactions", duration) if err != nil { @@ -190,7 +191,7 @@ func (m *TransactionModel) BatchInsert( } // 1. Flatten the transactions into parallel slices - hashes := make([]string, len(txs)) + hashes := make([][]byte, len(txs)) toIDs := make([]int64, len(txs)) envelopeXDRs := make([]*string, len(txs)) feesCharged := make([]int64, len(txs)) @@ -201,7 +202,11 @@ func (m *TransactionModel) BatchInsert( isFeeBumps := make([]bool, len(txs)) for i, t := range txs { - hashes[i] = t.Hash + hashBytes, err := t.Hash.Value() + if err != nil { + return nil, fmt.Errorf("converting hash %s to bytes: %w", t.Hash, err) + } + hashes[i] = hashBytes.([]byte) toIDs[i] = t.ToID envelopeXDRs[i] = t.EnvelopeXDR feesCharged[i] = t.FeeCharged @@ -218,15 +223,19 @@ func (m *TransactionModel) BatchInsert( ledgerCreatedAtByToID[tx.ToID] = tx.LedgerCreatedAt } - // 3. Flatten the stellarAddressesByToID into parallel slices + // 3. Flatten the stellarAddressesByToID into parallel slices, converting to BYTEA var txToIDs []int64 - var stellarAddresses []string + var stellarAddressBytes [][]byte var taLedgerCreatedAts []time.Time for toID, addresses := range stellarAddressesByToID { ledgerCreatedAt := ledgerCreatedAtByToID[toID] for address := range addresses.Iter() { txToIDs = append(txToIDs, toID) - stellarAddresses = append(stellarAddresses, address) + addrBytes, err := types.AddressBytea(address).Value() + if err != nil { + return nil, fmt.Errorf("converting address %s to bytes: %w", address, err) + } + stellarAddressBytes = append(stellarAddressBytes, addrBytes.([]byte)) taLedgerCreatedAts = append(taLedgerCreatedAts, ledgerCreatedAt) } } @@ -242,7 +251,7 @@ func (m *TransactionModel) BatchInsert( t.hash, t.to_id, t.envelope_xdr, t.fee_charged, t.result_code, t.meta_xdr, t.ledger_number, t.ledger_created_at, t.is_fee_bump FROM ( SELECT - UNNEST($1::text[]) AS hash, + UNNEST($1::bytea[]) AS hash, UNNEST($2::bigint[]) AS to_id, UNNEST($3::text[]) AS envelope_xdr, UNNEST($4::bigint[]) AS fee_charged, @@ -266,7 +275,7 @@ func (m *TransactionModel) BatchInsert( SELECT UNNEST($10::timestamptz[]) AS ledger_created_at, UNNEST($11::bigint[]) AS tx_to_id, - UNNEST($12::text[]) AS account_id + UNNEST($12::bytea[]) AS account_id ) ta ON CONFLICT DO NOTHING ) @@ -276,7 +285,7 @@ func (m *TransactionModel) BatchInsert( ` start := time.Now() - var insertedHashes []string + var insertedHashes []types.HashBytea err := sqlExecuter.SelectContext(ctx, &insertedHashes, insertQuery, pq.Array(hashes), pq.Array(toIDs), @@ -289,7 +298,7 @@ func (m *TransactionModel) BatchInsert( pq.Array(isFeeBumps), pq.Array(taLedgerCreatedAts), pq.Array(txToIDs), - pq.Array(stellarAddresses), + pq.Array(stellarAddressBytes), ) duration := time.Since(start).Seconds() for _, dbTableName := range []string{"transactions", "transactions_accounts"} { @@ -308,7 +317,13 @@ func (m *TransactionModel) BatchInsert( return nil, fmt.Errorf("batch inserting transactions and transactions_accounts: %w", err) } - return insertedHashes, nil + // Convert HashBytea to string for the return value + result := make([]string, len(insertedHashes)) + for i, h := range insertedHashes { + result[i] = h.String() + } + + return result, nil } // BatchCopy inserts transactions using pgx's binary COPY protocol. @@ -338,8 +353,12 @@ func (m *TransactionModel) BatchCopy( []string{"hash", "to_id", "envelope_xdr", "fee_charged", "result_code", "meta_xdr", "ledger_number", "ledger_created_at", "is_fee_bump"}, pgx.CopyFromSlice(len(txs), func(i int) ([]any, error) { tx := txs[i] + hashBytes, err := tx.Hash.Value() + if err != nil { + return nil, fmt.Errorf("converting hash %s to bytes: %w", tx.Hash, err) + } return []any{ - pgtype.Text{String: tx.Hash, Valid: true}, + hashBytes, pgtype.Int8{Int64: tx.ToID, Valid: true}, pgtypeTextFromPtr(tx.EnvelopeXDR), pgtype.Int8{Int64: tx.FeeCharged, Valid: true}, @@ -373,10 +392,14 @@ func (m *TransactionModel) BatchCopy( ledgerCreatedAtPgtype := pgtype.Timestamptz{Time: ledgerCreatedAt, Valid: true} toIDPgtype := pgtype.Int8{Int64: toID, Valid: true} for _, addr := range addresses.ToSlice() { + addrBytes, err := types.AddressBytea(addr).Value() + if err != nil { + return 0, fmt.Errorf("converting address %s to bytes: %w", addr, err) + } taRows = append(taRows, []any{ ledgerCreatedAtPgtype, toIDPgtype, - pgtype.Text{String: addr, Valid: true}, + addrBytes, }) } } From 864d23b09e015969af6b47fe1de33fb2132d1a7d Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 10 Feb 2026 14:51:40 -0500 Subject: [PATCH 18/77] Update test files for BYTEA types while preserving TimescaleDB patterns Adopt BYTEA types (HashBytea, AddressBytea, XDRBytea, NullAddressBytea) in test data while preserving TimescaleDB-specific patterns: - Keep ledger_created_at in junction table INSERTs - Use generic "duplicate key value violates unique constraint" assertions (TimescaleDB chunk-based constraint names differ from standard PG) - Keep 5-value return from processLedgersInBatch (includes startTime/endTime) --- internal/data/accounts_test.go | 98 +++++---- internal/data/operations_test.go | 201 ++++++++++------- internal/data/statechanges_test.go | 206 +++++++++-------- internal/data/transactions_test.go | 180 ++++++++------- .../serve/graphql/resolvers/test_utils.go | 23 +- internal/services/ingest_test.go | 207 +++++++++++------- 6 files changed, 541 insertions(+), 374 deletions(-) diff --git a/internal/data/accounts_test.go b/internal/data/accounts_test.go index 271bb5dcb..65729f27e 100644 --- a/internal/data/accounts_test.go +++ b/internal/data/accounts_test.go @@ -13,6 +13,7 @@ import ( "github.com/stellar/wallet-backend/internal/db" "github.com/stellar/wallet-backend/internal/db/dbtest" + "github.com/stellar/wallet-backend/internal/indexer/types" "github.com/stellar/wallet-backend/internal/metrics" ) @@ -33,6 +34,12 @@ func TestAccountModel_BatchGetByIDs(t *testing.T) { ctx := context.Background() + // Generate test addresses + account1 := keypair.MustRandom().Address() + account2 := keypair.MustRandom().Address() + nonexistent1 := keypair.MustRandom().Address() + nonexistent2 := keypair.MustRandom().Address() + t.Run("empty input returns empty result", func(t *testing.T) { var result []string err := db.RunInPgxTransaction(ctx, dbConnectionPool, func(tx pgx.Tx) error { @@ -44,8 +51,9 @@ func TestAccountModel_BatchGetByIDs(t *testing.T) { }) t.Run("returns existing accounts only", func(t *testing.T) { - // Insert some test accounts - _, err := dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)", "account1", "account2") + // Insert some test accounts using StellarAddress for BYTEA conversion + _, err := dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)", + types.AddressBytea(account1), types.AddressBytea(account2)) require.NoError(t, err) // Test with mix of existing and non-existing accounts @@ -55,17 +63,17 @@ func TestAccountModel_BatchGetByIDs(t *testing.T) { var result []string err = db.RunInPgxTransaction(ctx, dbConnectionPool, func(tx pgx.Tx) error { - result, err = accountModel.BatchGetByIDs(ctx, tx, []string{"account1", "nonexistent", "account2", "another_nonexistent"}) + result, err = accountModel.BatchGetByIDs(ctx, tx, []string{account1, nonexistent1, account2, nonexistent2}) return err }) require.NoError(t, err) // Should only return the existing accounts assert.Len(t, result, 2) - assert.Contains(t, result, "account1") - assert.Contains(t, result, "account2") - assert.NotContains(t, result, "nonexistent") - assert.NotContains(t, result, "another_nonexistent") + assert.Contains(t, result, account1) + assert.Contains(t, result, account2) + assert.NotContains(t, result, nonexistent1) + assert.NotContains(t, result, nonexistent2) }) t.Run("returns empty when no accounts exist", func(t *testing.T) { @@ -79,7 +87,7 @@ func TestAccountModel_BatchGetByIDs(t *testing.T) { var result []string err = db.RunInPgxTransaction(ctx, dbConnectionPool, func(tx pgx.Tx) error { - result, err = accountModel.BatchGetByIDs(ctx, tx, []string{"nonexistent1", "nonexistent2"}) + result, err = accountModel.BatchGetByIDs(ctx, tx, []string{nonexistent1, nonexistent2}) return err }) require.NoError(t, err) @@ -110,12 +118,11 @@ func TestAccountModel_Insert(t *testing.T) { err = m.Insert(ctx, address) require.NoError(t, err) - var dbAddress sql.NullString - err = m.DB.GetContext(ctx, &dbAddress, "SELECT stellar_address FROM accounts WHERE stellar_address = $1", address) + var dbAddress types.AddressBytea + err = m.DB.GetContext(ctx, &dbAddress, "SELECT stellar_address FROM accounts WHERE stellar_address = $1", types.AddressBytea(address)) require.NoError(t, err) - assert.True(t, dbAddress.Valid) - assert.Equal(t, address, dbAddress.String) + assert.Equal(t, address, string(dbAddress)) }) t.Run("duplicate insert fails", func(t *testing.T) { @@ -164,7 +171,7 @@ func TestAccountModel_Delete(t *testing.T) { ctx := context.Background() address := keypair.MustRandom().Address() - result, insertErr := m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", address) + result, insertErr := m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", types.AddressBytea(address)) require.NoError(t, insertErr) rowAffected, err := result.RowsAffected() require.NoError(t, err) @@ -173,7 +180,7 @@ func TestAccountModel_Delete(t *testing.T) { err = m.Delete(ctx, address) require.NoError(t, err) - var dbAddress sql.NullString + var dbAddress types.AddressBytea err = m.DB.GetContext(ctx, &dbAddress, "SELECT stellar_address FROM accounts LIMIT 1") assert.ErrorIs(t, err, sql.ErrNoRows) }) @@ -218,7 +225,7 @@ func TestAccountModelGet(t *testing.T) { address := keypair.MustRandom().Address() // Insert test account - result, err := m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", address) + result, err := m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", types.AddressBytea(address)) require.NoError(t, err) rowAffected, err := result.RowsAffected() require.NoError(t, err) @@ -227,7 +234,7 @@ func TestAccountModelGet(t *testing.T) { // Test Get function account, err := m.Get(ctx, address) require.NoError(t, err) - assert.Equal(t, address, account.StellarAddress) + assert.Equal(t, address, string(account.StellarAddress)) } func TestAccountModelBatchGetByToIDs(t *testing.T) { @@ -255,15 +262,19 @@ func TestAccountModelBatchGetByToIDs(t *testing.T) { toID2 := int64(2) // Insert test accounts - _, err = m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)", address1, address2) + _, err = m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)", + types.AddressBytea(address1), types.AddressBytea(address2)) require.NoError(t, err) - // Insert test transactions first - _, err = m.DB.ExecContext(ctx, "INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at) VALUES ('tx1', $1, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, NOW()), ('tx2', $2, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, NOW())", toID1, toID2) + // Insert test transactions first (hash is BYTEA, using valid 64-char hex strings) + testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001") + testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002") + _, err = m.DB.ExecContext(ctx, "INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at) VALUES ($1, $2, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, NOW()), ($3, $4, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, NOW())", testHash1, toID1, testHash2, toID2) require.NoError(t, err) // Insert test transactions_accounts links - _, err = m.DB.ExecContext(ctx, "INSERT INTO transactions_accounts (ledger_created_at, tx_to_id, account_id) VALUES (NOW(), $1, $2), (NOW(), $3, $4)", toID1, address1, toID2, address2) + _, err = m.DB.ExecContext(ctx, "INSERT INTO transactions_accounts (ledger_created_at, tx_to_id, account_id) VALUES (NOW(), $1, $2), (NOW(), $3, $4)", + toID1, types.AddressBytea(address1), toID2, types.AddressBytea(address2)) require.NoError(t, err) // Test BatchGetByToIDs function @@ -274,7 +285,7 @@ func TestAccountModelBatchGetByToIDs(t *testing.T) { // Verify accounts are returned with correct to_id addressSet := make(map[string]int64) for _, acc := range accounts { - addressSet[acc.StellarAddress] = acc.ToID + addressSet[string(acc.StellarAddress)] = acc.ToID } assert.Equal(t, toID1, addressSet[address1]) assert.Equal(t, toID2, addressSet[address2]) @@ -304,20 +315,26 @@ func TestAccountModelBatchGetByOperationIDs(t *testing.T) { operationID1 := int64(123) operationID2 := int64(456) - // Insert test accounts - _, err = m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)", address1, address2) + // Insert test accounts (stellar_address is BYTEA) + _, err = m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)", + types.AddressBytea(address1), types.AddressBytea(address2)) require.NoError(t, err) - // Insert test transactions first - _, err = m.DB.ExecContext(ctx, "INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at) VALUES ('tx1', 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, NOW()), ('tx2', 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, NOW())") + // Insert test transactions first (hash is BYTEA, using valid 64-char hex strings) + testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001") + testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002") + _, err = m.DB.ExecContext(ctx, "INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at) VALUES ($1, 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, NOW()), ($2, 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, NOW())", testHash1, testHash2) require.NoError(t, err) // Insert test operations (IDs don't need to be in TOID range here since we're just testing operations_accounts links) - _, err = m.DB.ExecContext(ctx, "INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) VALUES ($1, 'PAYMENT', 'xdr1', 'op_success', true, 1, NOW()), ($2, 'PAYMENT', 'xdr2', 'op_success', true, 2, NOW())", operationID1, operationID2) + xdr1 := types.XDRBytea([]byte("xdr1")) + xdr2 := types.XDRBytea([]byte("xdr2")) + _, err = m.DB.ExecContext(ctx, "INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) VALUES ($1, 'PAYMENT', $3, 'op_success', true, 1, NOW()), ($2, 'PAYMENT', $4, 'op_success', true, 2, NOW())", operationID1, operationID2, xdr1, xdr2) require.NoError(t, err) - // Insert test operations_accounts links - _, err = m.DB.ExecContext(ctx, "INSERT INTO operations_accounts (ledger_created_at, operation_id, account_id) VALUES (NOW(), $1, $2), (NOW(), $3, $4)", operationID1, address1, operationID2, address2) + // Insert test operations_accounts links (account_id is BYTEA) + _, err = m.DB.ExecContext(ctx, "INSERT INTO operations_accounts (ledger_created_at, operation_id, account_id) VALUES (NOW(), $1, $2), (NOW(), $3, $4)", + operationID1, types.AddressBytea(address1), operationID2, types.AddressBytea(address2)) require.NoError(t, err) // Test BatchGetByOperationID function @@ -328,7 +345,7 @@ func TestAccountModelBatchGetByOperationIDs(t *testing.T) { // Verify accounts are returned with correct operation_id addressSet := make(map[string]int64) for _, acc := range accounts { - addressSet[acc.StellarAddress] = acc.OperationID + addressSet[string(acc.StellarAddress)] = acc.OperationID } assert.Equal(t, operationID1, addressSet[address1]) assert.Equal(t, operationID2, addressSet[address2]) @@ -358,7 +375,7 @@ func TestAccountModel_IsAccountFeeBumpEligible(t *testing.T) { require.NoError(t, err) assert.False(t, isFeeBumpEligible) - result, err := m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", address) + result, err := m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", types.AddressBytea(address)) require.NoError(t, err) rowAffected, err := result.RowsAffected() require.NoError(t, err) @@ -395,19 +412,24 @@ func TestAccountModelBatchGetByStateChangeIDs(t *testing.T) { stateChangeOrder1 := int64(1) stateChangeOrder2 := int64(1) - // Insert test accounts - _, err = m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)", address1, address2) + // Insert test accounts (stellar_address is BYTEA) + _, err = m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)", + types.AddressBytea(address1), types.AddressBytea(address2)) require.NoError(t, err) - // Insert test transactions first - _, err = m.DB.ExecContext(ctx, "INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at) VALUES ('tx1', 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, NOW()), ('tx2', 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, NOW())") + // Insert test transactions first (hash is BYTEA, using valid 64-char hex strings) + testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001") + testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002") + _, err = m.DB.ExecContext(ctx, "INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at) VALUES ($1, 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, NOW()), ($2, 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, NOW())", testHash1, testHash2) require.NoError(t, err) // Insert test operations (IDs must be in TOID range for each transaction) - _, err = m.DB.ExecContext(ctx, "INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) VALUES (4097, 'PAYMENT', 'xdr1', 'op_success', true, 1, NOW()), (8193, 'PAYMENT', 'xdr2', 'op_success', true, 2, NOW())") + xdr1 := types.XDRBytea([]byte("xdr1")) + xdr2 := types.XDRBytea([]byte("xdr2")) + _, err = m.DB.ExecContext(ctx, "INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) VALUES (4097, 'PAYMENT', $1, 'op_success', true, 1, NOW()), (8193, 'PAYMENT', $2, 'op_success', true, 2, NOW())", xdr1, xdr2) require.NoError(t, err) - // Insert test state changes that reference the accounts + // Insert test state changes that reference the accounts (state_changes.account_id is TEXT) _, err = m.DB.ExecContext(ctx, ` INSERT INTO state_changes ( to_id, state_change_order, state_change_category, ledger_created_at, @@ -415,7 +437,7 @@ func TestAccountModelBatchGetByStateChangeIDs(t *testing.T) { ) VALUES ($1, $2, 'BALANCE', NOW(), 1, $3, 4097), ($4, $5, 'BALANCE', NOW(), 2, $6, 8193) - `, toID1, stateChangeOrder1, address1, toID2, stateChangeOrder2, address2) + `, toID1, stateChangeOrder1, types.AddressBytea(address1), toID2, stateChangeOrder2, types.AddressBytea(address2)) require.NoError(t, err) // Test BatchGetByStateChangeIDs function @@ -429,7 +451,7 @@ func TestAccountModelBatchGetByStateChangeIDs(t *testing.T) { // Verify accounts are returned with correct state_change_id (format: to_id-operation_id-state_change_order) addressSet := make(map[string]string) for _, acc := range accounts { - addressSet[acc.StellarAddress] = acc.StateChangeID + addressSet[string(acc.StellarAddress)] = acc.StateChangeID } assert.Equal(t, "4096-4097-1", addressSet[address1]) assert.Equal(t, "8192-8193-1", addressSet[address2]) diff --git a/internal/data/operations_test.go b/internal/data/operations_test.go index c96a939f5..2b9126367 100644 --- a/internal/data/operations_test.go +++ b/internal/data/operations_test.go @@ -33,7 +33,7 @@ func generateTestOperations(n int, startID int64) ([]*types.Operation, map[int64 ops[i] = &types.Operation{ ID: opID, OperationType: types.OperationTypePayment, - OperationXDR: fmt.Sprintf("operation_xdr_%d", i), + OperationXDR: types.XDRBytea([]byte(fmt.Sprintf("operation_xdr_%d", i))), LedgerNumber: uint32(i + 1), LedgerCreatedAt: now, } @@ -56,8 +56,8 @@ func Test_OperationModel_BatchInsert(t *testing.T) { // Create test data kp1 := keypair.MustRandom() kp2 := keypair.MustRandom() - const q = "INSERT INTO accounts (stellar_address) SELECT UNNEST(ARRAY[$1, $2])" - _, err = dbConnectionPool.ExecContext(ctx, q, kp1.Address(), kp2.Address()) + const q = "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)" + _, err = dbConnectionPool.ExecContext(ctx, q, types.AddressBytea(kp1.Address()), types.AddressBytea(kp2.Address())) require.NoError(t, err) // Create referenced transactions first with specific ToIDs @@ -65,7 +65,7 @@ func Test_OperationModel_BatchInsert(t *testing.T) { meta1, meta2 := "meta1", "meta2" envelope1, envelope2 := "envelope1", "envelope2" tx1 := types.Transaction{ - Hash: "tx1", + Hash: "d176b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", ToID: 4096, EnvelopeXDR: &envelope1, FeeCharged: 100, @@ -76,7 +76,7 @@ func Test_OperationModel_BatchInsert(t *testing.T) { IsFeeBump: false, } tx2 := types.Transaction{ - Hash: "tx2", + Hash: "e176b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", ToID: 8192, EnvelopeXDR: &envelope2, FeeCharged: 200, @@ -101,13 +101,13 @@ func Test_OperationModel_BatchInsert(t *testing.T) { op1 := types.Operation{ ID: 4097, // in range (4096, 8192) OperationType: types.OperationTypePayment, - OperationXDR: "operation1", + OperationXDR: types.XDRBytea([]byte("operation1")), LedgerCreatedAt: now, } op2 := types.Operation{ ID: 8193, // in range (8192, 12288) OperationType: types.OperationTypeCreateAccount, - OperationXDR: "operation2", + OperationXDR: types.XDRBytea([]byte("operation2")), LedgerCreatedAt: now, } @@ -206,8 +206,8 @@ func Test_OperationModel_BatchInsert(t *testing.T) { // Verify the account links if len(tc.wantAccountLinks) > 0 { var accountLinks []struct { - OperationID int64 `db:"operation_id"` - AccountID string `db:"account_id"` + OperationID int64 `db:"operation_id"` + AccountID types.AddressBytea `db:"account_id"` } err = sqlExecuter.SelectContext(ctx, &accountLinks, "SELECT operation_id, account_id FROM operations_accounts ORDER BY operation_id, account_id") require.NoError(t, err) @@ -215,7 +215,7 @@ func Test_OperationModel_BatchInsert(t *testing.T) { // Create a map of operation_id -> set of account_ids for O(1) lookups accountLinksMap := make(map[int64][]string) for _, link := range accountLinks { - accountLinksMap[link.OperationID] = append(accountLinksMap[link.OperationID], link.AccountID) + accountLinksMap[link.OperationID] = append(accountLinksMap[link.OperationID], string(link.AccountID)) } // Verify each operation has its expected account links @@ -243,8 +243,8 @@ func Test_OperationModel_BatchCopy(t *testing.T) { // Create test accounts kp1 := keypair.MustRandom() kp2 := keypair.MustRandom() - const q = "INSERT INTO accounts (stellar_address) SELECT UNNEST(ARRAY[$1, $2])" - _, err = dbConnectionPool.ExecContext(ctx, q, kp1.Address(), kp2.Address()) + const q = "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)" + _, err = dbConnectionPool.ExecContext(ctx, q, types.AddressBytea(kp1.Address()), types.AddressBytea(kp2.Address())) require.NoError(t, err) // Create referenced transactions first with specific ToIDs @@ -252,7 +252,7 @@ func Test_OperationModel_BatchCopy(t *testing.T) { meta1, meta2 := "meta1", "meta2" envelope1, envelope2 := "envelope1", "envelope2" tx1 := types.Transaction{ - Hash: "tx1", + Hash: "d176b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", ToID: 4096, EnvelopeXDR: &envelope1, FeeCharged: 100, @@ -263,7 +263,7 @@ func Test_OperationModel_BatchCopy(t *testing.T) { IsFeeBump: false, } tx2 := types.Transaction{ - Hash: "tx2", + Hash: "e176b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", ToID: 8192, EnvelopeXDR: &envelope2, FeeCharged: 200, @@ -288,13 +288,13 @@ func Test_OperationModel_BatchCopy(t *testing.T) { op1 := types.Operation{ ID: 4097, // in range (4096, 8192) OperationType: types.OperationTypePayment, - OperationXDR: "operation1", + OperationXDR: types.XDRBytea([]byte("operation1")), LedgerCreatedAt: now, } op2 := types.Operation{ ID: 8193, // in range (8192, 12288) OperationType: types.OperationTypeCreateAccount, - OperationXDR: "operation2", + OperationXDR: types.XDRBytea([]byte("operation2")), LedgerCreatedAt: now, } @@ -384,8 +384,8 @@ func Test_OperationModel_BatchCopy(t *testing.T) { // Verify account links if expected if len(tc.stellarAddressesByOpID) > 0 && tc.wantCount > 0 { var accountLinks []struct { - OperationID int64 `db:"operation_id"` - AccountID string `db:"account_id"` + OperationID int64 `db:"operation_id"` + AccountID types.AddressBytea `db:"account_id"` } err = dbConnectionPool.SelectContext(ctx, &accountLinks, "SELECT operation_id, account_id FROM operations_accounts ORDER BY operation_id, account_id") require.NoError(t, err) @@ -393,7 +393,7 @@ func Test_OperationModel_BatchCopy(t *testing.T) { // Create a map of operation_id -> set of account_ids accountLinksMap := make(map[int64][]string) for _, link := range accountLinks { - accountLinksMap[link.OperationID] = append(accountLinksMap[link.OperationID], link.AccountID) + accountLinksMap[link.OperationID] = append(accountLinksMap[link.OperationID], string(link.AccountID)) } // Verify each expected operation has its account links @@ -419,7 +419,7 @@ func Test_OperationModel_BatchCopy_DuplicateFails(t *testing.T) { // Create test accounts kp1 := keypair.MustRandom() const q = "INSERT INTO accounts (stellar_address) VALUES ($1)" - _, err = dbConnectionPool.ExecContext(ctx, q, kp1.Address()) + _, err = dbConnectionPool.ExecContext(ctx, q, types.AddressBytea(kp1.Address())) require.NoError(t, err) // Create a parent transaction that the operation will reference @@ -432,7 +432,7 @@ func Test_OperationModel_BatchCopy_DuplicateFails(t *testing.T) { op1 := types.Operation{ ID: 999, OperationType: types.OperationTypePayment, - OperationXDR: "operation_xdr_dup_test", + OperationXDR: types.XDRBytea([]byte("operation_xdr_dup_test")), LedgerNumber: 1, LedgerCreatedAt: now, } @@ -476,8 +476,7 @@ func Test_OperationModel_BatchCopy_DuplicateFails(t *testing.T) { // BatchCopy should fail with a unique constraint violation require.Error(t, err) - // TimescaleDB uses chunk-based constraint names like "2_3_operations_pkey" instead of "operations_pkey" - assert.Contains(t, err.Error(), "pgx CopyFrom operations: ERROR: duplicate key value violates unique constraint") + assert.Contains(t, err.Error(), "duplicate key value violates unique constraint") // Rollback the failed transaction require.NoError(t, pgxTx.Rollback(ctx)) @@ -503,24 +502,30 @@ func TestOperationModel_GetAll(t *testing.T) { ctx := context.Background() now := time.Now() - // Create test transactions first + // Create test transactions first (hash is BYTEA, using valid 64-char hex strings) + testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001") + testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002") + testHash3 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000003") _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) VALUES - ('tx1', 1, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), - ('tx2', 2, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), - ('tx3', 3, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) - `, now) + ($2, 1, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), + ($3, 2, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), + ($4, 3, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) + `, now, testHash1, testHash2, testHash3) require.NoError(t, err) // Create test operations (IDs must be in TOID range for each transaction: (to_id, to_id + 4096)) + xdr1 := types.XDRBytea([]byte("xdr1")) + xdr2 := types.XDRBytea([]byte("xdr2")) + xdr3 := types.XDRBytea([]byte("xdr3")) _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) VALUES - (2, 'PAYMENT', 'xdr1', 'op_success', true, 1, $1), - (4098, 'CREATE_ACCOUNT', 'xdr2', 'op_success', true, 2, $1), - (8194, 'PAYMENT', 'xdr3', 'op_success', true, 3, $1) - `, now) + (2, 'PAYMENT', $2, 'op_success', true, 1, $1), + (4098, 'CREATE_ACCOUNT', $3, 'op_success', true, 2, $1), + (8194, 'PAYMENT', $4, 'op_success', true, 3, $1) + `, now, xdr1, xdr2, xdr3) require.NoError(t, err) // Test GetAll without limit (gets all operations) @@ -553,29 +558,38 @@ func TestOperationModel_BatchGetByToIDs(t *testing.T) { // Create test transactions first with specific ToIDs // ToID encoding: operations for a tx with to_id are in range (to_id, to_id + 4096) // Using to_id values: 4096, 8192, 12288 (multiples of 4096 for clarity) + testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001") + testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002") + testHash3 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000003") _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) VALUES - ('tx1', 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), - ('tx2', 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), - ('tx3', 12288, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) - `, now) + ($2, 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), + ($3, 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), + ($4, 12288, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) + `, now, testHash1, testHash2, testHash3) require.NoError(t, err) // Create test operations - IDs must be in TOID range for each transaction // For tx1 (to_id=4096): ops 4097, 4098, 4099 // For tx2 (to_id=8192): ops 8193, 8194 // For tx3 (to_id=12288): op 12289 + xdr1 := types.XDRBytea([]byte("xdr1")) + xdr2 := types.XDRBytea([]byte("xdr2")) + xdr3 := types.XDRBytea([]byte("xdr3")) + xdr4 := types.XDRBytea([]byte("xdr4")) + xdr5 := types.XDRBytea([]byte("xdr5")) + xdr6 := types.XDRBytea([]byte("xdr6")) _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) VALUES - (4097, 'PAYMENT', 'xdr1', 'op_success', true, 1, $1), - (8193, 'CREATE_ACCOUNT', 'xdr2', 'op_success', true, 2, $1), - (4098, 'PAYMENT', 'xdr3', 'op_success', true, 3, $1), - (4099, 'MANAGE_SELL_OFFER', 'xdr4', 'op_success', true, 4, $1), - (8194, 'PAYMENT', 'xdr5', 'op_success', true, 5, $1), - (12289, 'CHANGE_TRUST', 'xdr6', 'op_success', true, 6, $1) - `, now) + (4097, 'PAYMENT', $2, 'op_success', true, 1, $1), + (8193, 'CREATE_ACCOUNT', $3, 'op_success', true, 2, $1), + (4098, 'PAYMENT', $4, 'op_success', true, 3, $1), + (4099, 'MANAGE_SELL_OFFER', $5, 'op_success', true, 4, $1), + (8194, 'PAYMENT', $6, 'op_success', true, 5, $1), + (12289, 'CHANGE_TRUST', $7, 'op_success', true, 6, $1) + `, now, xdr1, xdr2, xdr3, xdr4, xdr5, xdr6) require.NoError(t, err) testCases := []struct { @@ -743,33 +757,38 @@ func TestOperationModel_BatchGetByToID(t *testing.T) { ctx := context.Background() now := time.Now() - // Create test transactions first with specific ToIDs + // Create test transactions first with specific ToIDs (hash is BYTEA) + testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001") + testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002") _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) VALUES - ('tx1', 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), - ('tx2', 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true) - `, now) + ($2, 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), + ($3, 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true) + `, now, testHash1, testHash2) require.NoError(t, err) // Create test operations - IDs must be in TOID range for each transaction // For tx1 (to_id=4096): ops 4097, 4098 // For tx2 (to_id=8192): op 8193 + xdr1 := types.XDRBytea([]byte("xdr1")) + xdr2 := types.XDRBytea([]byte("xdr2")) + xdr3 := types.XDRBytea([]byte("xdr3")) _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) VALUES - (4097, 'PAYMENT', 'xdr1', 'op_success', true, 1, $1), - (8193, 'CREATE_ACCOUNT', 'xdr2', 'op_success', true, 2, $1), - (4098, 'PAYMENT', 'xdr3', 'op_success', true, 3, $1) - `, now) + (4097, 'PAYMENT', $2, 'op_success', true, 1, $1), + (8193, 'CREATE_ACCOUNT', $3, 'op_success', true, 2, $1), + (4098, 'PAYMENT', $4, 'op_success', true, 3, $1) + `, now, xdr1, xdr2, xdr3) require.NoError(t, err) // Test BatchGetByToID operations, err := m.BatchGetByToID(ctx, 4096, "", nil, nil, ASC) require.NoError(t, err) assert.Len(t, operations, 2) - assert.Equal(t, "xdr1", operations[0].OperationXDR) - assert.Equal(t, "xdr3", operations[1].OperationXDR) + assert.Equal(t, xdr1.String(), operations[0].OperationXDR.String()) + assert.Equal(t, xdr3.String(), operations[1].OperationXDR.String()) } func TestOperationModel_BatchGetByAccountAddresses(t *testing.T) { @@ -798,34 +817,40 @@ func TestOperationModel_BatchGetByAccountAddresses(t *testing.T) { _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)", address1, address2) require.NoError(t, err) - // Create test transactions first + // Create test transactions first (hash is BYTEA) + testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001") + testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002") + testHash3 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000003") _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) VALUES - ('tx1', 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), - ('tx2', 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), - ('tx3', 12288, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) - `, now) + ($2, 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), + ($3, 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), + ($4, 12288, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) + `, now, testHash1, testHash2, testHash3) require.NoError(t, err) // Create test operations (IDs must be in TOID range for each transaction) + xdr1 := types.XDRBytea([]byte("xdr1")) + xdr2 := types.XDRBytea([]byte("xdr2")) + xdr3 := types.XDRBytea([]byte("xdr3")) _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) VALUES - (4097, 'PAYMENT', 'xdr1', 'op_success', true, 1, $1), - (8193, 'CREATE_ACCOUNT', 'xdr2', 'op_success', true, 2, $1), - (12289, 'PAYMENT', 'xdr3', 'op_success', true, 3, $1) - `, now) + (4097, 'PAYMENT', $2, 'op_success', true, 1, $1), + (8193, 'CREATE_ACCOUNT', $3, 'op_success', true, 2, $1), + (12289, 'PAYMENT', $4, 'op_success', true, 3, $1) + `, now, xdr1, xdr2, xdr3) require.NoError(t, err) // Create test operations_accounts links _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO operations_accounts (ledger_created_at, operation_id, account_id) VALUES - ($1, 4097, $2), - ($1, 8193, $2), - ($1, 12289, $3) - `, now, address1, address2) + ($3, 4097, $1), + ($3, 8193, $1), + ($3, 12289, $2) + `, types.AddressBytea(address1), types.AddressBytea(address2), now) require.NoError(t, err) // Test BatchGetByAccount @@ -846,22 +871,26 @@ func TestOperationModel_GetByID(t *testing.T) { ctx := context.Background() now := time.Now() - // Create test transactions first + // Create test transactions first (hash is BYTEA) + testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001") + testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002") _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) VALUES - ('tx1', 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), - ('tx2', 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true) - `, now) + ($2, 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), + ($3, 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true) + `, now, testHash1, testHash2) require.NoError(t, err) // Create test operations (IDs must be in TOID range for each transaction) + opXdr1 := types.XDRBytea([]byte("xdr1")) + opXdr2 := types.XDRBytea([]byte("xdr2")) _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) VALUES - (4097, 'PAYMENT', 'xdr1', 'op_success', true, 1, $1), - (8193, 'CREATE_ACCOUNT', 'xdr2', 'op_success', true, 2, $1) - `, now) + (4097, 'PAYMENT', $2, 'op_success', true, 1, $1), + (8193, 'CREATE_ACCOUNT', $3, 'op_success', true, 2, $1) + `, now, opXdr1, opXdr2) require.NoError(t, err) mockMetricsService := metrics.NewMockMetricsService() @@ -877,7 +906,7 @@ func TestOperationModel_GetByID(t *testing.T) { operation, err := m.GetByID(ctx, 4097, "") require.NoError(t, err) assert.Equal(t, int64(4097), operation.ID) - assert.Equal(t, "xdr1", operation.OperationXDR) + assert.Equal(t, opXdr1.String(), operation.OperationXDR.String()) assert.Equal(t, uint32(1), operation.LedgerNumber) assert.WithinDuration(t, now, operation.LedgerCreatedAt, time.Second) } @@ -908,24 +937,30 @@ func TestOperationModel_BatchGetByStateChangeIDs(t *testing.T) { _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", address) require.NoError(t, err) - // Create test transactions first + // Create test transactions first (hash is BYTEA) + testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001") + testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002") + testHash3 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000003") _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) VALUES - ('tx1', 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), - ('tx2', 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), - ('tx3', 12288, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) - `, now) + ($2, 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), + ($3, 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), + ($4, 12288, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) + `, now, testHash1, testHash2, testHash3) require.NoError(t, err) // Create test operations (IDs must be in TOID range for each transaction) + xdr1 := types.XDRBytea([]byte("xdr1")) + xdr2 := types.XDRBytea([]byte("xdr2")) + xdr3 := types.XDRBytea([]byte("xdr3")) _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) VALUES - (4097, 'PAYMENT', 'xdr1', 'op_success', true, 1, $1), - (8193, 'CREATE_ACCOUNT', 'xdr2', 'op_success', true, 2, $1), - (12289, 'PAYMENT', 'xdr3', 'op_success', true, 3, $1) - `, now) + (4097, 'PAYMENT', $2, 'op_success', true, 1, $1), + (8193, 'CREATE_ACCOUNT', $3, 'op_success', true, 2, $1), + (12289, 'PAYMENT', $4, 'op_success', true, 3, $1) + `, now, xdr1, xdr2, xdr3) require.NoError(t, err) // Create test state changes diff --git a/internal/data/statechanges_test.go b/internal/data/statechanges_test.go index e36649687..8a44b6985 100644 --- a/internal/data/statechanges_test.go +++ b/internal/data/statechanges_test.go @@ -22,12 +22,15 @@ import ( // generateTestStateChanges creates n test state changes for benchmarking. // Populates all fields to provide an upper-bound benchmark. -func generateTestStateChanges(n int, accountID string, startToID int64) []types.StateChange { +// The auxAddresses parameter provides pre-generated valid Stellar addresses for nullable account_id fields. +func generateTestStateChanges(n int, accountID string, startToID int64, auxAddresses []string) []types.StateChange { scs := make([]types.StateChange, n) now := time.Now() reason := types.StateChangeReasonCredit for i := 0; i < n; i++ { + // Use modulo to cycle through auxiliary addresses for nullable account_id fields + auxIdx := i % len(auxAddresses) scs[i] = types.StateChange{ ToID: startToID + int64(i), StateChangeOrder: 1, @@ -35,20 +38,21 @@ func generateTestStateChanges(n int, accountID string, startToID int64) []types. StateChangeReason: &reason, LedgerCreatedAt: now, LedgerNumber: uint32(i + 1), - AccountID: accountID, + AccountID: types.AddressBytea(accountID), OperationID: int64(i + 1), // sql.NullString fields - TokenID: sql.NullString{String: fmt.Sprintf("token_%d", i), Valid: true}, - Amount: sql.NullString{String: fmt.Sprintf("%d", (i+1)*100), Valid: true}, - SignerAccountID: sql.NullString{String: fmt.Sprintf("GSIGNER%032d", i), Valid: true}, - SpenderAccountID: sql.NullString{String: fmt.Sprintf("GSPENDER%031d", i), Valid: true}, - SponsoredAccountID: sql.NullString{String: fmt.Sprintf("GSPONSORED%028d", i), Valid: true}, - SponsorAccountID: sql.NullString{String: fmt.Sprintf("GSPONSOR%030d", i), Valid: true}, - DeployerAccountID: sql.NullString{String: fmt.Sprintf("GDEPLOYER%029d", i), Valid: true}, - FunderAccountID: sql.NullString{String: fmt.Sprintf("GFUNDER%031d", i), Valid: true}, + TokenID: sql.NullString{String: fmt.Sprintf("token_%d", i), Valid: true}, + Amount: sql.NullString{String: fmt.Sprintf("%d", (i+1)*100), Valid: true}, + // NullAddressBytea fields + SignerAccountID: types.NullAddressBytea{AddressBytea: types.AddressBytea(auxAddresses[auxIdx]), Valid: true}, + SpenderAccountID: types.NullAddressBytea{AddressBytea: types.AddressBytea(auxAddresses[(auxIdx+1)%len(auxAddresses)]), Valid: true}, + SponsoredAccountID: types.NullAddressBytea{AddressBytea: types.AddressBytea(auxAddresses[(auxIdx+2)%len(auxAddresses)]), Valid: true}, + SponsorAccountID: types.NullAddressBytea{AddressBytea: types.AddressBytea(auxAddresses[(auxIdx+3)%len(auxAddresses)]), Valid: true}, + DeployerAccountID: types.NullAddressBytea{AddressBytea: types.AddressBytea(auxAddresses[(auxIdx+4)%len(auxAddresses)]), Valid: true}, + FunderAccountID: types.NullAddressBytea{AddressBytea: types.AddressBytea(auxAddresses[(auxIdx+5)%len(auxAddresses)]), Valid: true}, // Typed fields (previously JSONB) - SignerWeightOld: sql.NullInt16{Int16: int16(i), Valid: true}, - SignerWeightNew: sql.NullInt16{Int16: int16(i + 1), Valid: true}, + SignerWeightOld: sql.NullInt16{Int16: int16(i % 256), Valid: true}, + SignerWeightNew: sql.NullInt16{Int16: int16((i + 1) % 256), Valid: true}, ThresholdOld: sql.NullInt16{Int16: 1, Valid: true}, ThresholdNew: sql.NullInt16{Int16: 2, Valid: true}, TrustlineLimitOld: sql.NullString{String: fmt.Sprintf("%d", i*1000), Valid: true}, @@ -74,15 +78,15 @@ func TestStateChangeModel_BatchInsert(t *testing.T) { // Create test data kp1 := keypair.MustRandom() kp2 := keypair.MustRandom() - const q = "INSERT INTO accounts (stellar_address) SELECT UNNEST(ARRAY[$1, $2])" - _, err = dbConnectionPool.ExecContext(ctx, q, kp1.Address(), kp2.Address()) + const q = "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)" + _, err = dbConnectionPool.ExecContext(ctx, q, types.AddressBytea(kp1.Address()), types.AddressBytea(kp2.Address())) require.NoError(t, err) // Create referenced transactions first meta1, meta2 := "meta1", "meta2" envelope1, envelope2 := "envelope1", "envelope2" tx1 := types.Transaction{ - Hash: "tx1", + Hash: "f176b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", ToID: 1, EnvelopeXDR: &envelope1, FeeCharged: 100, @@ -93,7 +97,7 @@ func TestStateChangeModel_BatchInsert(t *testing.T) { IsFeeBump: false, } tx2 := types.Transaction{ - Hash: "tx2", + Hash: "0276b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", ToID: 2, EnvelopeXDR: &envelope2, FeeCharged: 200, @@ -120,7 +124,7 @@ func TestStateChangeModel_BatchInsert(t *testing.T) { StateChangeReason: &reason, LedgerCreatedAt: now, LedgerNumber: 1, - AccountID: kp1.Address(), + AccountID: types.AddressBytea(kp1.Address()), OperationID: 123, TokenID: sql.NullString{String: "token1", Valid: true}, Amount: sql.NullString{String: "100", Valid: true}, @@ -132,7 +136,7 @@ func TestStateChangeModel_BatchInsert(t *testing.T) { StateChangeReason: &reason, LedgerCreatedAt: now, LedgerNumber: 2, - AccountID: kp2.Address(), + AccountID: types.AddressBytea(kp2.Address()), OperationID: 456, } @@ -230,15 +234,15 @@ func TestStateChangeModel_BatchCopy(t *testing.T) { // Create test accounts kp1 := keypair.MustRandom() kp2 := keypair.MustRandom() - const q = "INSERT INTO accounts (stellar_address) SELECT UNNEST(ARRAY[$1, $2])" - _, err = dbConnectionPool.ExecContext(ctx, q, kp1.Address(), kp2.Address()) + const q = "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)" + _, err = dbConnectionPool.ExecContext(ctx, q, types.AddressBytea(kp1.Address()), types.AddressBytea(kp2.Address())) require.NoError(t, err) // Create referenced transactions first meta1, meta2 := "meta1", "meta2" envelope1, envelope2 := "envelope1", "envelope2" tx1 := types.Transaction{ - Hash: "tx1", + Hash: "f176b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", ToID: 1, EnvelopeXDR: &envelope1, FeeCharged: 100, @@ -249,7 +253,7 @@ func TestStateChangeModel_BatchCopy(t *testing.T) { IsFeeBump: false, } tx2 := types.Transaction{ - Hash: "tx2", + Hash: "0276b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", ToID: 2, EnvelopeXDR: &envelope2, FeeCharged: 200, @@ -276,7 +280,7 @@ func TestStateChangeModel_BatchCopy(t *testing.T) { StateChangeReason: &reason, LedgerCreatedAt: now, LedgerNumber: 1, - AccountID: kp1.Address(), + AccountID: types.AddressBytea(kp1.Address()), OperationID: 123, TokenID: sql.NullString{String: "token1", Valid: true}, Amount: sql.NullString{String: "100", Valid: true}, @@ -288,7 +292,7 @@ func TestStateChangeModel_BatchCopy(t *testing.T) { StateChangeReason: &reason, LedgerCreatedAt: now, LedgerNumber: 2, - AccountID: kp2.Address(), + AccountID: types.AddressBytea(kp2.Address()), OperationID: 456, } // State change with typed signer/threshold fields (uses to_id=1 to reference tx1) @@ -299,7 +303,7 @@ func TestStateChangeModel_BatchCopy(t *testing.T) { StateChangeReason: nil, LedgerCreatedAt: now, LedgerNumber: 3, - AccountID: kp1.Address(), + AccountID: types.AddressBytea(kp1.Address()), OperationID: 789, SignerWeightOld: sql.NullInt16{Int16: 0, Valid: true}, SignerWeightNew: sql.NullInt16{Int16: 10, Valid: true}, @@ -403,7 +407,7 @@ func TestStateChangeModel_BatchCopy_DuplicateFails(t *testing.T) { // Create test account kp1 := keypair.MustRandom() const q = "INSERT INTO accounts (stellar_address) VALUES ($1)" - _, err = dbConnectionPool.ExecContext(ctx, q, kp1.Address()) + _, err = dbConnectionPool.ExecContext(ctx, q, types.AddressBytea(kp1.Address())) require.NoError(t, err) // Create parent transaction @@ -421,7 +425,7 @@ func TestStateChangeModel_BatchCopy_DuplicateFails(t *testing.T) { StateChangeReason: &reason, LedgerCreatedAt: now, LedgerNumber: 1, - AccountID: kp1.Address(), + AccountID: types.AddressBytea(kp1.Address()), OperationID: 123, } @@ -460,8 +464,7 @@ func TestStateChangeModel_BatchCopy_DuplicateFails(t *testing.T) { // BatchCopy should fail with a unique constraint violation require.Error(t, err) - // TimescaleDB uses chunk-based constraint names like "2_3_state_changes_pkey" instead of "state_changes_pkey" - assert.Contains(t, err.Error(), "pgx CopyFrom state_changes: ERROR: duplicate key value violates unique constraint") + assert.Contains(t, err.Error(), "duplicate key value violates unique constraint") // Rollback the failed transaction require.NoError(t, pgxTx.Rollback(ctx)) @@ -480,17 +483,21 @@ func TestStateChangeModel_BatchGetByAccountAddress(t *testing.T) { // Create test accounts address1 := keypair.MustRandom().Address() address2 := keypair.MustRandom().Address() - _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)", address1, address2) + _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)", + types.AddressBytea(address1), types.AddressBytea(address2)) require.NoError(t, err) - // Create test transactions first + // Create test transactions first (hash is BYTEA) + testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001") + testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002") + testHash3 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000003") _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) VALUES - ('tx1', 1, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), - ('tx2', 2, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), - ('tx3', 3, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) - `, now) + ($2, 1, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), + ($3, 2, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), + ($4, 3, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) + `, now, testHash1, testHash2, testHash3) require.NoError(t, err) // Create test state changes @@ -500,7 +507,7 @@ func TestStateChangeModel_BatchGetByAccountAddress(t *testing.T) { (1, 1, 'BALANCE', $1, 1, $2, 123), (2, 1, 'BALANCE', $1, 2, $2, 456), (3, 1, 'BALANCE', $1, 3, $3, 789) - `, now, address1, address2) + `, now, types.AddressBytea(address1), types.AddressBytea(address2)) require.NoError(t, err) mockMetricsService := metrics.NewMockMetricsService() @@ -518,7 +525,7 @@ func TestStateChangeModel_BatchGetByAccountAddress(t *testing.T) { require.NoError(t, err) assert.Len(t, stateChanges, 2) for _, sc := range stateChanges { - assert.Equal(t, address1, sc.AccountID) + assert.Equal(t, address1, sc.AccountID.String()) } // Test BatchGetByAccount for address2 @@ -526,7 +533,7 @@ func TestStateChangeModel_BatchGetByAccountAddress(t *testing.T) { require.NoError(t, err) assert.Len(t, stateChanges, 1) for _, sc := range stateChanges { - assert.Equal(t, address2, sc.AccountID) + assert.Equal(t, address2, sc.AccountID.String()) } } @@ -542,17 +549,21 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { // Create test account address := keypair.MustRandom().Address() - _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", address) + _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", types.AddressBytea(address)) require.NoError(t, err) - // Create test transactions + // Create test transactions (hash is BYTEA) + testHash1 := "0000000000000000000000000000000000000000000000000000000000000001" + testHash2 := "0000000000000000000000000000000000000000000000000000000000000002" + testHash3 := "0000000000000000000000000000000000000000000000000000000000000003" + testHashNonExistent := "0000000000000000000000000000000000000000000000000000000000000004" _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at) VALUES - ('tx1', 1, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1), - ('tx2', 2, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1), - ('tx3', 3, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1) - `, now) + ($2, 1, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1), + ($3, 2, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1), + ($4, 3, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1) + `, now, types.HashBytea(testHash1), types.HashBytea(testHash2), types.HashBytea(testHash3)) require.NoError(t, err) // Create test state changes with different operation IDs, categories, and reasons @@ -565,7 +576,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { (3, 1, 'SIGNER', 'ADD', $1, 3, $2, 789), (1, 2, 'BALANCE', 'DEBIT', $1, 4, $2, 124), (2, 2, 'SIGNER', 'ADD', $1, 5, $2, 999) - `, now, address) + `, now, types.AddressBytea(address)) require.NoError(t, err) t.Run("filter by transaction hash only", func(t *testing.T) { @@ -579,14 +590,14 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { MetricsService: mockMetricsService, } - txHash := "tx1" + txHash := testHash1 stateChanges, err := m.BatchGetByAccountAddress(ctx, address, &txHash, nil, nil, nil, "", nil, nil, ASC) require.NoError(t, err) // tx1 has to_id=1, so we get state changes where to_id=1 (2 state changes now) assert.Len(t, stateChanges, 2) for _, sc := range stateChanges { assert.Equal(t, int64(1), sc.ToID) - assert.Equal(t, address, sc.AccountID) + assert.Equal(t, address, sc.AccountID.String()) } }) @@ -608,7 +619,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { assert.Len(t, stateChanges, 1) for _, sc := range stateChanges { assert.Equal(t, int64(123), sc.OperationID) - assert.Equal(t, address, sc.AccountID) + assert.Equal(t, address, sc.AccountID.String()) } }) @@ -623,7 +634,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { MetricsService: mockMetricsService, } - txHash := "tx1" + txHash := testHash1 operationID := int64(123) stateChanges, err := m.BatchGetByAccountAddress(ctx, address, &txHash, &operationID, nil, nil, "", nil, nil, ASC) require.NoError(t, err) @@ -632,7 +643,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { for _, sc := range stateChanges { assert.Equal(t, int64(1), sc.ToID) assert.Equal(t, int64(123), sc.OperationID) - assert.Equal(t, address, sc.AccountID) + assert.Equal(t, address, sc.AccountID.String()) } }) @@ -653,7 +664,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { assert.Len(t, stateChanges, 3) for _, sc := range stateChanges { assert.Equal(t, types.StateChangeCategoryBalance, sc.StateChangeCategory) - assert.Equal(t, address, sc.AccountID) + assert.Equal(t, address, sc.AccountID.String()) } }) @@ -674,7 +685,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { assert.Len(t, stateChanges, 2) for _, sc := range stateChanges { assert.Equal(t, types.StateChangeReasonAdd, *sc.StateChangeReason) - assert.Equal(t, address, sc.AccountID) + assert.Equal(t, address, sc.AccountID.String()) } }) @@ -697,7 +708,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { for _, sc := range stateChanges { assert.Equal(t, types.StateChangeCategorySigner, sc.StateChangeCategory) assert.Equal(t, types.StateChangeReasonAdd, *sc.StateChangeReason) - assert.Equal(t, address, sc.AccountID) + assert.Equal(t, address, sc.AccountID.String()) } }) @@ -712,7 +723,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { MetricsService: mockMetricsService, } - txHash := "tx1" + txHash := testHash1 operationID := int64(123) category := "BALANCE" reason := "CREDIT" @@ -724,7 +735,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { assert.Equal(t, int64(123), sc.OperationID) assert.Equal(t, types.StateChangeCategoryBalance, sc.StateChangeCategory) assert.Equal(t, types.StateChangeReasonCredit, *sc.StateChangeReason) - assert.Equal(t, address, sc.AccountID) + assert.Equal(t, address, sc.AccountID.String()) } }) @@ -739,7 +750,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { MetricsService: mockMetricsService, } - txHash := "nonexistent" + txHash := testHashNonExistent stateChanges, err := m.BatchGetByAccountAddress(ctx, address, &txHash, nil, nil, nil, "", nil, nil, ASC) require.NoError(t, err) assert.Empty(t, stateChanges) @@ -756,7 +767,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { MetricsService: mockMetricsService, } - txHash := "tx1" + txHash := testHash1 limit := int32(1) stateChanges, err := m.BatchGetByAccountAddress(ctx, address, &txHash, nil, nil, nil, "", &limit, nil, ASC) require.NoError(t, err) @@ -787,17 +798,20 @@ func TestStateChangeModel_GetAll(t *testing.T) { // Create test account address := keypair.MustRandom().Address() - _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", address) + _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", types.AddressBytea(address)) require.NoError(t, err) - // Create test transactions first + // Create test transactions first (hash is BYTEA) + testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001") + testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002") + testHash3 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000003") _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) VALUES - ('tx1', 1, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), - ('tx2', 2, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), - ('tx3', 3, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) - `, now) + ($2, 1, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), + ($3, 2, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), + ($4, 3, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) + `, now, testHash1, testHash2, testHash3) require.NoError(t, err) // Create test state changes @@ -807,7 +821,7 @@ func TestStateChangeModel_GetAll(t *testing.T) { (1, 1, 'BALANCE', $1, 1, $2, 123), (2, 1, 'BALANCE', $1, 2, $2, 456), (3, 1, 'BALANCE', $1, 3, $2, 789) - `, now, address) + `, now, types.AddressBytea(address)) require.NoError(t, err) // Test GetAll without limit @@ -834,17 +848,20 @@ func TestStateChangeModel_BatchGetByToIDs(t *testing.T) { // Create test account address := keypair.MustRandom().Address() - _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", address) + _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", types.AddressBytea(address)) require.NoError(t, err) - // Create test transactions first + // Create test transactions first (hash is BYTEA) + testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001") + testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002") + testHash3 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000003") _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) VALUES - ('tx1', 1, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), - ('tx2', 2, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), - ('tx3', 3, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) - `, now) + ($2, 1, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), + ($3, 2, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), + ($4, 3, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) + `, now, testHash1, testHash2, testHash3) require.NoError(t, err) // Create test state changes - multiple state changes per to_id to test ranking @@ -857,7 +874,7 @@ func TestStateChangeModel_BatchGetByToIDs(t *testing.T) { (2, 1, 'BALANCE', $1, 4, $2, 456), (2, 2, 'BALANCE', $1, 5, $2, 457), (3, 1, 'BALANCE', $1, 6, $2, 789) - `, now, address) + `, now, types.AddressBytea(address)) require.NoError(t, err) testCases := []struct { @@ -990,17 +1007,20 @@ func TestStateChangeModel_BatchGetByOperationIDs(t *testing.T) { // Create test account address := keypair.MustRandom().Address() - _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", address) + _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", types.AddressBytea(address)) require.NoError(t, err) - // Create test transactions first + // Create test transactions first (hash is BYTEA) + testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001") + testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002") + testHash3 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000003") _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) VALUES - ('tx1', 1, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), - ('tx2', 2, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), - ('tx3', 3, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) - `, now) + ($2, 1, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), + ($3, 2, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), + ($4, 3, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) + `, now, testHash1, testHash2, testHash3) require.NoError(t, err) // Create test state changes @@ -1010,7 +1030,7 @@ func TestStateChangeModel_BatchGetByOperationIDs(t *testing.T) { (1, 1, 'BALANCE', $1, 1, $2, 123), (2, 1, 'BALANCE', $1, 2, $2, 456), (3, 1, 'BALANCE', $1, 3, $2, 123) - `, now, address) + `, now, types.AddressBytea(address)) require.NoError(t, err) // Test BatchGetByOperationID @@ -1050,16 +1070,18 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { // Create test account address := keypair.MustRandom().Address() - _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", address) + _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", types.AddressBytea(address)) require.NoError(t, err) - // Create test transactions first + // Create test transactions first (hash is BYTEA) + testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001") + testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002") _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) VALUES - ('tx1', 1, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), - ('tx2', 2, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true) - `, now) + ($2, 1, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), + ($3, 2, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true) + `, now, testHash1, testHash2) require.NoError(t, err) // Create test state changes for to_id=1 (multiple state_change_orders) @@ -1070,7 +1092,7 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { (1, 2, 'BALANCE', $1, 2, $2, 124), (1, 3, 'BALANCE', $1, 3, $2, 125), (2, 1, 'BALANCE', $1, 4, $2, 456) - `, now, address) + `, now, types.AddressBytea(address)) require.NoError(t, err) t.Run("get all state changes for single to_id", func(t *testing.T) { @@ -1162,6 +1184,12 @@ func BenchmarkStateChangeModel_BatchInsert(b *testing.B) { b.Fatalf("failed to create parent transaction: %v", err) } + // Pre-generate auxiliary addresses for nullable account_id fields + auxAddresses := make([]string, 10) + for i := range auxAddresses { + auxAddresses[i] = keypair.MustRandom().Address() + } + batchSizes := []int{1000, 5000, 10000, 50000, 100000} for _, size := range batchSizes { @@ -1174,7 +1202,7 @@ func BenchmarkStateChangeModel_BatchInsert(b *testing.B) { //nolint:errcheck // truncate is best-effort cleanup in benchmarks dbConnectionPool.ExecContext(ctx, "TRUNCATE state_changes CASCADE") // Generate fresh test data for each iteration - scs := generateTestStateChanges(size, accountID, int64(i*size)) + scs := generateTestStateChanges(size, accountID, int64(i*size), auxAddresses) b.StartTimer() _, err := m.BatchInsert(ctx, nil, scs) @@ -1227,6 +1255,12 @@ func BenchmarkStateChangeModel_BatchCopy(b *testing.B) { b.Fatalf("failed to create parent transaction: %v", err) } + // Pre-generate auxiliary addresses for nullable account_id fields + auxAddresses := make([]string, 10) + for i := range auxAddresses { + auxAddresses[i] = keypair.MustRandom().Address() + } + batchSizes := []int{1000, 5000, 10000, 50000, 100000} for _, size := range batchSizes { @@ -1242,7 +1276,7 @@ func BenchmarkStateChangeModel_BatchCopy(b *testing.B) { } // Generate fresh test data for each iteration - scs := generateTestStateChanges(size, accountID, int64(i*size)) + scs := generateTestStateChanges(size, accountID, int64(i*size), auxAddresses) // Start a pgx transaction pgxTx, err := conn.Begin(ctx) diff --git a/internal/data/transactions_test.go b/internal/data/transactions_test.go index 4140c9349..aba928b98 100644 --- a/internal/data/transactions_test.go +++ b/internal/data/transactions_test.go @@ -31,13 +31,13 @@ func generateTestTransactions(n int, startLedger int32) ([]*types.Transaction, m ledgerSeq := startLedger + int32(i) txIndex := int32(1) // First transaction in each ledger toID := toid.New(ledgerSeq, txIndex, 0).ToInt64() - hash := fmt.Sprintf("e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760-%d", toID) + hash := fmt.Sprintf("e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0%08x", i) envelope := "AAAAAgAAAAB/NpQ+s+cP+ztX7ryuKgXrxowZPHd4qAxhseOye/JeUgAehIAC2NL/AAflugAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAwAAAAFQQUxMAAAAAKHc4IKbcW8HPPgy3zOhuqv851y72nfLGa0HVXxIRNzHAAAAAAAAAAAAQ3FwMxshxQAfwV8AAAAAYTGQ3QAAAAAAAAAMAAAAAAAAAAFQQUxMAAAAAKHc4IKbcW8HPPgy3zOhuqv851y72nfLGa0HVXxIRNzHAAAAAAAGXwFksiHwAEXz8QAAAABhoaQjAAAAAAAAAAF78l5SAAAAQD7LgvZA8Pdvfh5L2b9B9RC7DlacGBJuOchuZDHQdVD1P0bn6nGQJXxDDI4oN76J49JxB7bIgDVim39MU43MOgE=" meta := "AAAAAwAAAAAAAAAEAAAAAwM6nhwAAAAAAAAAAJjy0MY1CPlZ/co80nzufVmo4gd7NqWMb+RiGiPhiviJAAAAC4SozKUDMWgAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAQM6nhwAAAAAAAAAAJjy0MY1CPlZ/co80nzufVmo4gd7NqWMb+RiGiPhiviJAAAAC4SozKUDMWgAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAwM6LTkAAAAAAAAAAKl6DQcpepRdTbO/Vw4hYBENfE/95GevM7SNA0ftK0gtAAAAA8Kuf0AC+zZCAAAATAAAAAMAAAABAAAAAMRxxkNwYslQaok0LlOKGtpATS9Bzx06JV9DIffG4OF1AAAAAAAAAAlsb2JzdHIuY28AAAABAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAMAAAAAAyZ54QAAAABmrTXCAAAAAAAAAAEDOp4cAAAAAAAAAACpeg0HKXqUXU2zv1cOIWARDXxP/eRnrzO0jQNH7StILQAAAAPCrn9AAvs2QgAAAE0AAAADAAAAAQAAAADEccZDcGLJUGqJNC5TihraQE0vQc8dOiVfQyH3xuDhdQAAAAAAAAAJbG9ic3RyLmNvAAAAAQAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAADAAAAAAM6nhwAAAAAZyGdHwAAAAAAAAABAAAABAAAAAMDOp4cAAAAAAAAAACpeg0HKXqUXU2zv1cOIWARDXxP/eRnrzO0jQNH7StILQAAAAPCrn9AAvs2QgAAAE0AAAADAAAAAQAAAADEccZDcGLJUGqJNC5TihraQE0vQc8dOiVfQyH3xuDhdQAAAAAAAAAJbG9ic3RyLmNvAAAAAQAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAADAAAAAAM6nhwAAAAAZyGdHwAAAAAAAAABAzqeHAAAAAAAAAAAqXoNByl6lF1Ns79XDiFgEQ18T/3kZ68ztI0DR+0rSC0AAAACmKiNQAL7NkIAAABNAAAAAwAAAAEAAAAAxHHGQ3BiyVBqiTQuU4oa2kBNL0HPHTolX0Mh98bg4XUAAAAAAAAACWxvYnN0ci5jbwAAAAEAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAwAAAAADOp4cAAAAAGchnR8AAAAAAAAAAwM6nZoAAAAAAAAAALKxMozkOH3rgpz3/u3+93wsR4p6z4K82HmJ5NTuaZbYAAACZaqAwoIBqycyAABVlQAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAMAAAAAAzqSNgAAAABnIVdaAAAAAAAAAAEDOp4cAAAAAAAAAACysTKM5Dh964Kc9/7t/vd8LEeKes+CvNh5ieTU7mmW2AAAAmbUhrSCAasnMgAAVZUAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAADAAAAAAM6kjYAAAAAZyFXWgAAAAAAAAAAAAAAAA==" address := keypair.MustRandom().Address() txs[i] = &types.Transaction{ - Hash: hash, + Hash: types.HashBytea(hash), ToID: toID, EnvelopeXDR: &envelope, FeeCharged: int64(100 * (i + 1)), @@ -66,14 +66,14 @@ func Test_TransactionModel_BatchInsert(t *testing.T) { // Create test data kp1 := keypair.MustRandom() kp2 := keypair.MustRandom() - const q = "INSERT INTO accounts (stellar_address) SELECT UNNEST(ARRAY[$1, $2])" - _, err = dbConnectionPool.ExecContext(ctx, q, kp1.Address(), kp2.Address()) + const q = "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)" + _, err = dbConnectionPool.ExecContext(ctx, q, types.AddressBytea(kp1.Address()), types.AddressBytea(kp2.Address())) require.NoError(t, err) meta1, meta2 := "meta1", "meta2" envelope1, envelope2 := "envelope1", "envelope2" tx1 := types.Transaction{ - Hash: "tx1", + Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", ToID: 1, EnvelopeXDR: &envelope1, FeeCharged: 100, @@ -84,7 +84,7 @@ func Test_TransactionModel_BatchInsert(t *testing.T) { IsFeeBump: false, } tx2 := types.Transaction{ - Hash: "tx2", + Hash: "a76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48761", ToID: 2, EnvelopeXDR: &envelope2, FeeCharged: 200, @@ -111,7 +111,7 @@ func Test_TransactionModel_BatchInsert(t *testing.T) { stellarAddressesByToID: map[int64]set.Set[string]{tx1.ToID: set.NewSet(kp1.Address()), tx2.ToID: set.NewSet(kp2.Address())}, wantAccountLinks: map[int64][]string{tx1.ToID: {kp1.Address()}, tx2.ToID: {kp2.Address()}}, wantErrContains: "", - wantHashes: []string{tx1.Hash, tx2.Hash}, + wantHashes: []string{tx1.Hash.String(), tx2.Hash.String()}, }, { name: "🟢successful_insert_with_dbTx", @@ -120,7 +120,7 @@ func Test_TransactionModel_BatchInsert(t *testing.T) { stellarAddressesByToID: map[int64]set.Set[string]{tx1.ToID: set.NewSet(kp1.Address())}, wantAccountLinks: map[int64][]string{tx1.ToID: {kp1.Address()}}, wantErrContains: "", - wantHashes: []string{tx1.Hash}, + wantHashes: []string{tx1.Hash.String()}, }, { name: "🟢empty_input", @@ -138,7 +138,7 @@ func Test_TransactionModel_BatchInsert(t *testing.T) { stellarAddressesByToID: map[int64]set.Set[string]{tx1.ToID: set.NewSet(kp1.Address())}, wantAccountLinks: map[int64][]string{tx1.ToID: {kp1.Address()}}, wantErrContains: "", - wantHashes: []string{tx1.Hash}, + wantHashes: []string{tx1.Hash.String()}, }, } @@ -185,17 +185,22 @@ func Test_TransactionModel_BatchInsert(t *testing.T) { // Verify the results require.NoError(t, err) - var dbInsertedHashes []string + var dbInsertedHashes []types.HashBytea err = sqlExecuter.SelectContext(ctx, &dbInsertedHashes, "SELECT hash FROM transactions") require.NoError(t, err) - assert.ElementsMatch(t, tc.wantHashes, dbInsertedHashes) + // Convert HashBytea to string for comparison + dbHashStrings := make([]string, len(dbInsertedHashes)) + for i, h := range dbInsertedHashes { + dbHashStrings[i] = h.String() + } + assert.ElementsMatch(t, tc.wantHashes, dbHashStrings) assert.ElementsMatch(t, tc.wantHashes, gotInsertedHashes) // Verify the account links if len(tc.wantAccountLinks) > 0 { var accountLinks []struct { - TxToID int64 `db:"tx_to_id"` - AccountID string `db:"account_id"` + TxToID int64 `db:"tx_to_id"` + AccountID types.AddressBytea `db:"account_id"` } err = sqlExecuter.SelectContext(ctx, &accountLinks, "SELECT tx_to_id, account_id FROM transactions_accounts ORDER BY tx_to_id, account_id") require.NoError(t, err) @@ -203,7 +208,7 @@ func Test_TransactionModel_BatchInsert(t *testing.T) { // Create a map of tx_to_id -> set of account_ids for O(1) lookups accountLinksMap := make(map[int64][]string) for _, link := range accountLinks { - accountLinksMap[link.TxToID] = append(accountLinksMap[link.TxToID], link.AccountID) + accountLinksMap[link.TxToID] = append(accountLinksMap[link.TxToID], string(link.AccountID)) } // Verify each transaction has its expected account links @@ -231,14 +236,14 @@ func Test_TransactionModel_BatchCopy(t *testing.T) { // Create test accounts kp1 := keypair.MustRandom() kp2 := keypair.MustRandom() - const q = "INSERT INTO accounts (stellar_address) SELECT UNNEST(ARRAY[$1, $2])" - _, err = dbConnectionPool.ExecContext(ctx, q, kp1.Address(), kp2.Address()) + const q = "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)" + _, err = dbConnectionPool.ExecContext(ctx, q, types.AddressBytea(kp1.Address()), types.AddressBytea(kp2.Address())) require.NoError(t, err) meta1, meta2 := "meta1", "meta2" envelope1, envelope2 := "envelope1", "envelope2" - tx1 := types.Transaction{ - Hash: "tx1", + txCopy1 := types.Transaction{ + Hash: "b76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48762", ToID: 1, EnvelopeXDR: &envelope1, FeeCharged: 100, @@ -248,8 +253,8 @@ func Test_TransactionModel_BatchCopy(t *testing.T) { LedgerCreatedAt: now, IsFeeBump: false, } - tx2 := types.Transaction{ - Hash: "tx2", + txCopy2 := types.Transaction{ + Hash: "c76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48763", ToID: 2, EnvelopeXDR: &envelope2, FeeCharged: 200, @@ -260,8 +265,8 @@ func Test_TransactionModel_BatchCopy(t *testing.T) { IsFeeBump: true, } // Transaction with nullable fields (nil envelope and meta) - tx3 := types.Transaction{ - Hash: "tx3", + txCopy3 := types.Transaction{ + Hash: "d76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48764", ToID: 3, EnvelopeXDR: nil, FeeCharged: 300, @@ -281,8 +286,8 @@ func Test_TransactionModel_BatchCopy(t *testing.T) { }{ { name: "🟢successful_insert_multiple", - txs: []*types.Transaction{&tx1, &tx2}, - stellarAddressesByToID: map[int64]set.Set[string]{tx1.ToID: set.NewSet(kp1.Address()), tx2.ToID: set.NewSet(kp2.Address())}, + txs: []*types.Transaction{&txCopy1, &txCopy2}, + stellarAddressesByToID: map[int64]set.Set[string]{txCopy1.ToID: set.NewSet(kp1.Address()), txCopy2.ToID: set.NewSet(kp2.Address())}, wantCount: 2, }, { @@ -293,13 +298,13 @@ func Test_TransactionModel_BatchCopy(t *testing.T) { }, { name: "🟢nullable_fields", - txs: []*types.Transaction{&tx3}, - stellarAddressesByToID: map[int64]set.Set[string]{tx3.ToID: set.NewSet(kp1.Address())}, + txs: []*types.Transaction{&txCopy3}, + stellarAddressesByToID: map[int64]set.Set[string]{txCopy3.ToID: set.NewSet(kp1.Address())}, wantCount: 1, }, { name: "🟢no_participants", - txs: []*types.Transaction{&tx1}, + txs: []*types.Transaction{&txCopy1}, stellarAddressesByToID: map[int64]set.Set[string]{}, wantCount: 1, }, @@ -356,7 +361,7 @@ func Test_TransactionModel_BatchCopy(t *testing.T) { assert.Equal(t, tc.wantCount, gotCount) // Verify from DB - var dbInsertedHashes []string + var dbInsertedHashes []types.HashBytea err = dbConnectionPool.SelectContext(ctx, &dbInsertedHashes, "SELECT hash FROM transactions ORDER BY hash") require.NoError(t, err) assert.Len(t, dbInsertedHashes, tc.wantCount) @@ -364,8 +369,8 @@ func Test_TransactionModel_BatchCopy(t *testing.T) { // Verify account links if expected if len(tc.stellarAddressesByToID) > 0 && tc.wantCount > 0 { var accountLinks []struct { - TxToID int64 `db:"tx_to_id"` - AccountID string `db:"account_id"` + TxToID int64 `db:"tx_to_id"` + AccountID types.AddressBytea `db:"account_id"` } err = dbConnectionPool.SelectContext(ctx, &accountLinks, "SELECT tx_to_id, account_id FROM transactions_accounts ORDER BY tx_to_id, account_id") require.NoError(t, err) @@ -373,7 +378,7 @@ func Test_TransactionModel_BatchCopy(t *testing.T) { // Create a map of tx_to_id -> set of account_ids accountLinksMap := make(map[int64][]string) for _, link := range accountLinks { - accountLinksMap[link.TxToID] = append(accountLinksMap[link.TxToID], link.AccountID) + accountLinksMap[link.TxToID] = append(accountLinksMap[link.TxToID], string(link.AccountID)) } // Verify each expected transaction has its account links @@ -399,13 +404,13 @@ func Test_TransactionModel_BatchCopy_DuplicateFails(t *testing.T) { // Create test account kp1 := keypair.MustRandom() const q = "INSERT INTO accounts (stellar_address) VALUES ($1)" - _, err = dbConnectionPool.ExecContext(ctx, q, kp1.Address()) + _, err = dbConnectionPool.ExecContext(ctx, q, types.AddressBytea(kp1.Address())) require.NoError(t, err) meta := "meta1" envelope := "envelope1" - tx1 := types.Transaction{ - Hash: "tx_duplicate_test", + txDup := types.Transaction{ + Hash: "f76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48766", ToID: 100, EnvelopeXDR: &envelope, FeeCharged: 100, @@ -420,14 +425,14 @@ func Test_TransactionModel_BatchCopy_DuplicateFails(t *testing.T) { sqlxDB, err := dbConnectionPool.SqlxDB(ctx) require.NoError(t, err) txModel := &TransactionModel{DB: dbConnectionPool, MetricsService: metrics.NewMetricsService(sqlxDB)} - _, err = txModel.BatchInsert(ctx, nil, []*types.Transaction{&tx1}, map[int64]set.Set[string]{ - tx1.ToID: set.NewSet(kp1.Address()), + _, err = txModel.BatchInsert(ctx, nil, []*types.Transaction{&txDup}, map[int64]set.Set[string]{ + txDup.ToID: set.NewSet(kp1.Address()), }) require.NoError(t, err) // Verify the transaction was inserted var count int - err = dbConnectionPool.GetContext(ctx, &count, "SELECT COUNT(*) FROM transactions WHERE hash = $1", tx1.Hash) + err = dbConnectionPool.GetContext(ctx, &count, "SELECT COUNT(*) FROM transactions WHERE hash = $1", txDup.Hash) require.NoError(t, err) require.Equal(t, 1, count) @@ -449,14 +454,13 @@ func Test_TransactionModel_BatchCopy_DuplicateFails(t *testing.T) { pgxTx, err := conn.Begin(ctx) require.NoError(t, err) - _, err = m.BatchCopy(ctx, pgxTx, []*types.Transaction{&tx1}, map[int64]set.Set[string]{ - tx1.ToID: set.NewSet(kp1.Address()), + _, err = m.BatchCopy(ctx, pgxTx, []*types.Transaction{&txDup}, map[int64]set.Set[string]{ + txDup.ToID: set.NewSet(kp1.Address()), }) // BatchCopy should fail with a unique constraint violation require.Error(t, err) - // TimescaleDB uses chunk-based constraint names like "1_1_transactions_pkey" instead of "transactions_pkey" - assert.Contains(t, err.Error(), "pgx CopyFrom transactions: ERROR: duplicate key value violates unique constraint") + assert.Contains(t, err.Error(), "duplicate key value violates unique constraint") // Rollback the failed transaction require.NoError(t, pgxTx.Rollback(ctx)) @@ -483,7 +487,7 @@ func TestTransactionModel_GetByHash(t *testing.T) { now := time.Now() // Create test transaction - txHash := "test_tx_hash" + txHash := types.HashBytea("0076b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4876") _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) VALUES ($1, 1, 'envelope', 100, 'TransactionResultCodeTxSuccess', 'meta', 1, $2, false) @@ -491,7 +495,7 @@ func TestTransactionModel_GetByHash(t *testing.T) { require.NoError(t, err) // Test GetByHash - transaction, err := m.GetByHash(ctx, txHash, "") + transaction, err := m.GetByHash(ctx, txHash.String(), "") require.NoError(t, err) assert.Equal(t, txHash, transaction.Hash) assert.Equal(t, int64(1), transaction.ToID) @@ -520,13 +524,16 @@ func TestTransactionModel_GetAll(t *testing.T) { now := time.Now() // Create test transactions + testHash1 := types.HashBytea("1076b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4876") + testHash2 := types.HashBytea("2076b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4876") + testHash3 := types.HashBytea("3076b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4876") _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) VALUES - ('tx1', 1, 'envelope1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), - ('tx2', 2, 'envelope2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), - ('tx3', 3, 'envelope3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) - `, now) + ($1, 1, 'envelope1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $4, false), + ($2, 2, 'envelope2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $4, true), + ($3, 3, 'envelope3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $4, false) + `, testHash1, testHash2, testHash3, now) require.NoError(t, err) // Test GetAll without specifying cursor and limit (gets all transactions) @@ -569,27 +576,31 @@ func TestTransactionModel_BatchGetByAccountAddress(t *testing.T) { // Create test accounts address1 := keypair.MustRandom().Address() address2 := keypair.MustRandom().Address() - _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)", address1, address2) + _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)", + types.AddressBytea(address1), types.AddressBytea(address2)) require.NoError(t, err) // Create test transactions + accTestHash1 := types.HashBytea("4076b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4876") + accTestHash2 := types.HashBytea("5076b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4876") + accTestHash3 := types.HashBytea("6076b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4876") _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) VALUES - ('tx1', 1, 'envelope1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), - ('tx2', 2, 'envelope2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), - ('tx3', 3, 'envelope3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) - `, now) + ($1, 1, 'envelope1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $4, false), + ($2, 2, 'envelope2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $4, true), + ($3, 3, 'envelope3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $4, false) + `, accTestHash1, accTestHash2, accTestHash3, now) require.NoError(t, err) - // Create test transactions_accounts links + // Create test transactions_accounts links (account_id is BYTEA) _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO transactions_accounts (ledger_created_at, tx_to_id, account_id) VALUES - ($1, 1, $2), - ($1, 2, $2), - ($1, 3, $3) - `, now, address1, address2) + ($3, 1, $1), + ($3, 2, $1), + ($3, 3, $2) + `, types.AddressBytea(address1), types.AddressBytea(address2), now) require.NoError(t, err) // Test BatchGetByAccount @@ -624,25 +635,31 @@ func TestTransactionModel_BatchGetByOperationIDs(t *testing.T) { // Create test transactions with specific ToIDs // Operations IDs must be in TOID range for each transaction: (to_id, to_id + 4096) + opTestHash1 := types.HashBytea("7076b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4876") + opTestHash2 := types.HashBytea("8076b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4876") + opTestHash3 := types.HashBytea("9076b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877") _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) VALUES - ('tx1', 4096, 'envelope1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), - ('tx2', 8192, 'envelope2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), - ('tx3', 12288, 'envelope3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) - `, now) + ($1, 4096, 'envelope1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $4, false), + ($2, 8192, 'envelope2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $4, true), + ($3, 12288, 'envelope3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $4, false) + `, opTestHash1, opTestHash2, opTestHash3, now) require.NoError(t, err) // Create test operations (IDs must be in TOID range for each transaction) - // tx1 (to_id=4096): ops 4097, 4098 - // tx2 (to_id=8192): op 8193 + // opTestHash1 (to_id=4096): ops 4097, 4098 + // opTestHash2 (to_id=8192): op 8193 + xdr1 := types.XDRBytea([]byte("xdr1")) + xdr2 := types.XDRBytea([]byte("xdr2")) + xdr3 := types.XDRBytea([]byte("xdr3")) _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) VALUES - (4097, 'PAYMENT', 'xdr1', 'op_success', true, 1, $1), - (8193, 'CREATE_ACCOUNT', 'xdr2', 'op_success', true, 2, $1), - (4098, 'PAYMENT', 'xdr3', 'op_success', true, 3, $1) - `, now) + (4097, 'PAYMENT', $2, 'op_success', true, 1, $1), + (8193, 'CREATE_ACCOUNT', $3, 'op_success', true, 2, $1), + (4098, 'PAYMENT', $4, 'op_success', true, 3, $1) + `, now, xdr1, xdr2, xdr3) require.NoError(t, err) // Test BatchGetByOperationIDs @@ -651,13 +668,13 @@ func TestTransactionModel_BatchGetByOperationIDs(t *testing.T) { assert.Len(t, transactions, 3) // Verify transactions are for correct operation IDs - operationIDsFound := make(map[int64]string) + operationIDsFound := make(map[int64]types.HashBytea) for _, tx := range transactions { operationIDsFound[tx.OperationID] = tx.Hash } - assert.Equal(t, "tx1", operationIDsFound[4097]) - assert.Equal(t, "tx2", operationIDsFound[8193]) - assert.Equal(t, "tx1", operationIDsFound[4098]) + assert.Equal(t, opTestHash1, operationIDsFound[4097]) + assert.Equal(t, opTestHash2, operationIDsFound[8193]) + assert.Equal(t, opTestHash1, operationIDsFound[4098]) } func TestTransactionModel_BatchGetByStateChangeIDs(t *testing.T) { @@ -683,17 +700,20 @@ func TestTransactionModel_BatchGetByStateChangeIDs(t *testing.T) { // Create test account address := keypair.MustRandom().Address() - _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", address) + _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", types.AddressBytea(address)) require.NoError(t, err) // Create test transactions + scTestHash1 := types.HashBytea("a176b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877") + scTestHash2 := types.HashBytea("b176b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877") + scTestHash3 := types.HashBytea("c176b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877") _, err = dbConnectionPool.ExecContext(ctx, ` INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) VALUES - ('tx1', 1, 'envelope1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false), - ('tx2', 2, 'envelope2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true), - ('tx3', 3, 'envelope3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false) - `, now) + ($1, 1, 'envelope1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $4, false), + ($2, 2, 'envelope2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $4, true), + ($3, 3, 'envelope3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $4, false) + `, scTestHash1, scTestHash2, scTestHash3, now) require.NoError(t, err) // Create test state changes @@ -713,13 +733,13 @@ func TestTransactionModel_BatchGetByStateChangeIDs(t *testing.T) { // Verify transactions are for correct state change IDs (format: to_id-operation_id-state_change_order) // State change (to_id, operation_id, order) should return transaction with matching to_id - stateChangeIDsFound := make(map[string]string) + stateChangeIDsFound := make(map[string]types.HashBytea) for _, tx := range transactions { stateChangeIDsFound[tx.StateChangeID] = tx.Hash } - assert.Equal(t, "tx1", stateChangeIDsFound["1-1-1"]) // to_id=1 -> tx1 (to_id=1) - assert.Equal(t, "tx2", stateChangeIDsFound["2-2-1"]) // to_id=2 -> tx2 (to_id=2) - assert.Equal(t, "tx3", stateChangeIDsFound["3-3-1"]) // to_id=3 -> tx3 (to_id=3) + assert.Equal(t, scTestHash1, stateChangeIDsFound["1-1-1"]) // to_id=1 -> scTestHash1 (to_id=1) + assert.Equal(t, scTestHash2, stateChangeIDsFound["2-2-1"]) // to_id=2 -> scTestHash2 (to_id=2) + assert.Equal(t, scTestHash3, stateChangeIDsFound["3-3-1"]) // to_id=3 -> scTestHash3 (to_id=3) } func BenchmarkTransactionModel_BatchInsert(b *testing.B) { diff --git a/internal/serve/graphql/resolvers/test_utils.go b/internal/serve/graphql/resolvers/test_utils.go index 0fea49fc6..bed41d477 100644 --- a/internal/serve/graphql/resolvers/test_utils.go +++ b/internal/serve/graphql/resolvers/test_utils.go @@ -7,6 +7,7 @@ import ( "time" "github.com/99designs/gqlgen/graphql" + "github.com/stellar/go-stellar-sdk/keypair" "github.com/stellar/go-stellar-sdk/toid" "github.com/stretchr/testify/require" @@ -46,15 +47,31 @@ func ptr[T any](v T) *T { return &v } +// sharedTestAccountAddress is a fixed test address used by tests that rely on setupDB. +// It's generated once and reused to ensure test data consistency. +var sharedTestAccountAddress = keypair.MustRandom().Address() + +// sharedNonExistentAccountAddress is a valid Stellar address that doesn't exist in the test DB. +var sharedNonExistentAccountAddress = keypair.MustRandom().Address() + +// Test transaction hashes used by setupDB (64-char hex strings for BYTEA storage) +// These must match the pattern in setupDB: fmt.Sprintf("...487%x", i) where i = 0,1,2,3 +var ( + testTxHash1 = "3476b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4870" + testTxHash2 = "3476b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4871" + testTxHash3 = "3476b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4872" + testTxHash4 = "3476b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4873" +) + func setupDB(ctx context.Context, t *testing.T, dbConnectionPool db.ConnectionPool) { testLedger := int32(1000) - parentAccount := &types.Account{StellarAddress: "test-account"} + parentAccount := &types.Account{StellarAddress: types.AddressBytea(sharedTestAccountAddress)} txns := make([]*types.Transaction, 0, 4) ops := make([]*types.Operation, 0, 8) opIdx := 1 for i := range 4 { txn := &types.Transaction{ - Hash: fmt.Sprintf("tx%d", i+1), + Hash: types.HashBytea(fmt.Sprintf("3476b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa487%x", i)), ToID: toid.New(testLedger, int32(i+1), 0).ToInt64(), EnvelopeXDR: ptr(fmt.Sprintf("envelope%d", i+1)), FeeCharged: int64(100 * (i + 1)), @@ -71,7 +88,7 @@ func setupDB(ctx context.Context, t *testing.T, dbConnectionPool db.ConnectionPo ops = append(ops, &types.Operation{ ID: toid.New(testLedger, int32(i+1), int32(j+1)).ToInt64(), OperationType: "PAYMENT", - OperationXDR: fmt.Sprintf("opxdr%d", opIdx), + OperationXDR: types.XDRBytea([]byte(fmt.Sprintf("opxdr%d", opIdx))), ResultCode: "op_success", Successful: true, LedgerNumber: 1, diff --git a/internal/services/ingest_test.go b/internal/services/ingest_test.go index 60d4dfdc9..0f906e73b 100644 --- a/internal/services/ingest_test.go +++ b/internal/services/ingest_test.go @@ -11,6 +11,7 @@ import ( "github.com/jackc/pgx/v5" "github.com/lib/pq" "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" + "github.com/stellar/go-stellar-sdk/keypair" "github.com/stellar/go-stellar-sdk/network" "github.com/stellar/go-stellar-sdk/toid" "github.com/stellar/go-stellar-sdk/xdr" @@ -28,9 +29,37 @@ import ( "github.com/stellar/wallet-backend/internal/signing/store" ) +// Test addresses generated from valid keypairs for use in tests. +// These are deterministic seeds to ensure consistent test addresses. +var ( + testKP1 = keypair.MustRandom() + testKP2 = keypair.MustRandom() + testKP3 = keypair.MustRandom() + testAddr1 = testKP1.Address() + testAddr2 = testKP2.Address() + testAddrUnreg = testKP3.Address() +) + const ( defaultGetLedgersLimit = 50 + // Test hash constants for ingest tests (64-char hex strings for BYTEA storage) + flushTxHash1 = "f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f101" + flushTxHash2 = "f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f202" + flushTxHash3 = "f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f303" + flushTxHash4 = "f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f404" + flushTxHash5 = "f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f505" + flushTxHash6 = "f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f606" + catchupTxHash1 = "c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c101" + catchupTxHash2 = "c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c202" + catchupTxHash3 = "c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c303" + catchupTxHash4 = "c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c404" + catchupTxHash5 = "c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c505" + catchupTxHash6 = "c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c606" + prevTxHash = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + txHash1 = "1111111111111111111111111111111111111111111111111111111111111111" + txHash2 = "2222222222222222222222222222222222222222222222222222222222222222" + // Test fixtures for ledger metadata ledgerMetadataWith0Tx = "AAAAAQAAAACB7Zh2o0NTFwl1nvs7xr3SJ7w8PpwnSRb8QyG9k6acEwAAABaeASPlzu/ZFxwyyWsxtGoj3KCrybm2yN7WOweR0BWdLYjyoO5BI41g1PFT+iHW68giP49Koo+q3VmH8I4GdtW2AAAAAGhTTB8AAAAAAAAAAQAAAAC1XRCyu30oTtXAOkel4bWQyQ9Xg1VHHMRQe76CBNI8iwAAAEDSH4sE7cL7UJyOqUo9ZZeNqPT7pt7su8iijHjWYg4MbeFUh/gkGf6N40bZjP/dlIuGXmuEhWoEX0VTV58xOB4C3z9hmASpL9tAVxktxD3XSOp3itxSvEmM6AUkwBS4ERm+pITz+1V1m+3/v6eaEKglCnon3a5xkn02sLltJ9CSzwAAEYIN4Lazp2QAAAAAAAMtYtQzAAAAAAAAAAAAAAAMAAAAZABMS0AAAADIXukLfWC53MCmzxKd/+LBbaYxQkgxATFDLI3hWj7EqWgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGeASPlzu/ZFxwyyWsxtGoj3KCrybm2yN7WOweR0BWdLQAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9yHMAAAAAAAAAAA==" ledgerMetadataWith1Tx = "AAAAAQAAAAD8G2qemHnBKFkbq90RTagxAypNnA7DXDc63Giipq9mNwAAABYLEZ5DrTv6njXTOAFEdOO0yeLtJjCRyH4ryJkgpRh7VPJvwbisrc9A0yzFxxCdkICgB3Gv7qHOi8ZdsK2CNks2AAAAAGhTTAsAAAAAAAAAAQAAAACoJM0YvJ11Bk0pmltbrKQ7w6ovMmk4FT2ML5u1y23wMwAAAEAunZtorOSbnRpgnykoDe4kzAvLwNXefncy1R/1ynBWyDv0DfdnqJ6Hcy/0AJf6DkBZlRayg775h3HjV0GKF/oPua7l8wkLlJBtSk1kRDt55qSf6btSrgcupB/8bnpJfUUgZJ76saUrj29HukYHS1bq7SyuoCAY+5F9iBYTmW1G9QAAEX4N4Lazp2QAAAAAAAMtS3veAAAAAAAAAAAAAAAMAAAAZABMS0AAAADIXukLfWC53MCmzxKd/+LBbaYxQkgxATFDLI3hWj7EqWgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAELEZ5DrTv6njXTOAFEdOO0yeLtJjCRyH4ryJkgpRh7VAAAAAIAAAAAAAAAAQAAAAAAAAABAAAAAAAAAGQAAAABAAAAAgAAAADg4mtiLKjJVgrmOpO9+Ff3XAmnycHyNUKu/v9KhHevAAAAAGQAAA7FAAAAGgAAAAAAAAAAAAAAAQAAAAAAAAABAAAAALvqzdVyRxgBMcLzbw1wNWcJYHPNPok1GdVSgmy4sjR2AAAAAVVTREMAAAAA4OJrYiyoyVYK5jqTvfhX91wJp8nB8jVCrv7/SoR3rwAAAAACVAvkAAAAAAAAAAABhHevAAAAAEDq2yIDzXUoLboBHQkbr8U2oKqLzf0gfpwXbmRPLB6Ek3G8uCEYyry1vt5Sb+LCEd81fefFQcQN0nydr1FmiXcDAAAAAAAAAAAAAAABXFSiWcxpDRa8frBs1wbEaMUw4hMe7ctFtdw3Ci73IEwAAAAAAAAAZAAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAAAAAAIAAAADAAARfQAAAAAAAAAA4OJrYiyoyVYK5jqTvfhX91wJp8nB8jVCrv7/SoR3rwAAAAAukO3GPAAADsUAAAAZAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAwAAAAAAABF9AAAAAGhTTAYAAAAAAAAAAQAAEX4AAAAAAAAAAODia2IsqMlWCuY6k734V/dcCafJwfI1Qq7+/0qEd68AAAAALpDtxdgAAA7FAAAAGQAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAMAAAAAAAARfQAAAABoU0wGAAAAAAAAAAMAAAAAAAAAAgAAAAMAABF+AAAAAAAAAADg4mtiLKjJVgrmOpO9+Ff3XAmnycHyNUKu/v9KhHevAAAAAC6Q7cXYAAAOxQAAABkAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAADAAAAAAAAEX0AAAAAaFNMBgAAAAAAAAABAAARfgAAAAAAAAAA4OJrYiyoyVYK5jqTvfhX91wJp8nB8jVCrv7/SoR3rwAAAAAukO3F2AAADsUAAAAaAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAwAAAAAAABF+AAAAAGhTTAsAAAAAAAAAAQAAAAIAAAADAAARcwAAAAEAAAAAu+rN1XJHGAExwvNvDXA1Zwlgc80+iTUZ1VKCbLiyNHYAAAABVVNEQwAAAADg4mtiLKjJVgrmOpO9+Ff3XAmnycHyNUKu/v9KhHevAAAAAAlQL5AAf/////////8AAAABAAAAAAAAAAAAAAABAAARfgAAAAEAAAAAu+rN1XJHGAExwvNvDXA1Zwlgc80+iTUZ1VKCbLiyNHYAAAABVVNEQwAAAADg4mtiLKjJVgrmOpO9+Ff3XAmnycHyNUKu/v9KhHevAAAAAAukO3QAf/////////8AAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA8RxEAAAAAAAAAAA==" @@ -496,7 +525,7 @@ func createTestTransaction(hash string, toID int64) types.Transaction { envelope := "test_envelope_xdr" meta := "test_meta_xdr" return types.Transaction{ - Hash: hash, + Hash: types.HashBytea(hash), ToID: toID, EnvelopeXDR: &envelope, FeeCharged: 100, @@ -514,7 +543,7 @@ func createTestOperation(id int64) types.Operation { return types.Operation{ ID: id, OperationType: types.OperationTypePayment, - OperationXDR: "test_operation_xdr", + OperationXDR: types.XDRBytea([]byte("test_operation_xdr")), LedgerNumber: 1000, LedgerCreatedAt: now, IngestedAt: now, @@ -530,7 +559,7 @@ func createTestStateChange(toID int64, accountID string, opID int64) types.State StateChangeOrder: 1, StateChangeCategory: types.StateChangeCategoryBalance, StateChangeReason: &reason, - AccountID: accountID, + AccountID: types.AddressBytea(accountID), OperationID: opID, LedgerNumber: 1000, LedgerCreatedAt: now, @@ -1140,17 +1169,17 @@ func Test_ingestService_flushBatchBufferWithRetry(t *testing.T) { name: "flush_with_data_inserts_to_database", setupBuffer: func() *indexer.IndexerBuffer { buf := indexer.NewIndexerBuffer() - tx1 := createTestTransaction("flush_tx_1", 1) - tx2 := createTestTransaction("flush_tx_2", 2) + tx1 := createTestTransaction(flushTxHash1, 1) + tx2 := createTestTransaction(flushTxHash2, 2) op1 := createTestOperation(200) op2 := createTestOperation(201) - sc1 := createTestStateChange(1, "GABC1111111111111111111111111111111111111111111111111", 200) - sc2 := createTestStateChange(2, "GDEF2222222222222222222222222222222222222222222222222", 201) + sc1 := createTestStateChange(1, testAddr1, 200) + sc2 := createTestStateChange(2, testAddr2, 201) - buf.PushTransaction("GABC1111111111111111111111111111111111111111111111111", tx1) - buf.PushTransaction("GDEF2222222222222222222222222222222222222222222222222", tx2) - buf.PushOperation("GABC1111111111111111111111111111111111111111111111111", op1, tx1) - buf.PushOperation("GDEF2222222222222222222222222222222222222222222222222", op2, tx2) + buf.PushTransaction(testAddr1, tx1) + buf.PushTransaction(testAddr2, tx2) + buf.PushOperation(testAddr1, op1, tx1) + buf.PushOperation(testAddr2, op2, tx2) buf.PushStateChange(tx1, op1, sc1) buf.PushStateChange(tx2, op2, sc2) return buf @@ -1161,14 +1190,14 @@ func Test_ingestService_flushBatchBufferWithRetry(t *testing.T) { wantTxCount: 2, wantOpCount: 2, wantStateChangeCount: 2, - txHashes: []string{"flush_tx_1", "flush_tx_2"}, + txHashes: []string{flushTxHash1, flushTxHash2}, }, { name: "flush_with_cursor_update_to_lower_value", setupBuffer: func() *indexer.IndexerBuffer { buf := indexer.NewIndexerBuffer() - tx1 := createTestTransaction("flush_tx_3", 3) - buf.PushTransaction("GABC1111111111111111111111111111111111111111111111111", tx1) + tx1 := createTestTransaction(flushTxHash3, 3) + buf.PushTransaction(testAddr1, tx1) return buf }, updateCursorTo: ptrUint32(50), @@ -1177,14 +1206,14 @@ func Test_ingestService_flushBatchBufferWithRetry(t *testing.T) { wantTxCount: 1, wantOpCount: 0, wantStateChangeCount: 0, - txHashes: []string{"flush_tx_3"}, + txHashes: []string{flushTxHash3}, }, { name: "flush_with_cursor_update_to_higher_value_keeps_existing", setupBuffer: func() *indexer.IndexerBuffer { buf := indexer.NewIndexerBuffer() - tx1 := createTestTransaction("flush_tx_4", 4) - buf.PushTransaction("GABC1111111111111111111111111111111111111111111111111", tx1) + tx1 := createTestTransaction(flushTxHash4, 4) + buf.PushTransaction(testAddr1, tx1) return buf }, updateCursorTo: ptrUint32(150), @@ -1193,38 +1222,38 @@ func Test_ingestService_flushBatchBufferWithRetry(t *testing.T) { wantTxCount: 1, wantOpCount: 0, wantStateChangeCount: 0, - txHashes: []string{"flush_tx_4"}, + txHashes: []string{flushTxHash4}, }, { name: "flush_with_filtering_only_inserts_registered", setupBuffer: func() *indexer.IndexerBuffer { buf := indexer.NewIndexerBuffer() - tx1 := createTestTransaction("flush_tx_5", 5) // Registered participant - tx2 := createTestTransaction("flush_tx_6", 6) // No registered participant + tx1 := createTestTransaction(flushTxHash5, 5) // Registered participant + tx2 := createTestTransaction(flushTxHash6, 6) // No registered participant - buf.PushTransaction("GREGISTERED111111111111111111111111111111111111111", tx1) - buf.PushTransaction("GUNREGISTERED11111111111111111111111111111111111111", tx2) + buf.PushTransaction(testAddr1, tx1) + buf.PushTransaction(testAddrUnreg, tx2) return buf }, enableParticipantFiltering: true, - registeredAccounts: []string{"GREGISTERED111111111111111111111111111111111111111"}, + registeredAccounts: []string{testAddr1}, updateCursorTo: nil, initialCursor: 100, wantCursor: 100, wantTxCount: 1, // Only tx1 wantOpCount: 0, wantStateChangeCount: 0, - txHashes: []string{"flush_tx_5"}, + txHashes: []string{flushTxHash5}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - // Clean up test data from previous runs - for _, hash := range []string{"flush_tx_1", "flush_tx_2", "flush_tx_3", "flush_tx_4", "flush_tx_5", "flush_tx_6"} { - _, err = dbConnectionPool.ExecContext(ctx, `DELETE FROM state_changes WHERE to_id IN (SELECT to_id FROM transactions WHERE hash = $1)`, hash) + // Clean up test data from previous runs (using HashBytea for BYTEA column) + for _, hash := range []string{flushTxHash1, flushTxHash2, flushTxHash3, flushTxHash4, flushTxHash5, flushTxHash6} { + _, err = dbConnectionPool.ExecContext(ctx, `DELETE FROM state_changes WHERE to_id IN (SELECT to_id FROM transactions WHERE hash = $1)`, types.HashBytea(hash)) require.NoError(t, err) - _, err = dbConnectionPool.ExecContext(ctx, `DELETE FROM transactions WHERE hash = $1`, hash) + _, err = dbConnectionPool.ExecContext(ctx, `DELETE FROM transactions WHERE hash = $1`, types.HashBytea(hash)) require.NoError(t, err) } // Also clean up any orphan operations @@ -1239,7 +1268,7 @@ func Test_ingestService_flushBatchBufferWithRetry(t *testing.T) { // Add registered accounts if any for _, acc := range tc.registeredAccounts { _, insertErr := dbConnectionPool.ExecContext(ctx, - `INSERT INTO accounts (stellar_address) VALUES ($1) ON CONFLICT DO NOTHING`, acc) + `INSERT INTO accounts (stellar_address) VALUES ($1) ON CONFLICT DO NOTHING`, types.AddressBytea(acc)) require.NoError(t, insertErr) } @@ -1298,10 +1327,16 @@ func Test_ingestService_flushBatchBufferWithRetry(t *testing.T) { // Verify transaction count in database if len(tc.txHashes) > 0 { + hashBytes := make([][]byte, len(tc.txHashes)) + for i, h := range tc.txHashes { + val, err := types.HashBytea(h).Value() + require.NoError(t, err) + hashBytes[i] = val.([]byte) + } var txCount int err = dbConnectionPool.GetContext(ctx, &txCount, `SELECT COUNT(*) FROM transactions WHERE hash = ANY($1)`, - pq.Array(tc.txHashes)) + pq.Array(hashBytes)) require.NoError(t, err) assert.Equal(t, tc.wantTxCount, txCount, "transaction count mismatch") } @@ -1317,10 +1352,16 @@ func Test_ingestService_flushBatchBufferWithRetry(t *testing.T) { // Verify state change count in database if tc.wantStateChangeCount > 0 { + scHashBytes := make([][]byte, len(tc.txHashes)) + for i, h := range tc.txHashes { + val, err := types.HashBytea(h).Value() + require.NoError(t, err) + scHashBytes[i] = val.([]byte) + } var scCount int err = dbConnectionPool.GetContext(ctx, &scCount, `SELECT COUNT(*) FROM state_changes WHERE to_id IN (SELECT to_id FROM transactions WHERE hash = ANY($1))`, - pq.Array(tc.txHashes)) + pq.Array(scHashBytes)) require.NoError(t, err) assert.Equal(t, tc.wantStateChangeCount, scCount, "state change count mismatch") } @@ -1352,17 +1393,17 @@ func Test_ingestService_filterParticipantData(t *testing.T) { enableParticipantFiltering: false, setupBuffer: func() *indexer.IndexerBuffer { buf := indexer.NewIndexerBuffer() - tx1 := createTestTransaction("tx_hash_1", 1) - tx2 := createTestTransaction("tx_hash_2", 2) + tx1 := createTestTransaction(txHash1, 1) + tx2 := createTestTransaction(txHash2, 2) op1 := createTestOperation(100) op2 := createTestOperation(101) - sc1 := createTestStateChange(1, "GABC1111111111111111111111111111111111111111111111111", 100) - sc2 := createTestStateChange(2, "GDEF2222222222222222222222222222222222222222222222222", 101) + sc1 := createTestStateChange(1, testAddr1, 100) + sc2 := createTestStateChange(2, testAddr2, 101) - buf.PushTransaction("GABC1111111111111111111111111111111111111111111111111", tx1) - buf.PushTransaction("GDEF2222222222222222222222222222222222222222222222222", tx2) - buf.PushOperation("GABC1111111111111111111111111111111111111111111111111", op1, tx1) - buf.PushOperation("GDEF2222222222222222222222222222222222222222222222222", op2, tx2) + buf.PushTransaction(testAddr1, tx1) + buf.PushTransaction(testAddr2, tx2) + buf.PushOperation(testAddr1, op1, tx1) + buf.PushOperation(testAddr2, op2, tx2) buf.PushStateChange(tx1, op1, sc1) buf.PushStateChange(tx2, op2, sc2) return buf @@ -1374,17 +1415,17 @@ func Test_ingestService_filterParticipantData(t *testing.T) { { name: "filtering_enabled_includes_tx_with_registered_participant", enableParticipantFiltering: true, - registeredAccounts: []string{"GABC1111111111111111111111111111111111111111111111111"}, + registeredAccounts: []string{testAddr1}, setupBuffer: func() *indexer.IndexerBuffer { buf := indexer.NewIndexerBuffer() - tx1 := createTestTransaction("tx_hash_1", 1) + tx1 := createTestTransaction(txHash1, 1) op1 := createTestOperation(100) - sc1 := createTestStateChange(1, "GABC1111111111111111111111111111111111111111111111111", 100) + sc1 := createTestStateChange(1, testAddr1, 100) // Tx has 2 participants but only 1 is registered - buf.PushTransaction("GABC1111111111111111111111111111111111111111111111111", tx1) - buf.PushTransaction("GXYZ9999999999999999999999999999999999999999999999999", tx1) // Unregistered participant on same tx - buf.PushOperation("GABC1111111111111111111111111111111111111111111111111", op1, tx1) + buf.PushTransaction(testAddr1, tx1) + buf.PushTransaction(testAddrUnreg, tx1) // Unregistered participant on same tx + buf.PushOperation(testAddr1, op1, tx1) buf.PushStateChange(tx1, op1, sc1) return buf }, @@ -1395,25 +1436,25 @@ func Test_ingestService_filterParticipantData(t *testing.T) { // Verify ALL participants are preserved (not just registered ones) participants := filtered.txParticipants[int64(1)] assert.Equal(t, 2, participants.Cardinality()) - assert.True(t, participants.Contains("GABC1111111111111111111111111111111111111111111111111")) - assert.True(t, participants.Contains("GXYZ9999999999999999999999999999999999999999999999999")) + assert.True(t, participants.Contains(testAddr1)) + assert.True(t, participants.Contains(testAddrUnreg)) }, }, { name: "filtering_enabled_excludes_tx_without_registered", enableParticipantFiltering: true, - registeredAccounts: []string{"GABC1111111111111111111111111111111111111111111111111"}, + registeredAccounts: []string{testAddr1}, setupBuffer: func() *indexer.IndexerBuffer { buf := indexer.NewIndexerBuffer() - tx1 := createTestTransaction("tx_hash_1", 1) // Has registered - tx2 := createTestTransaction("tx_hash_2", 2) // No registered + tx1 := createTestTransaction(txHash1, 1) // Has registered + tx2 := createTestTransaction(txHash2, 2) // No registered op1 := createTestOperation(100) op2 := createTestOperation(101) - buf.PushTransaction("GABC1111111111111111111111111111111111111111111111111", tx1) - buf.PushTransaction("GUNREGISTERED11111111111111111111111111111111111111", tx2) - buf.PushOperation("GABC1111111111111111111111111111111111111111111111111", op1, tx1) - buf.PushOperation("GUNREGISTERED11111111111111111111111111111111111111", op2, tx2) + buf.PushTransaction(testAddr1, tx1) + buf.PushTransaction(testAddrUnreg, tx2) + buf.PushOperation(testAddr1, op1, tx1) + buf.PushOperation(testAddrUnreg, op2, tx2) return buf }, wantTxCount: 1, // Only tx1 @@ -1426,8 +1467,8 @@ func Test_ingestService_filterParticipantData(t *testing.T) { registeredAccounts: []string{}, setupBuffer: func() *indexer.IndexerBuffer { buf := indexer.NewIndexerBuffer() - tx1 := createTestTransaction("tx_hash_1", 1) - buf.PushTransaction("GUNREGISTERED11111111111111111111111111111111111111", tx1) + tx1 := createTestTransaction(txHash1, 1) + buf.PushTransaction(testAddrUnreg, tx1) return buf }, wantTxCount: 0, @@ -1437,19 +1478,19 @@ func Test_ingestService_filterParticipantData(t *testing.T) { { name: "filtering_state_changes_only_for_registered_accounts", enableParticipantFiltering: true, - registeredAccounts: []string{"GABC1111111111111111111111111111111111111111111111111", "GDEF2222222222222222222222222222222222222222222222222"}, + registeredAccounts: []string{testAddr1, testAddr2}, setupBuffer: func() *indexer.IndexerBuffer { buf := indexer.NewIndexerBuffer() - tx1 := createTestTransaction("tx_hash_1", 1) + tx1 := createTestTransaction(txHash1, 1) op1 := createTestOperation(100) // 3 state changes: 2 for registered accounts, 1 for unregistered - sc1 := createTestStateChange(1, "GABC1111111111111111111111111111111111111111111111111", 100) - sc2 := createTestStateChange(2, "GDEF2222222222222222222222222222222222222222222222222", 100) - sc3 := createTestStateChange(3, "GUNREGISTERED11111111111111111111111111111111111111", 100) + sc1 := createTestStateChange(1, testAddr1, 100) + sc2 := createTestStateChange(2, testAddr2, 100) + sc3 := createTestStateChange(3, testAddrUnreg, 100) - buf.PushTransaction("GABC1111111111111111111111111111111111111111111111111", tx1) - buf.PushOperation("GABC1111111111111111111111111111111111111111111111111", op1, tx1) + buf.PushTransaction(testAddr1, tx1) + buf.PushOperation(testAddr1, op1, tx1) buf.PushStateChange(tx1, op1, sc1) buf.PushStateChange(tx1, op1, sc2) buf.PushStateChange(tx1, op1, sc3) @@ -1470,7 +1511,7 @@ func Test_ingestService_filterParticipantData(t *testing.T) { // Add registered accounts if any for _, acc := range tc.registeredAccounts { _, insertErr := dbConnectionPool.ExecContext(ctx, - `INSERT INTO accounts (stellar_address) VALUES ($1) ON CONFLICT DO NOTHING`, acc) + `INSERT INTO accounts (stellar_address) VALUES ($1) ON CONFLICT DO NOTHING`, types.AddressBytea(acc)) require.NoError(t, insertErr) } @@ -2638,10 +2679,10 @@ func Test_ingestService_flushBatchBuffer_batchChanges(t *testing.T) { name: "collects_trustline_changes_when_batchChanges_provided", setupBuffer: func() *indexer.IndexerBuffer { buf := indexer.NewIndexerBuffer() - tx1 := createTestTransaction("catchup_tx_1", 1) - buf.PushTransaction("GTEST111111111111111111111111111111111111111111111111", tx1) + tx1 := createTestTransaction(catchupTxHash1, 1) + buf.PushTransaction(testAddr1, tx1) buf.PushTrustlineChange(types.TrustlineChange{ - AccountID: "GTEST111111111111111111111111111111111111111111111111", + AccountID: testAddr1, Asset: "USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN", OperationID: 100, LedgerNumber: 1000, @@ -2657,10 +2698,10 @@ func Test_ingestService_flushBatchBuffer_batchChanges(t *testing.T) { name: "collects_contract_changes_when_batchChanges_provided", setupBuffer: func() *indexer.IndexerBuffer { buf := indexer.NewIndexerBuffer() - tx1 := createTestTransaction("catchup_tx_2", 2) - buf.PushTransaction("GTEST222222222222222222222222222222222222222222222222", tx1) + tx1 := createTestTransaction(catchupTxHash2, 2) + buf.PushTransaction(testAddr2, tx1) buf.PushContractChange(types.ContractChange{ - AccountID: "GTEST222222222222222222222222222222222222222222222222", + AccountID: testAddr2, ContractID: "CCONTRACTID", OperationID: 101, LedgerNumber: 1001, @@ -2672,7 +2713,7 @@ func Test_ingestService_flushBatchBuffer_batchChanges(t *testing.T) { wantTrustlineChangesCount: 0, wantContractChanges: []types.ContractChange{ { - AccountID: "GTEST222222222222222222222222222222222222222222222222", + AccountID: testAddr2, ContractID: "CCONTRACTID", OperationID: 101, LedgerNumber: 1001, @@ -2684,10 +2725,10 @@ func Test_ingestService_flushBatchBuffer_batchChanges(t *testing.T) { name: "nil_batchChanges_does_not_collect", setupBuffer: func() *indexer.IndexerBuffer { buf := indexer.NewIndexerBuffer() - tx1 := createTestTransaction("catchup_tx_5", 5) - buf.PushTransaction("GTEST555555555555555555555555555555555555555555555555", tx1) + tx1 := createTestTransaction(catchupTxHash5, 5) + buf.PushTransaction(testAddr1, tx1) buf.PushTrustlineChange(types.TrustlineChange{ - AccountID: "GTEST555555555555555555555555555555555555555555555555", + AccountID: testAddr1, Asset: "EUR:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN", OperationID: 102, LedgerNumber: 1002, @@ -2703,10 +2744,10 @@ func Test_ingestService_flushBatchBuffer_batchChanges(t *testing.T) { name: "accumulates_across_multiple_flushes", setupBuffer: func() *indexer.IndexerBuffer { buf := indexer.NewIndexerBuffer() - tx1 := createTestTransaction("catchup_tx_6", 6) - buf.PushTransaction("GTEST666666666666666666666666666666666666666666666666", tx1) + tx1 := createTestTransaction(catchupTxHash6, 6) + buf.PushTransaction(testAddr1, tx1) buf.PushTrustlineChange(types.TrustlineChange{ - AccountID: "GTEST666666666666666666666666666666666666666666666666", + AccountID: testAddr1, Asset: "GBP:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN", OperationID: 103, LedgerNumber: 1003, @@ -2729,11 +2770,11 @@ func Test_ingestService_flushBatchBuffer_batchChanges(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - // Clean up test data from previous runs - for _, hash := range []string{"catchup_tx_1", "catchup_tx_2", "catchup_tx_3", "catchup_tx_4", "catchup_tx_5", "catchup_tx_6", "prev_tx"} { - _, err = dbConnectionPool.ExecContext(ctx, `DELETE FROM state_changes WHERE to_id IN (SELECT to_id FROM transactions WHERE hash = $1)`, hash) + // Clean up test data from previous runs (using HashBytea for BYTEA column) + for _, hash := range []string{catchupTxHash1, catchupTxHash2, catchupTxHash3, catchupTxHash4, catchupTxHash5, catchupTxHash6, prevTxHash} { + _, err = dbConnectionPool.ExecContext(ctx, `DELETE FROM state_changes WHERE to_id IN (SELECT to_id FROM transactions WHERE hash = $1)`, types.HashBytea(hash)) require.NoError(t, err) - _, err = dbConnectionPool.ExecContext(ctx, `DELETE FROM transactions WHERE hash = $1`, hash) + _, err = dbConnectionPool.ExecContext(ctx, `DELETE FROM transactions WHERE hash = $1`, types.HashBytea(hash)) require.NoError(t, err) } // Also clean up any orphan operations @@ -2884,12 +2925,10 @@ func Test_ingestService_processLedgersInBatch_catchupMode(t *testing.T) { require.NoError(t, err) batch := BackfillBatch{StartLedger: 4599, EndLedger: 4599} - ledgersProcessed, batchChanges, startTime, endTime, err := svc.processLedgersInBatch(ctx, mockLedgerBackend, batch, tc.mode) + ledgersProcessed, batchChanges, _, _, err := svc.processLedgersInBatch(ctx, mockLedgerBackend, batch, tc.mode) require.NoError(t, err) assert.Equal(t, 1, ledgersProcessed) - assert.False(t, startTime.IsZero(), "expected non-zero start time") - assert.False(t, endTime.IsZero(), "expected non-zero end time") if tc.wantBatchChangesNil { assert.Nil(t, batchChanges, "expected nil batch changes for historical mode") From ec91fe70e88e92affb5536459328ce2918972309 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 10 Feb 2026 14:56:12 -0500 Subject: [PATCH 19/77] Fix shadow variable warnings in BatchCopy address conversions Rename inner err to addrErr in address BYTEA conversion loops to avoid shadowing the outer err variable from pgx.CopyFrom. --- internal/data/operations.go | 6 +++--- internal/data/transactions.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/data/operations.go b/internal/data/operations.go index 313963035..346782cf4 100644 --- a/internal/data/operations.go +++ b/internal/data/operations.go @@ -459,9 +459,9 @@ func (m *OperationModel) BatchCopy( ledgerCreatedAtPgtype := pgtype.Timestamptz{Time: ledgerCreatedAt, Valid: true} opIDPgtype := pgtype.Int8{Int64: opID, Valid: true} for _, addr := range addresses.ToSlice() { - addrBytes, err := types.AddressBytea(addr).Value() - if err != nil { - return 0, fmt.Errorf("converting address %s to bytes: %w", addr, err) + addrBytes, addrErr := types.AddressBytea(addr).Value() + if addrErr != nil { + return 0, fmt.Errorf("converting address %s to bytes: %w", addr, addrErr) } oaRows = append(oaRows, []any{ ledgerCreatedAtPgtype, diff --git a/internal/data/transactions.go b/internal/data/transactions.go index 758dc9a4a..4a4f5e901 100644 --- a/internal/data/transactions.go +++ b/internal/data/transactions.go @@ -392,9 +392,9 @@ func (m *TransactionModel) BatchCopy( ledgerCreatedAtPgtype := pgtype.Timestamptz{Time: ledgerCreatedAt, Valid: true} toIDPgtype := pgtype.Int8{Int64: toID, Valid: true} for _, addr := range addresses.ToSlice() { - addrBytes, err := types.AddressBytea(addr).Value() - if err != nil { - return 0, fmt.Errorf("converting address %s to bytes: %w", addr, err) + addrBytes, addrErr := types.AddressBytea(addr).Value() + if addrErr != nil { + return 0, fmt.Errorf("converting address %s to bytes: %w", addr, addrErr) } taRows = append(taRows, []any{ ledgerCreatedAtPgtype, From d013efaa869a701ea737e5cad2528ebd088363a4 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 10 Feb 2026 15:02:52 -0500 Subject: [PATCH 20/77] Update go.yaml --- .github/workflows/go.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml index 7aed3f3a0..a19b86c39 100644 --- a/.github/workflows/go.yaml +++ b/.github/workflows/go.yaml @@ -87,7 +87,7 @@ jobs: runs-on: ubuntu-latest services: postgres: - image: postgres:12-alpine + image: timescale/timescaledb:latest-pg17 env: POSTGRES_USER: postgres POSTGRES_DB: postgres From 3abf801a28d580f2197e168aa3139ae92ba3d8ec Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 10 Feb 2026 15:22:51 -0500 Subject: [PATCH 21/77] Update go.yaml --- .github/workflows/go.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml index a19b86c39..b72661f5a 100644 --- a/.github/workflows/go.yaml +++ b/.github/workflows/go.yaml @@ -87,7 +87,7 @@ jobs: runs-on: ubuntu-latest services: postgres: - image: timescale/timescaledb:latest-pg17 + image: timescale/timescaledb:2.25.0-pg17 env: POSTGRES_USER: postgres POSTGRES_DB: postgres @@ -101,7 +101,7 @@ jobs: ports: - 5432:5432 env: - PGHOST: localhost + PGHOST: /var/run/postgresql PGPORT: 5432 PGUSER: postgres PGPASSWORD: postgres From 2b9e98e629ed6f7ac4db32397c14d61ee55c3e5e Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 10 Feb 2026 15:25:37 -0500 Subject: [PATCH 22/77] Update go.yaml --- .github/workflows/go.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml index b72661f5a..3ad8c5886 100644 --- a/.github/workflows/go.yaml +++ b/.github/workflows/go.yaml @@ -92,7 +92,7 @@ jobs: POSTGRES_USER: postgres POSTGRES_DB: postgres POSTGRES_PASSWORD: postgres - PGHOST: localhost + PGHOST: /var/run/postgresql options: >- --health-cmd pg_isready --health-interval 10s @@ -101,7 +101,7 @@ jobs: ports: - 5432:5432 env: - PGHOST: /var/run/postgresql + PGHOST: localhost PGPORT: 5432 PGUSER: postgres PGPASSWORD: postgres From 6c55228c47d67ab404b53a3132356fb687444ee9 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 10 Feb 2026 15:27:00 -0500 Subject: [PATCH 23/77] Update docker-compose.yaml --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 5a28d0f09..f1ac51598 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,7 +7,7 @@ services: db: container_name: db - image: timescale/timescaledb:latest-pg17 + image: timescale/timescaledb:2.25.0-pg17 healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres -d wallet-backend"] interval: 10s From 9f41d08126fddf7650ba4027d85b3b825689a75b Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 10 Feb 2026 15:54:20 -0500 Subject: [PATCH 24/77] Update query_utils.go --- internal/data/query_utils.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/data/query_utils.go b/internal/data/query_utils.go index d2304136a..35fbd70d4 100644 --- a/internal/data/query_utils.go +++ b/internal/data/query_utils.go @@ -162,7 +162,12 @@ func prepareColumnsWithID(columns string, model any, prefix string, idColumns .. dbColumns = getDBColumns(model) } else { dbColumns = set.NewSet[string]() - dbColumns.Add(columns) + for _, col := range strings.Split(columns, ",") { + col = strings.TrimSpace(col) + if col != "" { + dbColumns.Add(col) + } + } } if prefix != "" { From d34dc37a61fe262699c4a59b5f7bd58db9a44f4f Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 10 Feb 2026 16:06:55 -0500 Subject: [PATCH 25/77] remove flaky RPC test --- .../infrastructure/main_setup.go | 6 - internal/services/rpc_service.go | 75 ------ internal/services/rpc_service_test.go | 241 ------------------ 3 files changed, 322 deletions(-) diff --git a/internal/integrationtests/infrastructure/main_setup.go b/internal/integrationtests/infrastructure/main_setup.go index ab6dccff7..6d3f89d53 100644 --- a/internal/integrationtests/infrastructure/main_setup.go +++ b/internal/integrationtests/infrastructure/main_setup.go @@ -577,12 +577,6 @@ func createRPCService(ctx context.Context, containers *SharedContainers) (servic // This prevents the immediate health check from failing due to transient unavailability. time.Sleep(3 * time.Second) - // Start tracking RPC health - go func() { - //nolint:errcheck // Error is expected on context cancellation during shutdown - rpcService.TrackRPCServiceHealth(ctx, nil) - }() - return rpcService, nil } diff --git a/internal/services/rpc_service.go b/internal/services/rpc_service.go index b7792ec1a..cecd00209 100644 --- a/internal/services/rpc_service.go +++ b/internal/services/rpc_service.go @@ -10,7 +10,6 @@ import ( "net/http" "time" - "github.com/stellar/go-stellar-sdk/support/log" "github.com/stellar/go-stellar-sdk/xdr" "github.com/stellar/stellar-rpc/protocol" @@ -34,14 +33,6 @@ type RPCService interface { GetLedgerEntries(keys []string) (entities.RPCGetLedgerEntriesResult, error) GetAccountLedgerSequence(address string) (int64, error) GetHeartbeatChannel() chan entities.RPCGetHealthResult - // TrackRPCServiceHealth continuously monitors the health of the RPC service and updates metrics. - // It runs health checks at regular intervals and can be triggered on-demand via immediateHealthCheckTrigger. - // - // The immediateHealthCheckTrigger channel allows external components to request an immediate health check, - // which is particularly useful when the ingestor needs to catch up with the RPC service. - // - // Returns an error if the context is cancelled. The caller is responsible for handling shutdown signals. - TrackRPCServiceHealth(ctx context.Context, immediateHealthCheckTrigger <-chan any) error SimulateTransaction(transactionXDR string, resourceConfig entities.RPCResourceConfig) (entities.RPCSimulateTransactionResult, error) NetworkPassphrase() string } @@ -327,72 +318,6 @@ func (r *rpcService) HealthCheckTickInterval() time.Duration { return r.healthCheckTickInterval } -// TrackRPCServiceHealth continuously monitors the health of the RPC service and updates metrics. -// It runs health checks at regular intervals and can be triggered on-demand via immediateHealthCheckTrigger. -// -// The immediateHealthCheckTrigger channel allows external components to request an immediate health check, -// which is particularly useful when the ingestor needs to catch up with the RPC service. -// -// Returns an error if the context is cancelled. The caller is responsible for handling shutdown signals. -func (r *rpcService) TrackRPCServiceHealth(ctx context.Context, immediateHealthCheckTrigger <-chan any) error { - // Handle nil channel by creating a never-firing channel - if immediateHealthCheckTrigger == nil { - immediateHealthCheckTrigger = make(chan any) - } - - healthCheckTicker := time.NewTicker(r.HealthCheckTickInterval()) - unhealthyWarningTicker := time.NewTicker(r.HealthCheckWarningInterval()) - defer func() { - healthCheckTicker.Stop() - unhealthyWarningTicker.Stop() - close(r.heartbeatChannel) - }() - - // performHealthCheck is a function that performs a health check and updates the metrics. - performHealthCheck := func() { - health, err := r.GetHealth() - if err != nil { - log.Ctx(ctx).Warnf("RPC health check failed: %v", err) - r.metricsService.SetRPCServiceHealth(false) - return - } - - unhealthyWarningTicker.Reset(r.HealthCheckWarningInterval()) - select { - case r.heartbeatChannel <- health: - // sent successfully - default: - // channel is full, clear it and send latest - <-r.heartbeatChannel - r.heartbeatChannel <- health - } - r.metricsService.SetRPCServiceHealth(true) - r.metricsService.SetRPCLatestLedger(int64(health.LatestLedger)) - } - - // Perform immediate health check at startup to avoid 5-second delay - performHealthCheck() - - for { - select { - case <-ctx.Done(): - log.Ctx(ctx).Infof("RPC health tracking stopped due to context cancellation: %v", ctx.Err()) - return fmt.Errorf("context cancelled: %w", ctx.Err()) - - case <-unhealthyWarningTicker.C: - log.Ctx(ctx).Warnf("RPC service unhealthy for over %s", r.HealthCheckWarningInterval()) - r.metricsService.SetRPCServiceHealth(false) - - case <-healthCheckTicker.C: - performHealthCheck() - - case <-immediateHealthCheckTrigger: - healthCheckTicker.Reset(r.HealthCheckTickInterval()) - performHealthCheck() - } - } -} - func (r *rpcService) sendRPCRequest(method string, params entities.RPCParams) (json.RawMessage, error) { startTime := time.Now() r.metricsService.IncRPCRequests(method) diff --git a/internal/services/rpc_service_test.go b/internal/services/rpc_service_test.go index d29331f97..6d14fbbf0 100644 --- a/internal/services/rpc_service_test.go +++ b/internal/services/rpc_service_test.go @@ -2,7 +2,6 @@ package services import ( "bytes" - "context" "encoding/json" "errors" "fmt" @@ -13,7 +12,6 @@ import ( "time" "github.com/stellar/go-stellar-sdk/network" - "github.com/stellar/go-stellar-sdk/support/log" "github.com/stellar/go-stellar-sdk/xdr" "github.com/stellar/stellar-rpc/protocol" "github.com/stretchr/testify/assert" @@ -876,242 +874,3 @@ func Test_rpcService_GetLedgers(t *testing.T) { assert.Equal(t, "sending getLedgers request: sending POST request to RPC: connection failed", err.Error()) }) } - -func TestTrackRPCServiceHealth_HealthyService(t *testing.T) { - dbt := dbtest.Open(t) - defer dbt.Close() - - dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) - require.NoError(t, err) - defer dbConnectionPool.Close() - - mockMetricsService := metrics.NewMockMetricsService() - mockMetricsService.On("IncRPCMethodCalls", "GetHealth").Once() - mockMetricsService.On("ObserveRPCMethodDuration", "GetHealth", mock.AnythingOfType("float64")).Once() - mockMetricsService.On("IncRPCRequests", "getHealth").Once() - mockMetricsService.On("IncRPCEndpointSuccess", "getHealth").Once() - mockMetricsService.On("ObserveRPCRequestDuration", "getHealth", mock.AnythingOfType("float64")).Once() - mockMetricsService.On("SetRPCServiceHealth", true).Once() - mockMetricsService.On("SetRPCLatestLedger", int64(100)).Once() - defer mockMetricsService.AssertExpectations(t) - - mockHTTPClient := &utils.MockHTTPClient{} - rpcURL := "http://test-url-track-rpc-service-health" - rpcService, err := NewRPCService(rpcURL, network.TestNetworkPassphrase, mockHTTPClient, mockMetricsService) - require.NoError(t, err) - - healthResult := entities.RPCGetHealthResult{ - Status: "healthy", - LatestLedger: 100, - OldestLedger: 1, - LedgerRetentionWindow: 0, - } - - // Mock the HTTP response for GetHealth - ctx, cancel := context.WithTimeout(context.Background(), rpcService.HealthCheckTickInterval()*2) - defer cancel() - mockResponse := &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewBuffer([]byte(`{ - "jsonrpc": "2.0", - "id": 1, - "result": { - "status": "healthy", - "latestLedger": 100, - "oldestLedger": 1, - "ledgerRetentionWindow": 0 - } - }`))), - } - mockHTTPClient.On("Post", rpcURL, "application/json", mock.Anything).Return(mockResponse, nil).Run(func(args mock.Arguments) { - cancel() - }) - err = rpcService.TrackRPCServiceHealth(ctx, nil) - require.Error(t, err) - - // Get result from heartbeat channel - select { - case result := <-rpcService.GetHeartbeatChannel(): - assert.Equal(t, healthResult, result) - case <-time.After(10 * time.Second): - t.Fatal("timeout waiting for heartbeat") - } - - mockHTTPClient.AssertExpectations(t) -} - -func TestTrackRPCServiceHealth_UnhealthyService(t *testing.T) { - healthCheckTickInterval := 300 * time.Millisecond - healthCheckWarningInterval := 400 * time.Millisecond - contextTimeout := healthCheckWarningInterval + time.Millisecond*190 - ctx, cancel := context.WithTimeout(context.Background(), contextTimeout) - defer cancel() - - dbt := dbtest.Open(t) - defer dbt.Close() - - dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) - require.NoError(t, err) - defer dbConnectionPool.Close() - - mockMetricsService := metrics.NewMockMetricsService() - mockMetricsService.On("IncRPCMethodCalls", "GetHealth").Once() - mockMetricsService.On("IncRPCMethodCalls", "GetHealth").Maybe() - mockMetricsService.On("ObserveRPCMethodDuration", "GetHealth", mock.AnythingOfType("float64")).Once() - mockMetricsService.On("ObserveRPCMethodDuration", "GetHealth", mock.AnythingOfType("float64")).Maybe() - mockMetricsService.On("IncRPCRequests", "getHealth").Once() - mockMetricsService.On("IncRPCRequests", "getHealth").Maybe() - mockMetricsService.On("IncRPCEndpointFailure", "getHealth").Once() - mockMetricsService.On("IncRPCEndpointFailure", "getHealth").Maybe() - mockMetricsService.On("ObserveRPCRequestDuration", "getHealth", mock.AnythingOfType("float64")).Once() - mockMetricsService.On("ObserveRPCRequestDuration", "getHealth", mock.AnythingOfType("float64")).Maybe() - mockMetricsService.On("IncRPCMethodErrors", "GetHealth", "rpc_error").Once() - mockMetricsService.On("IncRPCMethodErrors", "GetHealth", "rpc_error").Maybe() - mockMetricsService.On("SetRPCServiceHealth", false).Once() - mockMetricsService.On("SetRPCServiceHealth", false).Maybe() - defer mockMetricsService.AssertExpectations(t) - getLogs := log.DefaultLogger.StartTest(log.WarnLevel) - - mockHTTPClient := &utils.MockHTTPClient{} - defer mockHTTPClient.AssertExpectations(t) - rpcURL := "http://test-url-track-rpc-service-health" - rpcService, err := NewRPCService(rpcURL, network.TestNetworkPassphrase, mockHTTPClient, mockMetricsService) - require.NoError(t, err) - rpcService.healthCheckTickInterval = healthCheckTickInterval - rpcService.healthCheckWarningInterval = healthCheckWarningInterval - - // Mock error response for GetHealth with a valid http.Response - getHealthRequestBody, err := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": 1, "method": "getHealth"}) - require.NoError(t, err) - getHealthResponseBody := `{ - "jsonrpc": "2.0", - "id": 1, - "error": { - "code": -32601, - "message": "rpc error" - } - }` - mockResponse := &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(strings.NewReader(getHealthResponseBody)), - } - mockHTTPClient.On("Post", rpcURL, "application/json", bytes.NewBuffer(getHealthRequestBody)). - Return(mockResponse, nil) - - // The ctx will timeout after {contextTimeout}, which is enough for the warning to trigger - err = rpcService.TrackRPCServiceHealth(ctx, nil) - require.Error(t, err) - - entries := getLogs() - testSucceeded := false - logMessages := []string{} - for _, entry := range entries { - logMessages = append(logMessages, entry.Message) - if strings.Contains(entry.Message, "RPC service unhealthy for over "+healthCheckWarningInterval.String()) { - testSucceeded = true - break - } - } - assert.Truef(t, testSucceeded, "couldn't find log entry containing %q in %v", "rpc service unhealthy for over "+healthCheckWarningInterval.String(), logMessages) -} - -func TestTrackRPCService_ContextCancelled(t *testing.T) { - // Create and immediately cancel context to test cancellation handling - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - dbt := dbtest.Open(t) - defer dbt.Close() - - dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) - require.NoError(t, err) - defer dbConnectionPool.Close() - mockMetricsService := metrics.NewMockMetricsService() - mockHTTPClient := &utils.MockHTTPClient{} - rpcURL := "http://test-url-track-rpc-service-health" - rpcService, err := NewRPCService(rpcURL, network.TestNetworkPassphrase, mockHTTPClient, mockMetricsService) - require.NoError(t, err) - - // Mock metrics for the initial health check that happens before context check - mockMetricsService.On("IncRPCMethodCalls", "GetHealth").Maybe() - mockMetricsService.On("ObserveRPCMethodDuration", "GetHealth", mock.AnythingOfType("float64")).Maybe() - mockMetricsService.On("IncRPCRequests", "getHealth").Maybe() - mockMetricsService.On("IncRPCEndpointFailure", "getHealth").Maybe() - mockMetricsService.On("IncRPCMethodErrors", "GetHealth", "rpc_error").Maybe() - mockMetricsService.On("ObserveRPCRequestDuration", "getHealth", mock.AnythingOfType("float64")).Maybe() - mockMetricsService.On("SetRPCServiceHealth", false).Maybe() - - // Mock HTTP client to return error (simulating cancelled context) - mockHTTPClient.On("Post", rpcURL, "application/json", mock.Anything). - Return(&http.Response{}, context.Canceled).Maybe() - - err = rpcService.TrackRPCServiceHealth(ctx, nil) - require.Error(t, err) - assert.Contains(t, err.Error(), "context") - - // Verify channel is closed after context cancellation - _, ok := <-rpcService.GetHeartbeatChannel() - assert.False(t, ok, "channel should be closed") -} - -func TestTrackRPCService_DeadlockPrevention(t *testing.T) { - dbt := dbtest.Open(t) - defer dbt.Close() - dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) - require.NoError(t, err) - defer dbConnectionPool.Close() - - // Create fresh RPC service for each test case - mockMetricsService := metrics.NewMockMetricsService() - mockHTTPClient := &utils.MockHTTPClient{} - rpcURL := "http://test-url-deadlock-prevention" - rpcService, err := NewRPCService(rpcURL, network.TestNetworkPassphrase, mockHTTPClient, mockMetricsService) - require.NoError(t, err) - rpcService.healthCheckTickInterval = 10 * time.Millisecond - - // Mock metrics expectations - mockMetricsService.On("IncRPCMethodCalls", "GetHealth").Maybe() - mockMetricsService.On("ObserveRPCMethodDuration", "GetHealth", mock.AnythingOfType("float64")).Maybe() - mockMetricsService.On("IncRPCRequests", "getHealth").Maybe() - mockMetricsService.On("IncRPCEndpointSuccess", "getHealth").Maybe() - mockMetricsService.On("ObserveRPCRequestDuration", "getHealth", mock.AnythingOfType("float64")).Maybe() - mockMetricsService.On("SetRPCServiceHealth", true).Maybe() - mockMetricsService.On("SetRPCLatestLedger", mock.AnythingOfType("int64")).Maybe() - defer mockMetricsService.AssertExpectations(t) - - // Mock successful health response - create fresh response for each call - mockHTTPClient.On("Post", rpcURL, "application/json", mock.Anything).Return(func(url, contentType string, body io.Reader) *http.Response { - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(strings.NewReader(`{ - "jsonrpc": "2.0", - "id": 8675309, - "result": { - "status": "healthy", - "latestLedger": 100, - "oldestLedger": 1, - "ledgerRetentionWindow": 0 - } - }`)), - } - }, nil).Maybe() - - // Start health tracking - this should not deadlock - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - - manualTriggerChan := make(chan any, 1) - go func() { - //nolint:errcheck // Error is expected on context cancellation - rpcService.TrackRPCServiceHealth(ctx, manualTriggerChan) - }() - time.Sleep(20 * time.Millisecond) - manualTriggerChan <- nil - - select { - case <-ctx.Done(): - require.Fail(t, "😵 deadlock occurred!") - case manualTriggerChan <- nil: - t.Log("🎉 Deadlock prevented!") - } -} From 516251b8e14de811cf2b2e751e6b03ef21770057 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 10 Feb 2026 16:46:51 -0500 Subject: [PATCH 26/77] Update helpers.go --- .../infrastructure/helpers.go | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/internal/integrationtests/infrastructure/helpers.go b/internal/integrationtests/infrastructure/helpers.go index fde006c11..275db44f7 100644 --- a/internal/integrationtests/infrastructure/helpers.go +++ b/internal/integrationtests/infrastructure/helpers.go @@ -43,27 +43,27 @@ func WaitForRPCHealthAndRun(ctx context.Context, rpcService services.RPCService, defer cancel() log.Ctx(ctx).Info("⏳ Waiting for RPC service to become healthy...") - rpcHeartbeatChannel := rpcService.GetHeartbeatChannel() signalChan := make(chan os.Signal, 1) signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT) defer signal.Stop(signalChan) - select { - case <-ctx.Done(): - return fmt.Errorf("context canceled while waiting for RPC service to become healthy: %w", ctx.Err()) + for { + select { + case <-ctx.Done(): + return fmt.Errorf("context canceled while waiting for RPC service to become healthy: %w", ctx.Err()) - case sig := <-signalChan: - return fmt.Errorf("received signal %s while waiting for RPC service to become healthy", sig) + case sig := <-signalChan: + return fmt.Errorf("received signal %s while waiting for RPC service to become healthy", sig) - case <-rpcHeartbeatChannel: - log.Ctx(ctx).Info("👍 RPC service is healthy") - if onReady != nil { - if err := onReady(); err != nil { - return fmt.Errorf("executing onReady after RPC became healthy: %w", err) + default: + healthRes, err := rpcService.GetHealth() + if err != nil { + if healthRes.Status == "healthy" { + return nil + } } } - return nil } } From bb35d7577793a17de52dd0ce71c0f9e32e958a30 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 10 Feb 2026 16:50:32 -0500 Subject: [PATCH 27/77] Update helpers.go --- internal/integrationtests/infrastructure/helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/integrationtests/infrastructure/helpers.go b/internal/integrationtests/infrastructure/helpers.go index 275db44f7..283938309 100644 --- a/internal/integrationtests/infrastructure/helpers.go +++ b/internal/integrationtests/infrastructure/helpers.go @@ -58,7 +58,7 @@ func WaitForRPCHealthAndRun(ctx context.Context, rpcService services.RPCService, default: healthRes, err := rpcService.GetHealth() - if err != nil { + if err == nil { if healthRes.Status == "healthy" { return nil } From 71869d70dc4871d76c403d4f7a9a1c30259f7e0c Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 10 Feb 2026 17:23:43 -0500 Subject: [PATCH 28/77] add bloom filter on account_id --- internal/db/migrations/2025-06-10.2-transactions.sql | 3 ++- internal/db/migrations/2025-06-10.3-operations.sql | 3 ++- internal/db/migrations/2025-06-10.4-statechanges.sql | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/db/migrations/2025-06-10.2-transactions.sql b/internal/db/migrations/2025-06-10.2-transactions.sql index 09c16f764..1a2cb1cb4 100644 --- a/internal/db/migrations/2025-06-10.2-transactions.sql +++ b/internal/db/migrations/2025-06-10.2-transactions.sql @@ -33,7 +33,8 @@ CREATE TABLE transactions_accounts ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', tsdb.chunk_interval = '1 day', - tsdb.orderby = 'ledger_created_at DESC' + tsdb.orderby = 'ledger_created_at DESC', + tsdb.sparse_index = 'bloom(account_id)' ); CREATE INDEX idx_transactions_accounts_tx_to_id ON transactions_accounts(tx_to_id); diff --git a/internal/db/migrations/2025-06-10.3-operations.sql b/internal/db/migrations/2025-06-10.3-operations.sql index b4ccd1800..6169e8e0a 100644 --- a/internal/db/migrations/2025-06-10.3-operations.sql +++ b/internal/db/migrations/2025-06-10.3-operations.sql @@ -44,7 +44,8 @@ CREATE TABLE operations_accounts ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', tsdb.chunk_interval = '1 day', - tsdb.orderby = 'ledger_created_at DESC' + tsdb.orderby = 'ledger_created_at DESC', + tsdb.sparse_index = 'bloom(account_id)' ); CREATE INDEX idx_operations_accounts_operation_id ON operations_accounts(operation_id); diff --git a/internal/db/migrations/2025-06-10.4-statechanges.sql b/internal/db/migrations/2025-06-10.4-statechanges.sql index 509564b28..bff53f7e0 100644 --- a/internal/db/migrations/2025-06-10.4-statechanges.sql +++ b/internal/db/migrations/2025-06-10.4-statechanges.sql @@ -48,8 +48,8 @@ CREATE TABLE state_changes ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', tsdb.chunk_interval = '1 day', - tsdb.segmentby = 'state_change_category', - tsdb.orderby = 'ledger_created_at DESC' + tsdb.orderby = 'ledger_created_at DESC', + tsdb.sparse_index = 'bloom(account_id)' ); CREATE INDEX idx_state_changes_account_id ON state_changes(account_id); From 147a67ffc36e0751c0b7d1cc92f26ab0b8654a45 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 11 Feb 2026 10:37:54 -0500 Subject: [PATCH 29/77] remove BatchInsert --- internal/data/operations.go | 137 +-------------- internal/data/operations_test.go | 260 ++--------------------------- internal/data/statechanges.go | 229 +------------------------ internal/data/statechanges_test.go | 243 ++------------------------- internal/data/transactions.go | 155 +---------------- internal/data/transactions_test.go | 229 +------------------------ 6 files changed, 42 insertions(+), 1211 deletions(-) diff --git a/internal/data/operations.go b/internal/data/operations.go index 346782cf4..ad4ddda34 100644 --- a/internal/data/operations.go +++ b/internal/data/operations.go @@ -269,144 +269,13 @@ func (m *OperationModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs [ return operationsWithStateChanges, nil } -// BatchInsert inserts the operations and the operations_accounts links. -// It returns the IDs of the successfully inserted operations. -func (m *OperationModel) BatchInsert( - ctx context.Context, - sqlExecuter db.SQLExecuter, - operations []*types.Operation, - stellarAddressesByOpID map[int64]set.Set[string], -) ([]int64, error) { - if sqlExecuter == nil { - sqlExecuter = m.DB - } - - // 1. Flatten the operations into parallel slices - ids := make([]int64, len(operations)) - operationTypes := make([]string, len(operations)) - operationXDRs := make([][]byte, len(operations)) - resultCodes := make([]string, len(operations)) - successfulFlags := make([]bool, len(operations)) - ledgerNumbers := make([]uint32, len(operations)) - ledgerCreatedAts := make([]time.Time, len(operations)) - - for i, op := range operations { - ids[i] = op.ID - operationTypes[i] = string(op.OperationType) - operationXDRs[i] = []byte(op.OperationXDR) - resultCodes[i] = op.ResultCode - successfulFlags[i] = op.Successful - ledgerNumbers[i] = op.LedgerNumber - ledgerCreatedAts[i] = op.LedgerCreatedAt - } - - // 2. Build OpID -> LedgerCreatedAt lookup from operations - ledgerCreatedAtByOpID := make(map[int64]time.Time, len(operations)) - for _, op := range operations { - ledgerCreatedAtByOpID[op.ID] = op.LedgerCreatedAt - } - - // 3. Flatten the stellarAddressesByOpID into parallel slices, converting to BYTEA - var opIDs []int64 - var stellarAddressBytes [][]byte - var oaLedgerCreatedAts []time.Time - for opID, addresses := range stellarAddressesByOpID { - ledgerCreatedAt := ledgerCreatedAtByOpID[opID] - for address := range addresses.Iter() { - opIDs = append(opIDs, opID) - addrBytes, err := types.AddressBytea(address).Value() - if err != nil { - return nil, fmt.Errorf("converting address %s to bytes: %w", address, err) - } - stellarAddressBytes = append(stellarAddressBytes, addrBytes.([]byte)) - oaLedgerCreatedAts = append(oaLedgerCreatedAts, ledgerCreatedAt) - } - } - - // Insert operations and operations_accounts links. - const insertQuery = ` - WITH - -- Insert operations - inserted_operations AS ( - INSERT INTO operations - (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) - SELECT - o.id, o.operation_type, o.operation_xdr, o.result_code, o.successful, o.ledger_number, o.ledger_created_at - FROM ( - SELECT - UNNEST($1::bigint[]) AS id, - UNNEST($2::text[]) AS operation_type, - UNNEST($3::bytea[]) AS operation_xdr, - UNNEST($4::text[]) AS result_code, - UNNEST($5::boolean[]) AS successful, - UNNEST($6::bigint[]) AS ledger_number, - UNNEST($7::timestamptz[]) AS ledger_created_at - ) o - ON CONFLICT (ledger_created_at, id) DO NOTHING - RETURNING id - ), - - -- Insert operations_accounts links - inserted_operations_accounts AS ( - INSERT INTO operations_accounts - (ledger_created_at, operation_id, account_id) - SELECT - oa.ledger_created_at, oa.op_id, oa.account_id - FROM ( - SELECT - UNNEST($8::timestamptz[]) AS ledger_created_at, - UNNEST($9::bigint[]) AS op_id, - UNNEST($10::bytea[]) AS account_id - ) oa - ON CONFLICT DO NOTHING - ) - - -- Return the IDs of successfully inserted operations - SELECT id FROM inserted_operations; - ` - - start := time.Now() - var insertedIDs []int64 - err := sqlExecuter.SelectContext(ctx, &insertedIDs, insertQuery, - pq.Array(ids), - pq.Array(operationTypes), - pq.Array(operationXDRs), - pq.Array(resultCodes), - pq.Array(successfulFlags), - pq.Array(ledgerNumbers), - pq.Array(ledgerCreatedAts), - pq.Array(oaLedgerCreatedAts), - pq.Array(opIDs), - pq.Array(stellarAddressBytes), - ) - duration := time.Since(start).Seconds() - for _, dbTableName := range []string{"operations", "operations_accounts"} { - m.MetricsService.ObserveDBQueryDuration("BatchInsert", dbTableName, duration) - if dbTableName == "operations" { - m.MetricsService.ObserveDBBatchSize("BatchInsert", dbTableName, len(operations)) - } - if err == nil { - m.MetricsService.IncDBQuery("BatchInsert", dbTableName) - } - } - if err != nil { - for _, dbTableName := range []string{"operations", "operations_accounts"} { - m.MetricsService.IncDBQueryError("BatchInsert", dbTableName, utils.GetDBErrorType(err)) - } - return nil, fmt.Errorf("batch inserting operations and operations_accounts: %w", err) - } - - return insertedIDs, nil -} - // BatchCopy inserts operations using pgx's binary COPY protocol. // Uses pgx.Tx for binary format which is faster than lib/pq's text format. // Uses native pgtype types for optimal performance (see https://github.com/jackc/pgx/issues/763). // -// IMPORTANT: Unlike BatchInsert which uses ON CONFLICT DO NOTHING, BatchCopy will FAIL -// if any duplicate records exist. The PostgreSQL COPY protocol does not support conflict -// handling. Callers must ensure no duplicates exist before calling this method, or handle -// the unique constraint violation error appropriately. +// IMPORTANT: BatchCopy will FAIL if any duplicate records exist. The PostgreSQL COPY +// protocol does not support conflict handling. Callers must ensure no duplicates exist +// before calling this method, or handle the unique constraint violation error appropriately. func (m *OperationModel) BatchCopy( ctx context.Context, pgxTx pgx.Tx, diff --git a/internal/data/operations_test.go b/internal/data/operations_test.go index 2b9126367..6221b40a5 100644 --- a/internal/data/operations_test.go +++ b/internal/data/operations_test.go @@ -43,193 +43,6 @@ func generateTestOperations(n int, startID int64) ([]*types.Operation, map[int64 return ops, addressesByOpID } -func Test_OperationModel_BatchInsert(t *testing.T) { - dbt := dbtest.Open(t) - defer dbt.Close() - dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) - require.NoError(t, err) - defer dbConnectionPool.Close() - - ctx := context.Background() - now := time.Now() - - // Create test data - kp1 := keypair.MustRandom() - kp2 := keypair.MustRandom() - const q = "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)" - _, err = dbConnectionPool.ExecContext(ctx, q, types.AddressBytea(kp1.Address()), types.AddressBytea(kp2.Address())) - require.NoError(t, err) - - // Create referenced transactions first with specific ToIDs - // Operations IDs must be in TOID range for each transaction: (to_id, to_id + 4096) - meta1, meta2 := "meta1", "meta2" - envelope1, envelope2 := "envelope1", "envelope2" - tx1 := types.Transaction{ - Hash: "d176b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", - ToID: 4096, - EnvelopeXDR: &envelope1, - FeeCharged: 100, - ResultCode: "TransactionResultCodeTxSuccess", - MetaXDR: &meta1, - LedgerNumber: 1, - LedgerCreatedAt: now, - IsFeeBump: false, - } - tx2 := types.Transaction{ - Hash: "e176b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", - ToID: 8192, - EnvelopeXDR: &envelope2, - FeeCharged: 200, - ResultCode: "TransactionResultCodeTxSuccess", - MetaXDR: &meta2, - LedgerNumber: 2, - LedgerCreatedAt: now, - IsFeeBump: true, - } - - // Insert transactions - sqlxDB, err := dbConnectionPool.SqlxDB(ctx) - require.NoError(t, err) - txModel := &TransactionModel{DB: dbConnectionPool, MetricsService: metrics.NewMetricsService(sqlxDB)} - _, err = txModel.BatchInsert(ctx, nil, []*types.Transaction{&tx1, &tx2}, map[int64]set.Set[string]{ - tx1.ToID: set.NewSet(kp1.Address()), - tx2.ToID: set.NewSet(kp2.Address()), - }) - require.NoError(t, err) - - // Operations IDs must be in TOID range: (to_id, to_id + 4096) - op1 := types.Operation{ - ID: 4097, // in range (4096, 8192) - OperationType: types.OperationTypePayment, - OperationXDR: types.XDRBytea([]byte("operation1")), - LedgerCreatedAt: now, - } - op2 := types.Operation{ - ID: 8193, // in range (8192, 12288) - OperationType: types.OperationTypeCreateAccount, - OperationXDR: types.XDRBytea([]byte("operation2")), - LedgerCreatedAt: now, - } - - testCases := []struct { - name string - useDBTx bool - operations []*types.Operation - stellarAddressesByOpID map[int64]set.Set[string] - wantAccountLinks map[int64][]string - wantErrContains string - wantIDs []int64 - }{ - { - name: "🟢successful_insert_without_dbTx", - useDBTx: false, - operations: []*types.Operation{&op1, &op2}, - stellarAddressesByOpID: map[int64]set.Set[string]{op1.ID: set.NewSet(kp1.Address(), kp1.Address(), kp1.Address(), kp1.Address()), op2.ID: set.NewSet(kp2.Address(), kp2.Address())}, - wantAccountLinks: map[int64][]string{op1.ID: {kp1.Address()}, op2.ID: {kp2.Address()}}, - wantErrContains: "", - wantIDs: []int64{op1.ID, op2.ID}, - }, - { - name: "🟢successful_insert_with_dbTx", - useDBTx: true, - operations: []*types.Operation{&op1}, - stellarAddressesByOpID: map[int64]set.Set[string]{op1.ID: set.NewSet(kp1.Address())}, - wantAccountLinks: map[int64][]string{op1.ID: {kp1.Address()}}, - wantErrContains: "", - wantIDs: []int64{op1.ID}, - }, - { - name: "🟢empty_input", - useDBTx: false, - operations: []*types.Operation{}, - stellarAddressesByOpID: map[int64]set.Set[string]{}, - wantAccountLinks: map[int64][]string{}, - wantErrContains: "", - wantIDs: nil, - }, - { - name: "🟡duplicate_operation", - useDBTx: false, - operations: []*types.Operation{&op1, &op1}, - stellarAddressesByOpID: map[int64]set.Set[string]{op1.ID: set.NewSet(kp1.Address())}, - wantAccountLinks: map[int64][]string{op1.ID: {kp1.Address()}}, - wantErrContains: "", - wantIDs: []int64{op1.ID}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // Clear the database before each test - _, err = dbConnectionPool.ExecContext(ctx, "TRUNCATE operations, operations_accounts CASCADE") - require.NoError(t, err) - - // Create fresh mock for each test case - mockMetricsService := metrics.NewMockMetricsService() - mockMetricsService. - On("ObserveDBQueryDuration", "BatchInsert", "operations", mock.Anything).Return().Once(). - On("ObserveDBQueryDuration", "BatchInsert", "operations_accounts", mock.Anything).Return().Once(). - On("ObserveDBBatchSize", "BatchInsert", "operations", mock.Anything).Return().Once(). - On("IncDBQuery", "BatchInsert", "operations").Return().Once(). - On("IncDBQuery", "BatchInsert", "operations_accounts").Return().Once() - defer mockMetricsService.AssertExpectations(t) - - m := &OperationModel{ - DB: dbConnectionPool, - MetricsService: mockMetricsService, - } - - var sqlExecuter db.SQLExecuter = dbConnectionPool - if tc.useDBTx { - tx, err := dbConnectionPool.BeginTxx(ctx, nil) - require.NoError(t, err) - defer tx.Rollback() - sqlExecuter = tx - } - - gotInsertedIDs, err := m.BatchInsert(ctx, sqlExecuter, tc.operations, tc.stellarAddressesByOpID) - - if tc.wantErrContains != "" { - require.Error(t, err) - assert.Contains(t, err.Error(), tc.wantErrContains) - return - } - - // Verify the results - require.NoError(t, err) - var dbInsertedIDs []int64 - err = sqlExecuter.SelectContext(ctx, &dbInsertedIDs, "SELECT id FROM operations") - require.NoError(t, err) - assert.ElementsMatch(t, tc.wantIDs, dbInsertedIDs) - assert.ElementsMatch(t, tc.wantIDs, gotInsertedIDs) - - // Verify the account links - if len(tc.wantAccountLinks) > 0 { - var accountLinks []struct { - OperationID int64 `db:"operation_id"` - AccountID types.AddressBytea `db:"account_id"` - } - err = sqlExecuter.SelectContext(ctx, &accountLinks, "SELECT operation_id, account_id FROM operations_accounts ORDER BY operation_id, account_id") - require.NoError(t, err) - - // Create a map of operation_id -> set of account_ids for O(1) lookups - accountLinksMap := make(map[int64][]string) - for _, link := range accountLinks { - accountLinksMap[link.OperationID] = append(accountLinksMap[link.OperationID], string(link.AccountID)) - } - - // Verify each operation has its expected account links - require.Equal(t, len(tc.wantAccountLinks), len(accountLinksMap), "number of elements in the maps don't match") - for key, expectedSlice := range tc.wantAccountLinks { - actualSlice, exists := accountLinksMap[key] - require.True(t, exists, "key %s not found in actual map", key) - assert.ElementsMatch(t, expectedSlice, actualSlice, "slices for key %s don't match", key) - } - } - }) - } -} - func Test_OperationModel_BatchCopy(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -274,14 +87,12 @@ func Test_OperationModel_BatchCopy(t *testing.T) { IsFeeBump: true, } - // Insert transactions - sqlxDB, err := dbConnectionPool.SqlxDB(ctx) - require.NoError(t, err) - txModel := &TransactionModel{DB: dbConnectionPool, MetricsService: metrics.NewMetricsService(sqlxDB)} - _, err = txModel.BatchInsert(ctx, nil, []*types.Transaction{&tx1, &tx2}, map[int64]set.Set[string]{ - tx1.ToID: set.NewSet(kp1.Address()), - tx2.ToID: set.NewSet(kp2.Address()), - }) + // Insert transactions using direct SQL + _, err = dbConnectionPool.ExecContext(ctx, ` + INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9), ($10, $11, $12, $13, $14, $15, $16, $17, $18) + `, tx1.Hash, tx1.ToID, *tx1.EnvelopeXDR, tx1.FeeCharged, tx1.ResultCode, *tx1.MetaXDR, tx1.LedgerNumber, tx1.LedgerCreatedAt, tx1.IsFeeBump, + tx2.Hash, tx2.ToID, *tx2.EnvelopeXDR, tx2.FeeCharged, tx2.ResultCode, *tx2.MetaXDR, tx2.LedgerNumber, tx2.LedgerCreatedAt, tx2.IsFeeBump) require.NoError(t, err) // Operations IDs must be in TOID range: (to_id, to_id + 4096) @@ -437,13 +248,15 @@ func Test_OperationModel_BatchCopy_DuplicateFails(t *testing.T) { LedgerCreatedAt: now, } - // Pre-insert the operation using BatchInsert (which uses ON CONFLICT DO NOTHING) - sqlxDB, err := dbConnectionPool.SqlxDB(ctx) + // Pre-insert the operation using direct SQL + _, err = dbConnectionPool.ExecContext(ctx, ` + INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) + VALUES ($1, $2, $3, '', true, $4, $5) + `, op1.ID, string(op1.OperationType), []byte(op1.OperationXDR), op1.LedgerNumber, op1.LedgerCreatedAt) require.NoError(t, err) - opModel := &OperationModel{DB: dbConnectionPool, MetricsService: metrics.NewMetricsService(sqlxDB)} - _, err = opModel.BatchInsert(ctx, nil, []*types.Operation{&op1}, map[int64]set.Set[string]{ - op1.ID: set.NewSet(kp1.Address()), - }) + _, err = dbConnectionPool.ExecContext(ctx, ` + INSERT INTO operations_accounts (ledger_created_at, operation_id, account_id) VALUES ($1, $2, $3) + `, op1.LedgerCreatedAt, op1.ID, types.AddressBytea(kp1.Address())) require.NoError(t, err) // Verify the operation was inserted @@ -988,51 +801,6 @@ func TestOperationModel_BatchGetByStateChangeIDs(t *testing.T) { assert.Equal(t, int64(4097), stateChangeIDsFound["12288-4097-1"]) } -func BenchmarkOperationModel_BatchInsert(b *testing.B) { - dbt := dbtest.OpenB(b) - defer dbt.Close() - dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) - if err != nil { - b.Fatalf("failed to open db connection pool: %v", err) - } - defer dbConnectionPool.Close() - - ctx := context.Background() - sqlxDB, err := dbConnectionPool.SqlxDB(ctx) - if err != nil { - b.Fatalf("failed to get sqlx db: %v", err) - } - metricsService := metrics.NewMetricsService(sqlxDB) - - m := &OperationModel{ - DB: dbConnectionPool, - MetricsService: metricsService, - } - - batchSizes := []int{1000, 5000, 10000, 50000, 100000} - - for _, size := range batchSizes { - b.Run(fmt.Sprintf("size=%d", size), func(b *testing.B) { - b.ReportAllocs() - - for i := 0; i < b.N; i++ { - b.StopTimer() - // Clean up operations before each iteration - //nolint:errcheck // truncate is best-effort cleanup in benchmarks - dbConnectionPool.ExecContext(ctx, "TRUNCATE operations, operations_accounts CASCADE") - // Generate fresh test data for each iteration - ops, addressesByOpID := generateTestOperations(size, int64(i*size)) - b.StartTimer() - - _, err := m.BatchInsert(ctx, nil, ops, addressesByOpID) - if err != nil { - b.Fatalf("BatchInsert failed: %v", err) - } - } - }) - } -} - // BenchmarkOperationModel_BatchCopy benchmarks bulk insert using pgx's binary COPY protocol. func BenchmarkOperationModel_BatchCopy(b *testing.B) { dbt := dbtest.OpenB(b) diff --git a/internal/data/statechanges.go b/internal/data/statechanges.go index a4a10a4bf..0560e0b44 100644 --- a/internal/data/statechanges.go +++ b/internal/data/statechanges.go @@ -168,236 +168,13 @@ func (m *StateChangeModel) GetAll(ctx context.Context, columns string, limit *in return stateChanges, nil } -func (m *StateChangeModel) BatchInsert( - ctx context.Context, - sqlExecuter db.SQLExecuter, - stateChanges []types.StateChange, -) ([]string, error) { - if sqlExecuter == nil { - sqlExecuter = m.DB - } - - // Flatten the state changes into parallel slices - stateChangeOrders := make([]int64, len(stateChanges)) - toIDs := make([]int64, len(stateChanges)) - categories := make([]string, len(stateChanges)) - reasons := make([]*string, len(stateChanges)) - ledgerCreatedAts := make([]time.Time, len(stateChanges)) - ledgerNumbers := make([]int, len(stateChanges)) - accountIDBytes := make([][]byte, len(stateChanges)) - operationIDs := make([]int64, len(stateChanges)) - tokenIDs := make([]*string, len(stateChanges)) - amounts := make([]*string, len(stateChanges)) - signerAccountIDBytes := make([][]byte, len(stateChanges)) - spenderAccountIDBytes := make([][]byte, len(stateChanges)) - sponsoredAccountIDBytes := make([][]byte, len(stateChanges)) - sponsorAccountIDBytes := make([][]byte, len(stateChanges)) - deployerAccountIDBytes := make([][]byte, len(stateChanges)) - funderAccountIDBytes := make([][]byte, len(stateChanges)) - claimableBalanceIDs := make([]*string, len(stateChanges)) - liquidityPoolIDs := make([]*string, len(stateChanges)) - sponsoredDataValues := make([]*string, len(stateChanges)) - signerWeightOlds := make([]*int16, len(stateChanges)) - signerWeightNews := make([]*int16, len(stateChanges)) - thresholdOlds := make([]*int16, len(stateChanges)) - thresholdNews := make([]*int16, len(stateChanges)) - trustlineLimitOlds := make([]*string, len(stateChanges)) - trustlineLimitNews := make([]*string, len(stateChanges)) - flags := make([]*int16, len(stateChanges)) - keyValues := make([]*types.NullableJSONB, len(stateChanges)) - - for i, sc := range stateChanges { - stateChangeOrders[i] = sc.StateChangeOrder - toIDs[i] = sc.ToID - categories[i] = string(sc.StateChangeCategory) - ledgerCreatedAts[i] = sc.LedgerCreatedAt - ledgerNumbers[i] = int(sc.LedgerNumber) - operationIDs[i] = sc.OperationID - - // Convert account_id to BYTEA (required field) - addrBytes, err := sc.AccountID.Value() - if err != nil { - return nil, fmt.Errorf("converting account_id: %w", err) - } - accountIDBytes[i] = addrBytes.([]byte) - - // Nullable fields - if sc.StateChangeReason != nil { - reason := string(*sc.StateChangeReason) - reasons[i] = &reason - } - if sc.TokenID.Valid { - tokenIDs[i] = &sc.TokenID.String - } - if sc.Amount.Valid { - amounts[i] = &sc.Amount.String - } - - // Convert nullable account_id fields to BYTEA - signerAccountIDBytes[i], err = pgtypeBytesFromNullAddressBytea(sc.SignerAccountID) - if err != nil { - return nil, fmt.Errorf("converting signer_account_id: %w", err) - } - spenderAccountIDBytes[i], err = pgtypeBytesFromNullAddressBytea(sc.SpenderAccountID) - if err != nil { - return nil, fmt.Errorf("converting spender_account_id: %w", err) - } - sponsoredAccountIDBytes[i], err = pgtypeBytesFromNullAddressBytea(sc.SponsoredAccountID) - if err != nil { - return nil, fmt.Errorf("converting sponsored_account_id: %w", err) - } - sponsorAccountIDBytes[i], err = pgtypeBytesFromNullAddressBytea(sc.SponsorAccountID) - if err != nil { - return nil, fmt.Errorf("converting sponsor_account_id: %w", err) - } - deployerAccountIDBytes[i], err = pgtypeBytesFromNullAddressBytea(sc.DeployerAccountID) - if err != nil { - return nil, fmt.Errorf("converting deployer_account_id: %w", err) - } - funderAccountIDBytes[i], err = pgtypeBytesFromNullAddressBytea(sc.FunderAccountID) - if err != nil { - return nil, fmt.Errorf("converting funder_account_id: %w", err) - } - if sc.ClaimableBalanceID.Valid { - claimableBalanceIDs[i] = &sc.ClaimableBalanceID.String - } - if sc.LiquidityPoolID.Valid { - liquidityPoolIDs[i] = &sc.LiquidityPoolID.String - } - if sc.SponsoredData.Valid { - sponsoredDataValues[i] = &sc.SponsoredData.String - } - if sc.SignerWeightOld.Valid { - signerWeightOlds[i] = &sc.SignerWeightOld.Int16 - } - if sc.SignerWeightNew.Valid { - signerWeightNews[i] = &sc.SignerWeightNew.Int16 - } - if sc.ThresholdOld.Valid { - thresholdOlds[i] = &sc.ThresholdOld.Int16 - } - if sc.ThresholdNew.Valid { - thresholdNews[i] = &sc.ThresholdNew.Int16 - } - if sc.TrustlineLimitOld.Valid { - trustlineLimitOlds[i] = &sc.TrustlineLimitOld.String - } - if sc.TrustlineLimitNew.Valid { - trustlineLimitNews[i] = &sc.TrustlineLimitNew.String - } - if sc.Flags.Valid { - flags[i] = &sc.Flags.Int16 - } - if sc.KeyValue != nil { - keyValues[i] = &sc.KeyValue - } - } - - const insertQuery = ` - -- Insert state changes - WITH input_data AS ( - SELECT - UNNEST($1::bigint[]) AS state_change_order, - UNNEST($2::bigint[]) AS to_id, - UNNEST($3::text[]) AS state_change_category, - UNNEST($4::text[]) AS state_change_reason, - UNNEST($5::timestamptz[]) AS ledger_created_at, - UNNEST($6::integer[]) AS ledger_number, - UNNEST($7::bytea[]) AS account_id, - UNNEST($8::bigint[]) AS operation_id, - UNNEST($9::text[]) AS token_id, - UNNEST($10::text[]) AS amount, - UNNEST($11::bytea[]) AS signer_account_id, - UNNEST($12::bytea[]) AS spender_account_id, - UNNEST($13::bytea[]) AS sponsored_account_id, - UNNEST($14::bytea[]) AS sponsor_account_id, - UNNEST($15::bytea[]) AS deployer_account_id, - UNNEST($16::bytea[]) AS funder_account_id, - UNNEST($17::text[]) AS claimable_balance_id, - UNNEST($18::text[]) AS liquidity_pool_id, - UNNEST($19::text[]) AS sponsored_data, - UNNEST($20::smallint[]) AS signer_weight_old, - UNNEST($21::smallint[]) AS signer_weight_new, - UNNEST($22::smallint[]) AS threshold_old, - UNNEST($23::smallint[]) AS threshold_new, - UNNEST($24::text[]) AS trustline_limit_old, - UNNEST($25::text[]) AS trustline_limit_new, - UNNEST($26::smallint[]) AS flags, - UNNEST($27::jsonb[]) AS key_value - ), - inserted_state_changes AS ( - INSERT INTO state_changes - (state_change_order, to_id, state_change_category, state_change_reason, ledger_created_at, - ledger_number, account_id, operation_id, token_id, amount, - signer_account_id, spender_account_id, sponsored_account_id, sponsor_account_id, - deployer_account_id, funder_account_id, claimable_balance_id, liquidity_pool_id, sponsored_data, - signer_weight_old, signer_weight_new, threshold_old, threshold_new, - trustline_limit_old, trustline_limit_new, flags, key_value) - SELECT - sc.state_change_order, sc.to_id, sc.state_change_category, sc.state_change_reason, sc.ledger_created_at, - sc.ledger_number, sc.account_id, sc.operation_id, sc.token_id, sc.amount, - sc.signer_account_id, sc.spender_account_id, sc.sponsored_account_id, sc.sponsor_account_id, - sc.deployer_account_id, sc.funder_account_id, sc.claimable_balance_id, sc.liquidity_pool_id, sc.sponsored_data, - sc.signer_weight_old, sc.signer_weight_new, sc.threshold_old, sc.threshold_new, - sc.trustline_limit_old, sc.trustline_limit_new, sc.flags, sc.key_value - FROM input_data sc - ON CONFLICT (ledger_created_at, to_id, operation_id, state_change_order) DO NOTHING - RETURNING to_id, operation_id, state_change_order - ) - SELECT CONCAT(to_id, '-', operation_id, '-', state_change_order) FROM inserted_state_changes; - ` - - start := time.Now() - var insertedIDs []string - err := sqlExecuter.SelectContext(ctx, &insertedIDs, insertQuery, - pq.Array(stateChangeOrders), - pq.Array(toIDs), - pq.Array(categories), - pq.Array(reasons), - pq.Array(ledgerCreatedAts), - pq.Array(ledgerNumbers), - pq.Array(accountIDBytes), - pq.Array(operationIDs), - pq.Array(tokenIDs), - pq.Array(amounts), - pq.Array(signerAccountIDBytes), - pq.Array(spenderAccountIDBytes), - pq.Array(sponsoredAccountIDBytes), - pq.Array(sponsorAccountIDBytes), - pq.Array(deployerAccountIDBytes), - pq.Array(funderAccountIDBytes), - pq.Array(claimableBalanceIDs), - pq.Array(liquidityPoolIDs), - pq.Array(sponsoredDataValues), - pq.Array(signerWeightOlds), - pq.Array(signerWeightNews), - pq.Array(thresholdOlds), - pq.Array(thresholdNews), - pq.Array(trustlineLimitOlds), - pq.Array(trustlineLimitNews), - pq.Array(flags), - pq.Array(keyValues), - ) - duration := time.Since(start).Seconds() - m.MetricsService.ObserveDBQueryDuration("BatchInsert", "state_changes", duration) - m.MetricsService.ObserveDBBatchSize("BatchInsert", "state_changes", len(stateChanges)) - if err != nil { - m.MetricsService.IncDBQueryError("BatchInsert", "state_changes", utils.GetDBErrorType(err)) - return nil, fmt.Errorf("batch inserting state changes: %w", err) - } - m.MetricsService.IncDBQuery("BatchInsert", "state_changes") - - return insertedIDs, nil -} - // BatchCopy inserts state changes using pgx's binary COPY protocol. // Uses pgx.Tx for binary format which is faster than lib/pq's text format. // Uses native pgtype types for optimal performance (see https://github.com/jackc/pgx/issues/763). // -// IMPORTANT: Unlike BatchInsert which uses ON CONFLICT DO NOTHING, BatchCopy will FAIL -// if any duplicate records exist. The PostgreSQL COPY protocol does not support conflict -// handling. Callers must ensure no duplicates exist before calling this method, or handle -// the unique constraint violation error appropriately. +// IMPORTANT: BatchCopy will FAIL if any duplicate records exist. The PostgreSQL COPY +// protocol does not support conflict handling. Callers must ensure no duplicates exist +// before calling this method, or handle the unique constraint violation error appropriately. func (m *StateChangeModel) BatchCopy( ctx context.Context, pgxTx pgx.Tx, diff --git a/internal/data/statechanges_test.go b/internal/data/statechanges_test.go index 8a44b6985..71dd5431b 100644 --- a/internal/data/statechanges_test.go +++ b/internal/data/statechanges_test.go @@ -7,7 +7,6 @@ import ( "testing" "time" - set "github.com/deckarep/golang-set/v2" "github.com/jackc/pgx/v5" "github.com/stellar/go-stellar-sdk/keypair" "github.com/stretchr/testify/assert" @@ -65,162 +64,6 @@ func generateTestStateChanges(n int, accountID string, startToID int64, auxAddre return scs } -func TestStateChangeModel_BatchInsert(t *testing.T) { - dbt := dbtest.Open(t) - defer dbt.Close() - dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) - require.NoError(t, err) - defer dbConnectionPool.Close() - - ctx := context.Background() - now := time.Now() - - // Create test data - kp1 := keypair.MustRandom() - kp2 := keypair.MustRandom() - const q = "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)" - _, err = dbConnectionPool.ExecContext(ctx, q, types.AddressBytea(kp1.Address()), types.AddressBytea(kp2.Address())) - require.NoError(t, err) - - // Create referenced transactions first - meta1, meta2 := "meta1", "meta2" - envelope1, envelope2 := "envelope1", "envelope2" - tx1 := types.Transaction{ - Hash: "f176b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", - ToID: 1, - EnvelopeXDR: &envelope1, - FeeCharged: 100, - ResultCode: "TransactionResultCodeTxSuccess", - MetaXDR: &meta1, - LedgerNumber: 1, - LedgerCreatedAt: now, - IsFeeBump: false, - } - tx2 := types.Transaction{ - Hash: "0276b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", - ToID: 2, - EnvelopeXDR: &envelope2, - FeeCharged: 200, - ResultCode: "TransactionResultCodeTxSuccess", - MetaXDR: &meta2, - LedgerNumber: 2, - LedgerCreatedAt: now, - IsFeeBump: true, - } - sqlxDB, err := dbConnectionPool.SqlxDB(ctx) - require.NoError(t, err) - txModel := &TransactionModel{DB: dbConnectionPool, MetricsService: metrics.NewMetricsService(sqlxDB)} - _, err = txModel.BatchInsert(ctx, nil, []*types.Transaction{&tx1, &tx2}, map[int64]set.Set[string]{ - tx1.ToID: set.NewSet(kp1.Address()), - tx2.ToID: set.NewSet(kp2.Address()), - }) - require.NoError(t, err) - - reason := types.StateChangeReasonAdd - sc1 := types.StateChange{ - ToID: 1, - StateChangeOrder: 1, - StateChangeCategory: types.StateChangeCategoryBalance, - StateChangeReason: &reason, - LedgerCreatedAt: now, - LedgerNumber: 1, - AccountID: types.AddressBytea(kp1.Address()), - OperationID: 123, - TokenID: sql.NullString{String: "token1", Valid: true}, - Amount: sql.NullString{String: "100", Valid: true}, - } - sc2 := types.StateChange{ - ToID: 2, - StateChangeOrder: 1, - StateChangeCategory: types.StateChangeCategoryBalance, - StateChangeReason: &reason, - LedgerCreatedAt: now, - LedgerNumber: 2, - AccountID: types.AddressBytea(kp2.Address()), - OperationID: 456, - } - - testCases := []struct { - name string - useDBTx bool - stateChanges []types.StateChange - wantIDs []string - wantErrContains string - }{ - { - name: "🟢successful_insert_without_dbTx", - useDBTx: false, - stateChanges: []types.StateChange{sc1, sc2}, - wantIDs: []string{fmt.Sprintf("%d-%d-%d", sc1.ToID, sc1.OperationID, sc1.StateChangeOrder), fmt.Sprintf("%d-%d-%d", sc2.ToID, sc2.OperationID, sc2.StateChangeOrder)}, - }, - { - name: "🟢successful_insert_with_dbTx", - useDBTx: true, - stateChanges: []types.StateChange{sc1}, - wantIDs: []string{fmt.Sprintf("%d-%d-%d", sc1.ToID, sc1.OperationID, sc1.StateChangeOrder)}, - }, - { - name: "🟢empty_input", - useDBTx: false, - stateChanges: []types.StateChange{}, - wantIDs: nil, - }, - { - name: "🟡duplicate_state_change", - useDBTx: false, - stateChanges: []types.StateChange{sc1, sc1}, - wantIDs: []string{fmt.Sprintf("%d-%d-%d", sc1.ToID, sc1.OperationID, sc1.StateChangeOrder)}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - _, err = dbConnectionPool.ExecContext(ctx, "TRUNCATE state_changes CASCADE") - require.NoError(t, err) - - mockMetricsService := metrics.NewMockMetricsService() - mockMetricsService. - On("ObserveDBQueryDuration", "BatchInsert", "state_changes", mock.Anything).Return().Once() - mockMetricsService. - On("ObserveDBBatchSize", "BatchInsert", "state_changes", mock.Anything).Return().Once() - mockMetricsService. - On("IncDBQuery", "BatchInsert", "state_changes").Return().Once() - - m := &StateChangeModel{ - DB: dbConnectionPool, - MetricsService: mockMetricsService, - } - - var sqlExecuter db.SQLExecuter = dbConnectionPool - if tc.useDBTx { - tx, err := dbConnectionPool.BeginTxx(ctx, nil) - require.NoError(t, err) - defer tx.Rollback() // nolint: errcheck - sqlExecuter = tx - } - - gotInsertedIDs, err := m.BatchInsert(ctx, sqlExecuter, tc.stateChanges) - - if tc.wantErrContains != "" { - require.Error(t, err) - assert.Contains(t, err.Error(), tc.wantErrContains) - return - } - - require.NoError(t, err) - assert.ElementsMatch(t, tc.wantIDs, gotInsertedIDs) - - // Verify from DB - var dbInsertedIDs []string - err = sqlExecuter.SelectContext(ctx, &dbInsertedIDs, "SELECT CONCAT(to_id, '-', operation_id, '-', state_change_order) FROM state_changes") - require.NoError(t, err) - assert.ElementsMatch(t, tc.wantIDs, dbInsertedIDs) - - mockMetricsService.AssertExpectations(t) - }) - } -} - func TestStateChangeModel_BatchCopy(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -263,13 +106,12 @@ func TestStateChangeModel_BatchCopy(t *testing.T) { LedgerCreatedAt: now, IsFeeBump: true, } - sqlxDB, err := dbConnectionPool.SqlxDB(ctx) - require.NoError(t, err) - txModel := &TransactionModel{DB: dbConnectionPool, MetricsService: metrics.NewMetricsService(sqlxDB)} - _, err = txModel.BatchInsert(ctx, nil, []*types.Transaction{&tx1, &tx2}, map[int64]set.Set[string]{ - tx1.ToID: set.NewSet(kp1.Address()), - tx2.ToID: set.NewSet(kp2.Address()), - }) + // Insert transactions using direct SQL + _, err = dbConnectionPool.ExecContext(ctx, ` + INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9), ($10, $11, $12, $13, $14, $15, $16, $17, $18) + `, tx1.Hash, tx1.ToID, *tx1.EnvelopeXDR, tx1.FeeCharged, tx1.ResultCode, *tx1.MetaXDR, tx1.LedgerNumber, tx1.LedgerCreatedAt, tx1.IsFeeBump, + tx2.Hash, tx2.ToID, *tx2.EnvelopeXDR, tx2.FeeCharged, tx2.ResultCode, *tx2.MetaXDR, tx2.LedgerNumber, tx2.LedgerCreatedAt, tx2.IsFeeBump) require.NoError(t, err) reason := types.StateChangeReasonAdd @@ -429,11 +271,11 @@ func TestStateChangeModel_BatchCopy_DuplicateFails(t *testing.T) { OperationID: 123, } - // Pre-insert the state change using BatchInsert (which uses ON CONFLICT DO NOTHING) - sqlxDB, err := dbConnectionPool.SqlxDB(ctx) - require.NoError(t, err) - scModel := &StateChangeModel{DB: dbConnectionPool, MetricsService: metrics.NewMetricsService(sqlxDB)} - _, err = scModel.BatchInsert(ctx, nil, []types.StateChange{sc1}) + // Pre-insert the state change using direct SQL + _, err = dbConnectionPool.ExecContext(ctx, ` + INSERT INTO state_changes (to_id, state_change_order, state_change_category, state_change_reason, ledger_created_at, ledger_number, account_id, operation_id) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + `, sc1.ToID, sc1.StateChangeOrder, string(sc1.StateChangeCategory), string(*sc1.StateChangeReason), sc1.LedgerCreatedAt, sc1.LedgerNumber, sc1.AccountID, sc1.OperationID) require.NoError(t, err) // Verify the state change was inserted @@ -1151,69 +993,6 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { }) } -func BenchmarkStateChangeModel_BatchInsert(b *testing.B) { - dbt := dbtest.OpenB(b) - defer dbt.Close() - dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) - if err != nil { - b.Fatalf("failed to open db connection pool: %v", err) - } - defer dbConnectionPool.Close() - - ctx := context.Background() - sqlxDB, err := dbConnectionPool.SqlxDB(ctx) - if err != nil { - b.Fatalf("failed to get sqlx db: %v", err) - } - metricsService := metrics.NewMetricsService(sqlxDB) - - m := &StateChangeModel{ - DB: dbConnectionPool, - MetricsService: metricsService, - } - - // Create a parent transaction that state changes will reference - const txHash = "benchmark_tx_hash" - accountID := keypair.MustRandom().Address() - now := time.Now() - _, err = dbConnectionPool.ExecContext(ctx, ` - INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) - VALUES ($1, 1, 'env', 100, 'TransactionResultCodeTxSuccess', 'meta', 1, $2, false) - `, txHash, now) - if err != nil { - b.Fatalf("failed to create parent transaction: %v", err) - } - - // Pre-generate auxiliary addresses for nullable account_id fields - auxAddresses := make([]string, 10) - for i := range auxAddresses { - auxAddresses[i] = keypair.MustRandom().Address() - } - - batchSizes := []int{1000, 5000, 10000, 50000, 100000} - - for _, size := range batchSizes { - b.Run(fmt.Sprintf("size=%d", size), func(b *testing.B) { - b.ReportAllocs() - - for i := 0; i < b.N; i++ { - b.StopTimer() - // Clean up state changes before each iteration (keep the parent transaction) - //nolint:errcheck // truncate is best-effort cleanup in benchmarks - dbConnectionPool.ExecContext(ctx, "TRUNCATE state_changes CASCADE") - // Generate fresh test data for each iteration - scs := generateTestStateChanges(size, accountID, int64(i*size), auxAddresses) - b.StartTimer() - - _, err := m.BatchInsert(ctx, nil, scs) - if err != nil { - b.Fatalf("BatchInsert failed: %v", err) - } - } - }) - } -} - // BenchmarkStateChangeModel_BatchCopy benchmarks bulk insert using pgx's binary COPY protocol. func BenchmarkStateChangeModel_BatchCopy(b *testing.B) { dbt := dbtest.OpenB(b) diff --git a/internal/data/transactions.go b/internal/data/transactions.go index 4a4f5e901..5209ffcfc 100644 --- a/internal/data/transactions.go +++ b/internal/data/transactions.go @@ -178,162 +178,13 @@ func (m *TransactionModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs return transactions, nil } -// BatchInsert inserts the transactions and the transactions_accounts links. -// It returns the hashes of the successfully inserted transactions. -func (m *TransactionModel) BatchInsert( - ctx context.Context, - sqlExecuter db.SQLExecuter, - txs []*types.Transaction, - stellarAddressesByToID map[int64]set.Set[string], -) ([]string, error) { - if sqlExecuter == nil { - sqlExecuter = m.DB - } - - // 1. Flatten the transactions into parallel slices - hashes := make([][]byte, len(txs)) - toIDs := make([]int64, len(txs)) - envelopeXDRs := make([]*string, len(txs)) - feesCharged := make([]int64, len(txs)) - resultCodes := make([]string, len(txs)) - metaXDRs := make([]*string, len(txs)) - ledgerNumbers := make([]int, len(txs)) - ledgerCreatedAts := make([]time.Time, len(txs)) - isFeeBumps := make([]bool, len(txs)) - - for i, t := range txs { - hashBytes, err := t.Hash.Value() - if err != nil { - return nil, fmt.Errorf("converting hash %s to bytes: %w", t.Hash, err) - } - hashes[i] = hashBytes.([]byte) - toIDs[i] = t.ToID - envelopeXDRs[i] = t.EnvelopeXDR - feesCharged[i] = t.FeeCharged - resultCodes[i] = t.ResultCode - metaXDRs[i] = t.MetaXDR - ledgerNumbers[i] = int(t.LedgerNumber) - ledgerCreatedAts[i] = t.LedgerCreatedAt - isFeeBumps[i] = t.IsFeeBump - } - - // 2. Build ToID -> LedgerCreatedAt lookup from transactions - ledgerCreatedAtByToID := make(map[int64]time.Time, len(txs)) - for _, tx := range txs { - ledgerCreatedAtByToID[tx.ToID] = tx.LedgerCreatedAt - } - - // 3. Flatten the stellarAddressesByToID into parallel slices, converting to BYTEA - var txToIDs []int64 - var stellarAddressBytes [][]byte - var taLedgerCreatedAts []time.Time - for toID, addresses := range stellarAddressesByToID { - ledgerCreatedAt := ledgerCreatedAtByToID[toID] - for address := range addresses.Iter() { - txToIDs = append(txToIDs, toID) - addrBytes, err := types.AddressBytea(address).Value() - if err != nil { - return nil, fmt.Errorf("converting address %s to bytes: %w", address, err) - } - stellarAddressBytes = append(stellarAddressBytes, addrBytes.([]byte)) - taLedgerCreatedAts = append(taLedgerCreatedAts, ledgerCreatedAt) - } - } - - // Insert transactions and transactions_accounts links with minimal account validation. - const insertQuery = ` - WITH - -- Insert transactions - inserted_transactions AS ( - INSERT INTO transactions - (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) - SELECT - t.hash, t.to_id, t.envelope_xdr, t.fee_charged, t.result_code, t.meta_xdr, t.ledger_number, t.ledger_created_at, t.is_fee_bump - FROM ( - SELECT - UNNEST($1::bytea[]) AS hash, - UNNEST($2::bigint[]) AS to_id, - UNNEST($3::text[]) AS envelope_xdr, - UNNEST($4::bigint[]) AS fee_charged, - UNNEST($5::text[]) AS result_code, - UNNEST($6::text[]) AS meta_xdr, - UNNEST($7::bigint[]) AS ledger_number, - UNNEST($8::timestamptz[]) AS ledger_created_at, - UNNEST($9::boolean[]) AS is_fee_bump - ) t - ON CONFLICT (ledger_created_at, to_id) DO NOTHING - RETURNING hash - ), - - -- Insert transactions_accounts links - inserted_transactions_accounts AS ( - INSERT INTO transactions_accounts - (ledger_created_at, tx_to_id, account_id) - SELECT - ta.ledger_created_at, ta.tx_to_id, ta.account_id - FROM ( - SELECT - UNNEST($10::timestamptz[]) AS ledger_created_at, - UNNEST($11::bigint[]) AS tx_to_id, - UNNEST($12::bytea[]) AS account_id - ) ta - ON CONFLICT DO NOTHING - ) - - -- Return the hashes of successfully inserted transactions - SELECT hash FROM inserted_transactions; - ` - - start := time.Now() - var insertedHashes []types.HashBytea - err := sqlExecuter.SelectContext(ctx, &insertedHashes, insertQuery, - pq.Array(hashes), - pq.Array(toIDs), - pq.Array(envelopeXDRs), - pq.Array(feesCharged), - pq.Array(resultCodes), - pq.Array(metaXDRs), - pq.Array(ledgerNumbers), - pq.Array(ledgerCreatedAts), - pq.Array(isFeeBumps), - pq.Array(taLedgerCreatedAts), - pq.Array(txToIDs), - pq.Array(stellarAddressBytes), - ) - duration := time.Since(start).Seconds() - for _, dbTableName := range []string{"transactions", "transactions_accounts"} { - m.MetricsService.ObserveDBQueryDuration("BatchInsert", dbTableName, duration) - if dbTableName == "transactions" { - m.MetricsService.ObserveDBBatchSize("BatchInsert", dbTableName, len(txs)) - } - if err == nil { - m.MetricsService.IncDBQuery("BatchInsert", dbTableName) - } - } - if err != nil { - for _, dbTableName := range []string{"transactions", "transactions_accounts"} { - m.MetricsService.IncDBQueryError("BatchInsert", dbTableName, utils.GetDBErrorType(err)) - } - return nil, fmt.Errorf("batch inserting transactions and transactions_accounts: %w", err) - } - - // Convert HashBytea to string for the return value - result := make([]string, len(insertedHashes)) - for i, h := range insertedHashes { - result[i] = h.String() - } - - return result, nil -} - // BatchCopy inserts transactions using pgx's binary COPY protocol. // Uses pgx.Tx for binary format which is faster than lib/pq's text format. // Uses native pgtype types for optimal performance (see https://github.com/jackc/pgx/issues/763). // -// IMPORTANT: Unlike BatchInsert which uses ON CONFLICT DO NOTHING, BatchCopy will FAIL -// if any duplicate records exist. The PostgreSQL COPY protocol does not support conflict -// handling. Callers must ensure no duplicates exist before calling this method, or handle -// the unique constraint violation error appropriately. +// IMPORTANT: BatchCopy will FAIL if any duplicate records exist. The PostgreSQL COPY +// protocol does not support conflict handling. Callers must ensure no duplicates exist +// before calling this method, or handle the unique constraint violation error appropriately. func (m *TransactionModel) BatchCopy( ctx context.Context, pgxTx pgx.Tx, diff --git a/internal/data/transactions_test.go b/internal/data/transactions_test.go index aba928b98..b15e3ea41 100644 --- a/internal/data/transactions_test.go +++ b/internal/data/transactions_test.go @@ -53,176 +53,6 @@ func generateTestTransactions(n int, startLedger int32) ([]*types.Transaction, m return txs, addressesByToID } -func Test_TransactionModel_BatchInsert(t *testing.T) { - dbt := dbtest.Open(t) - defer dbt.Close() - dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) - require.NoError(t, err) - defer dbConnectionPool.Close() - - ctx := context.Background() - now := time.Now() - - // Create test data - kp1 := keypair.MustRandom() - kp2 := keypair.MustRandom() - const q = "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)" - _, err = dbConnectionPool.ExecContext(ctx, q, types.AddressBytea(kp1.Address()), types.AddressBytea(kp2.Address())) - require.NoError(t, err) - - meta1, meta2 := "meta1", "meta2" - envelope1, envelope2 := "envelope1", "envelope2" - tx1 := types.Transaction{ - Hash: "e76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48760", - ToID: 1, - EnvelopeXDR: &envelope1, - FeeCharged: 100, - ResultCode: "TransactionResultCodeTxSuccess", - MetaXDR: &meta1, - LedgerNumber: 1, - LedgerCreatedAt: now, - IsFeeBump: false, - } - tx2 := types.Transaction{ - Hash: "a76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48761", - ToID: 2, - EnvelopeXDR: &envelope2, - FeeCharged: 200, - ResultCode: "TransactionResultCodeTxSuccess", - MetaXDR: &meta2, - LedgerNumber: 2, - LedgerCreatedAt: now, - IsFeeBump: true, - } - - testCases := []struct { - name string - useDBTx bool - txs []*types.Transaction - stellarAddressesByToID map[int64]set.Set[string] - wantAccountLinks map[int64][]string - wantErrContains string - wantHashes []string - }{ - { - name: "🟢successful_insert_without_dbTx", - useDBTx: false, - txs: []*types.Transaction{&tx1, &tx2}, - stellarAddressesByToID: map[int64]set.Set[string]{tx1.ToID: set.NewSet(kp1.Address()), tx2.ToID: set.NewSet(kp2.Address())}, - wantAccountLinks: map[int64][]string{tx1.ToID: {kp1.Address()}, tx2.ToID: {kp2.Address()}}, - wantErrContains: "", - wantHashes: []string{tx1.Hash.String(), tx2.Hash.String()}, - }, - { - name: "🟢successful_insert_with_dbTx", - useDBTx: true, - txs: []*types.Transaction{&tx1}, - stellarAddressesByToID: map[int64]set.Set[string]{tx1.ToID: set.NewSet(kp1.Address())}, - wantAccountLinks: map[int64][]string{tx1.ToID: {kp1.Address()}}, - wantErrContains: "", - wantHashes: []string{tx1.Hash.String()}, - }, - { - name: "🟢empty_input", - useDBTx: false, - txs: []*types.Transaction{}, - stellarAddressesByToID: map[int64]set.Set[string]{}, - wantAccountLinks: map[int64][]string{}, - wantErrContains: "", - wantHashes: nil, - }, - { - name: "🟡duplicate_transaction", - useDBTx: false, - txs: []*types.Transaction{&tx1, &tx1}, - stellarAddressesByToID: map[int64]set.Set[string]{tx1.ToID: set.NewSet(kp1.Address())}, - wantAccountLinks: map[int64][]string{tx1.ToID: {kp1.Address()}}, - wantErrContains: "", - wantHashes: []string{tx1.Hash.String()}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // Clear the database before each test - _, err = dbConnectionPool.ExecContext(ctx, "TRUNCATE transactions, transactions_accounts CASCADE") - require.NoError(t, err) - - // Create fresh mock for each test case - mockMetricsService := metrics.NewMockMetricsService() - // The implementation always loops through both tables and calls ObserveDBQueryDuration for each - mockMetricsService. - On("ObserveDBQueryDuration", "BatchInsert", "transactions", mock.Anything).Return().Once(). - On("ObserveDBQueryDuration", "BatchInsert", "transactions_accounts", mock.Anything).Return().Once() - // ObserveDBBatchSize is only called for transactions table (not transactions_accounts) - mockMetricsService.On("ObserveDBBatchSize", "BatchInsert", "transactions", mock.Anything).Return().Once() - // IncDBQuery is called for both tables on success - mockMetricsService. - On("IncDBQuery", "BatchInsert", "transactions").Return().Once(). - On("IncDBQuery", "BatchInsert", "transactions_accounts").Return().Once() - defer mockMetricsService.AssertExpectations(t) - - m := &TransactionModel{ - DB: dbConnectionPool, - MetricsService: mockMetricsService, - } - - var sqlExecuter db.SQLExecuter = dbConnectionPool - if tc.useDBTx { - tx, err := dbConnectionPool.BeginTxx(ctx, nil) - require.NoError(t, err) - defer tx.Rollback() - sqlExecuter = tx - } - - gotInsertedHashes, err := m.BatchInsert(ctx, sqlExecuter, tc.txs, tc.stellarAddressesByToID) - - if tc.wantErrContains != "" { - require.Error(t, err) - assert.Contains(t, err.Error(), tc.wantErrContains) - return - } - - // Verify the results - require.NoError(t, err) - var dbInsertedHashes []types.HashBytea - err = sqlExecuter.SelectContext(ctx, &dbInsertedHashes, "SELECT hash FROM transactions") - require.NoError(t, err) - // Convert HashBytea to string for comparison - dbHashStrings := make([]string, len(dbInsertedHashes)) - for i, h := range dbInsertedHashes { - dbHashStrings[i] = h.String() - } - assert.ElementsMatch(t, tc.wantHashes, dbHashStrings) - assert.ElementsMatch(t, tc.wantHashes, gotInsertedHashes) - - // Verify the account links - if len(tc.wantAccountLinks) > 0 { - var accountLinks []struct { - TxToID int64 `db:"tx_to_id"` - AccountID types.AddressBytea `db:"account_id"` - } - err = sqlExecuter.SelectContext(ctx, &accountLinks, "SELECT tx_to_id, account_id FROM transactions_accounts ORDER BY tx_to_id, account_id") - require.NoError(t, err) - - // Create a map of tx_to_id -> set of account_ids for O(1) lookups - accountLinksMap := make(map[int64][]string) - for _, link := range accountLinks { - accountLinksMap[link.TxToID] = append(accountLinksMap[link.TxToID], string(link.AccountID)) - } - - // Verify each transaction has its expected account links - require.Equal(t, len(tc.wantAccountLinks), len(accountLinksMap), "number of elements in the maps don't match") - for key, expectedSlice := range tc.wantAccountLinks { - actualSlice, exists := accountLinksMap[key] - require.True(t, exists, "key %d not found in actual map", key) - assert.ElementsMatch(t, expectedSlice, actualSlice, "slices for key %d don't match", key) - } - } - }) - } -} - func Test_TransactionModel_BatchCopy(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -421,13 +251,15 @@ func Test_TransactionModel_BatchCopy_DuplicateFails(t *testing.T) { IsFeeBump: false, } - // Pre-insert the transaction using BatchInsert (which uses ON CONFLICT DO NOTHING) - sqlxDB, err := dbConnectionPool.SqlxDB(ctx) + // Pre-insert the transaction using direct SQL + _, err = dbConnectionPool.ExecContext(ctx, ` + INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + `, txDup.Hash, txDup.ToID, *txDup.EnvelopeXDR, txDup.FeeCharged, txDup.ResultCode, *txDup.MetaXDR, txDup.LedgerNumber, txDup.LedgerCreatedAt, txDup.IsFeeBump) require.NoError(t, err) - txModel := &TransactionModel{DB: dbConnectionPool, MetricsService: metrics.NewMetricsService(sqlxDB)} - _, err = txModel.BatchInsert(ctx, nil, []*types.Transaction{&txDup}, map[int64]set.Set[string]{ - txDup.ToID: set.NewSet(kp1.Address()), - }) + _, err = dbConnectionPool.ExecContext(ctx, ` + INSERT INTO transactions_accounts (ledger_created_at, tx_to_id, account_id) VALUES ($1, $2, $3) + `, txDup.LedgerCreatedAt, txDup.ToID, types.AddressBytea(kp1.Address())) require.NoError(t, err) // Verify the transaction was inserted @@ -742,51 +574,6 @@ func TestTransactionModel_BatchGetByStateChangeIDs(t *testing.T) { assert.Equal(t, scTestHash3, stateChangeIDsFound["3-3-1"]) // to_id=3 -> scTestHash3 (to_id=3) } -func BenchmarkTransactionModel_BatchInsert(b *testing.B) { - dbt := dbtest.OpenB(b) - defer dbt.Close() - dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) - if err != nil { - b.Fatalf("failed to open db connection pool: %v", err) - } - defer dbConnectionPool.Close() - - ctx := context.Background() - sqlxDB, err := dbConnectionPool.SqlxDB(ctx) - if err != nil { - b.Fatalf("failed to get sqlx db: %v", err) - } - metricsService := metrics.NewMetricsService(sqlxDB) - - m := &TransactionModel{ - DB: dbConnectionPool, - MetricsService: metricsService, - } - - batchSizes := []int{1000, 5000, 10000, 50000, 100000} - - for _, size := range batchSizes { - b.Run(fmt.Sprintf("size=%d", size), func(b *testing.B) { - b.ReportAllocs() - - for i := 0; i < b.N; i++ { - b.StopTimer() - // Clean up before each iteration - //nolint:errcheck // truncate is best-effort cleanup in benchmarks - dbConnectionPool.ExecContext(ctx, "TRUNCATE transactions, transactions_accounts CASCADE") - // Generate fresh test data for each iteration - txs, addressesByToID := generateTestTransactions(size, int32(i*size)) - b.StartTimer() - - _, err := m.BatchInsert(ctx, nil, txs, addressesByToID) - if err != nil { - b.Fatalf("BatchInsert failed: %v", err) - } - } - }) - } -} - // BenchmarkTransactionModel_BatchCopy benchmarks bulk insert using pgx's binary COPY protocol. func BenchmarkTransactionModel_BatchCopy(b *testing.B) { dbt := dbtest.OpenB(b) From 15616300a8778ee8d79e41b2020c7e70ed891d5e Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 11 Feb 2026 11:20:24 -0500 Subject: [PATCH 30/77] update hypertable config - 1 --- internal/db/migrations/2025-06-10.2-transactions.sql | 11 ++++------- internal/db/migrations/2025-06-10.3-operations.sql | 11 ++++------- internal/db/migrations/2025-06-10.4-statechanges.sql | 8 +++----- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/internal/db/migrations/2025-06-10.2-transactions.sql b/internal/db/migrations/2025-06-10.2-transactions.sql index 1a2cb1cb4..6aca01827 100644 --- a/internal/db/migrations/2025-06-10.2-transactions.sql +++ b/internal/db/migrations/2025-06-10.2-transactions.sql @@ -2,7 +2,6 @@ -- Table: transactions (TimescaleDB hypertable with columnstore) CREATE TABLE transactions ( - ledger_created_at TIMESTAMPTZ NOT NULL, to_id BIGINT NOT NULL, hash BYTEA NOT NULL, envelope_xdr TEXT, @@ -12,7 +11,7 @@ CREATE TABLE transactions ( ledger_number INTEGER NOT NULL, is_fee_bump BOOLEAN NOT NULL DEFAULT false, ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (ledger_created_at, to_id) + ledger_created_at TIMESTAMPTZ NOT NULL ) WITH ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', @@ -25,20 +24,18 @@ CREATE INDEX idx_transactions_to_id ON transactions(to_id); -- Table: transactions_accounts (TimescaleDB hypertable for automatic cleanup with retention) CREATE TABLE transactions_accounts ( - ledger_created_at TIMESTAMPTZ NOT NULL, tx_to_id BIGINT NOT NULL, account_id BYTEA NOT NULL, - PRIMARY KEY (ledger_created_at, account_id, tx_to_id) + ledger_created_at TIMESTAMPTZ NOT NULL ) WITH ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', tsdb.chunk_interval = '1 day', - tsdb.orderby = 'ledger_created_at DESC', - tsdb.sparse_index = 'bloom(account_id)' + tsdb.orderby = 'ledger_created_at DESC' ); CREATE INDEX idx_transactions_accounts_tx_to_id ON transactions_accounts(tx_to_id); -CREATE INDEX idx_transactions_accounts_account_id ON transactions_accounts(account_id); +CREATE INDEX idx_transactions_accounts_account_id ON transactions_accounts(account_id, ledger_created_at DESC); -- +migrate Down diff --git a/internal/db/migrations/2025-06-10.3-operations.sql b/internal/db/migrations/2025-06-10.3-operations.sql index 6169e8e0a..380f25c1b 100644 --- a/internal/db/migrations/2025-06-10.3-operations.sql +++ b/internal/db/migrations/2025-06-10.3-operations.sql @@ -2,7 +2,6 @@ -- Table: operations (TimescaleDB hypertable with columnstore) CREATE TABLE operations ( - ledger_created_at TIMESTAMPTZ NOT NULL, id BIGINT NOT NULL, operation_type TEXT NOT NULL CHECK ( operation_type IN ( @@ -23,7 +22,7 @@ CREATE TABLE operations ( successful BOOLEAN NOT NULL, ledger_number INTEGER NOT NULL, ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (ledger_created_at, id) + ledger_created_at TIMESTAMPTZ NOT NULL ) WITH ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', @@ -36,20 +35,18 @@ CREATE INDEX idx_operations_id ON operations(id); -- Table: operations_accounts (TimescaleDB hypertable for automatic cleanup with retention) CREATE TABLE operations_accounts ( - ledger_created_at TIMESTAMPTZ NOT NULL, operation_id BIGINT NOT NULL, account_id BYTEA NOT NULL, - PRIMARY KEY (ledger_created_at, account_id, operation_id) + ledger_created_at TIMESTAMPTZ NOT NULL ) WITH ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', tsdb.chunk_interval = '1 day', - tsdb.orderby = 'ledger_created_at DESC', - tsdb.sparse_index = 'bloom(account_id)' + tsdb.orderby = 'ledger_created_at DESC' ); CREATE INDEX idx_operations_accounts_operation_id ON operations_accounts(operation_id); -CREATE INDEX idx_operations_accounts_account_id ON operations_accounts(account_id); +CREATE INDEX idx_operations_accounts_account_id ON operations_accounts(account_id, ledger_created_at DESC); -- +migrate Down diff --git a/internal/db/migrations/2025-06-10.4-statechanges.sql b/internal/db/migrations/2025-06-10.4-statechanges.sql index bff53f7e0..94bcc69af 100644 --- a/internal/db/migrations/2025-06-10.4-statechanges.sql +++ b/internal/db/migrations/2025-06-10.4-statechanges.sql @@ -3,7 +3,6 @@ -- Table: state_changes (TimescaleDB hypertable with columnstore) -- Note: FK to transactions removed (hypertable FKs not supported) CREATE TABLE state_changes ( - ledger_created_at TIMESTAMPTZ NOT NULL, to_id BIGINT NOT NULL, operation_id BIGINT NOT NULL, state_change_order BIGINT NOT NULL CHECK (state_change_order >= 1), @@ -43,16 +42,15 @@ CREATE TABLE state_changes ( trustline_limit_new TEXT, flags SMALLINT, key_value JSONB, - PRIMARY KEY (ledger_created_at, to_id, operation_id, state_change_order) + ledger_created_at TIMESTAMPTZ NOT NULL ) WITH ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', tsdb.chunk_interval = '1 day', - tsdb.orderby = 'ledger_created_at DESC', - tsdb.sparse_index = 'bloom(account_id)' + tsdb.orderby = 'ledger_created_at DESC' ); -CREATE INDEX idx_state_changes_account_id ON state_changes(account_id); +CREATE INDEX idx_state_changes_account_id ON state_changes(account_id, ledger_created_at DESC); CREATE INDEX idx_state_changes_to_id ON state_changes(to_id); CREATE INDEX idx_state_changes_operation_id ON state_changes(operation_id); From ff75bf719bbb23530d9016fb688b6b851c5c7a66 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 11 Feb 2026 11:30:50 -0500 Subject: [PATCH 31/77] remove Duplicate failure tests --- internal/data/operations_test.go | 78 ------------------------------ internal/data/statechanges_test.go | 76 ----------------------------- internal/data/transactions_test.go | 77 ----------------------------- 3 files changed, 231 deletions(-) diff --git a/internal/data/operations_test.go b/internal/data/operations_test.go index 6221b40a5..fe27139d0 100644 --- a/internal/data/operations_test.go +++ b/internal/data/operations_test.go @@ -217,84 +217,6 @@ func Test_OperationModel_BatchCopy(t *testing.T) { } } -func Test_OperationModel_BatchCopy_DuplicateFails(t *testing.T) { - dbt := dbtest.Open(t) - defer dbt.Close() - dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) - require.NoError(t, err) - defer dbConnectionPool.Close() - - ctx := context.Background() - now := time.Now() - - // Create test accounts - kp1 := keypair.MustRandom() - const q = "INSERT INTO accounts (stellar_address) VALUES ($1)" - _, err = dbConnectionPool.ExecContext(ctx, q, types.AddressBytea(kp1.Address())) - require.NoError(t, err) - - // Create a parent transaction that the operation will reference - _, err = dbConnectionPool.ExecContext(ctx, ` - INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) - VALUES ('tx_for_dup_test', 1, 'env', 100, 'TransactionResultCodeTxSuccess', 'meta', 1, $1, false) - `, now) - require.NoError(t, err) - - op1 := types.Operation{ - ID: 999, - OperationType: types.OperationTypePayment, - OperationXDR: types.XDRBytea([]byte("operation_xdr_dup_test")), - LedgerNumber: 1, - LedgerCreatedAt: now, - } - - // Pre-insert the operation using direct SQL - _, err = dbConnectionPool.ExecContext(ctx, ` - INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) - VALUES ($1, $2, $3, '', true, $4, $5) - `, op1.ID, string(op1.OperationType), []byte(op1.OperationXDR), op1.LedgerNumber, op1.LedgerCreatedAt) - require.NoError(t, err) - _, err = dbConnectionPool.ExecContext(ctx, ` - INSERT INTO operations_accounts (ledger_created_at, operation_id, account_id) VALUES ($1, $2, $3) - `, op1.LedgerCreatedAt, op1.ID, types.AddressBytea(kp1.Address())) - require.NoError(t, err) - - // Verify the operation was inserted - var count int - err = dbConnectionPool.GetContext(ctx, &count, "SELECT COUNT(*) FROM operations WHERE id = $1", op1.ID) - require.NoError(t, err) - require.Equal(t, 1, count) - - // Now try to insert the same operation using BatchCopy - this should FAIL - // because COPY does not support ON CONFLICT handling - mockMetricsService := metrics.NewMockMetricsService() - mockMetricsService.On("IncDBQueryError", "BatchCopy", "operations", mock.Anything).Return().Once() - defer mockMetricsService.AssertExpectations(t) - - m := &OperationModel{ - DB: dbConnectionPool, - MetricsService: mockMetricsService, - } - - conn, err := pgx.Connect(ctx, dbt.DSN) - require.NoError(t, err) - defer conn.Close(ctx) - - pgxTx, err := conn.Begin(ctx) - require.NoError(t, err) - - _, err = m.BatchCopy(ctx, pgxTx, []*types.Operation{&op1}, map[int64]set.Set[string]{ - op1.ID: set.NewSet(kp1.Address()), - }) - - // BatchCopy should fail with a unique constraint violation - require.Error(t, err) - assert.Contains(t, err.Error(), "duplicate key value violates unique constraint") - - // Rollback the failed transaction - require.NoError(t, pgxTx.Rollback(ctx)) -} - func TestOperationModel_GetAll(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() diff --git a/internal/data/statechanges_test.go b/internal/data/statechanges_test.go index 71dd5431b..e2cf539fc 100644 --- a/internal/data/statechanges_test.go +++ b/internal/data/statechanges_test.go @@ -236,82 +236,6 @@ func TestStateChangeModel_BatchCopy(t *testing.T) { } } -func TestStateChangeModel_BatchCopy_DuplicateFails(t *testing.T) { - dbt := dbtest.Open(t) - defer dbt.Close() - dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) - require.NoError(t, err) - defer dbConnectionPool.Close() - - ctx := context.Background() - now := time.Now() - - // Create test account - kp1 := keypair.MustRandom() - const q = "INSERT INTO accounts (stellar_address) VALUES ($1)" - _, err = dbConnectionPool.ExecContext(ctx, q, types.AddressBytea(kp1.Address())) - require.NoError(t, err) - - // Create parent transaction - _, err = dbConnectionPool.ExecContext(ctx, ` - INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) - VALUES ('tx_for_sc_dup_test', 1, 'env', 100, 'TransactionResultCodeTxSuccess', 'meta', 1, $1, false) - `, now) - require.NoError(t, err) - - reason := types.StateChangeReasonCredit - sc1 := types.StateChange{ - ToID: 1, // Must reference the transaction created above with to_id=1 - StateChangeOrder: 1, - StateChangeCategory: types.StateChangeCategoryBalance, - StateChangeReason: &reason, - LedgerCreatedAt: now, - LedgerNumber: 1, - AccountID: types.AddressBytea(kp1.Address()), - OperationID: 123, - } - - // Pre-insert the state change using direct SQL - _, err = dbConnectionPool.ExecContext(ctx, ` - INSERT INTO state_changes (to_id, state_change_order, state_change_category, state_change_reason, ledger_created_at, ledger_number, account_id, operation_id) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) - `, sc1.ToID, sc1.StateChangeOrder, string(sc1.StateChangeCategory), string(*sc1.StateChangeReason), sc1.LedgerCreatedAt, sc1.LedgerNumber, sc1.AccountID, sc1.OperationID) - require.NoError(t, err) - - // Verify the state change was inserted - var count int - err = dbConnectionPool.GetContext(ctx, &count, "SELECT COUNT(*) FROM state_changes WHERE to_id = $1 AND state_change_order = $2", sc1.ToID, sc1.StateChangeOrder) - require.NoError(t, err) - require.Equal(t, 1, count) - - // Now try to insert the same state change using BatchCopy - this should FAIL - // because COPY does not support ON CONFLICT handling - mockMetricsService := metrics.NewMockMetricsService() - mockMetricsService.On("IncDBQueryError", "BatchCopy", "state_changes", mock.Anything).Return().Once() - defer mockMetricsService.AssertExpectations(t) - - m := &StateChangeModel{ - DB: dbConnectionPool, - MetricsService: mockMetricsService, - } - - conn, err := pgx.Connect(ctx, dbt.DSN) - require.NoError(t, err) - defer conn.Close(ctx) - - pgxTx, err := conn.Begin(ctx) - require.NoError(t, err) - - _, err = m.BatchCopy(ctx, pgxTx, []types.StateChange{sc1}) - - // BatchCopy should fail with a unique constraint violation - require.Error(t, err) - assert.Contains(t, err.Error(), "duplicate key value violates unique constraint") - - // Rollback the failed transaction - require.NoError(t, pgxTx.Rollback(ctx)) -} - func TestStateChangeModel_BatchGetByAccountAddress(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() diff --git a/internal/data/transactions_test.go b/internal/data/transactions_test.go index b15e3ea41..3e1b952c6 100644 --- a/internal/data/transactions_test.go +++ b/internal/data/transactions_test.go @@ -221,83 +221,6 @@ func Test_TransactionModel_BatchCopy(t *testing.T) { } } -func Test_TransactionModel_BatchCopy_DuplicateFails(t *testing.T) { - dbt := dbtest.Open(t) - defer dbt.Close() - dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) - require.NoError(t, err) - defer dbConnectionPool.Close() - - ctx := context.Background() - now := time.Now() - - // Create test account - kp1 := keypair.MustRandom() - const q = "INSERT INTO accounts (stellar_address) VALUES ($1)" - _, err = dbConnectionPool.ExecContext(ctx, q, types.AddressBytea(kp1.Address())) - require.NoError(t, err) - - meta := "meta1" - envelope := "envelope1" - txDup := types.Transaction{ - Hash: "f76b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa48766", - ToID: 100, - EnvelopeXDR: &envelope, - FeeCharged: 100, - ResultCode: "TransactionResultCodeTxSuccess", - MetaXDR: &meta, - LedgerNumber: 1, - LedgerCreatedAt: now, - IsFeeBump: false, - } - - // Pre-insert the transaction using direct SQL - _, err = dbConnectionPool.ExecContext(ctx, ` - INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - `, txDup.Hash, txDup.ToID, *txDup.EnvelopeXDR, txDup.FeeCharged, txDup.ResultCode, *txDup.MetaXDR, txDup.LedgerNumber, txDup.LedgerCreatedAt, txDup.IsFeeBump) - require.NoError(t, err) - _, err = dbConnectionPool.ExecContext(ctx, ` - INSERT INTO transactions_accounts (ledger_created_at, tx_to_id, account_id) VALUES ($1, $2, $3) - `, txDup.LedgerCreatedAt, txDup.ToID, types.AddressBytea(kp1.Address())) - require.NoError(t, err) - - // Verify the transaction was inserted - var count int - err = dbConnectionPool.GetContext(ctx, &count, "SELECT COUNT(*) FROM transactions WHERE hash = $1", txDup.Hash) - require.NoError(t, err) - require.Equal(t, 1, count) - - // Now try to insert the same transaction using BatchCopy - this should FAIL - // because COPY does not support ON CONFLICT handling - mockMetricsService := metrics.NewMockMetricsService() - mockMetricsService.On("IncDBQueryError", "BatchCopy", "transactions", mock.Anything).Return().Once() - defer mockMetricsService.AssertExpectations(t) - - m := &TransactionModel{ - DB: dbConnectionPool, - MetricsService: mockMetricsService, - } - - conn, err := pgx.Connect(ctx, dbt.DSN) - require.NoError(t, err) - defer conn.Close(ctx) - - pgxTx, err := conn.Begin(ctx) - require.NoError(t, err) - - _, err = m.BatchCopy(ctx, pgxTx, []*types.Transaction{&txDup}, map[int64]set.Set[string]{ - txDup.ToID: set.NewSet(kp1.Address()), - }) - - // BatchCopy should fail with a unique constraint violation - require.Error(t, err) - assert.Contains(t, err.Error(), "duplicate key value violates unique constraint") - - // Rollback the failed transaction - require.NoError(t, pgxTx.Rollback(ctx)) -} - func TestTransactionModel_GetByHash(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() From e85fb639c95860908e3b97832215e00c2fda998f Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 11 Feb 2026 12:16:33 -0500 Subject: [PATCH 32/77] Add extra orderby columns and other hypertable configration --- internal/db/migrations/2025-06-10.2-transactions.sql | 9 +++++++-- internal/db/migrations/2025-06-10.3-operations.sql | 10 +++++++--- internal/db/migrations/2025-06-10.4-statechanges.sql | 6 +++++- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/internal/db/migrations/2025-06-10.2-transactions.sql b/internal/db/migrations/2025-06-10.2-transactions.sql index 6aca01827..05cd88606 100644 --- a/internal/db/migrations/2025-06-10.2-transactions.sql +++ b/internal/db/migrations/2025-06-10.2-transactions.sql @@ -16,9 +16,11 @@ CREATE TABLE transactions ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', tsdb.chunk_interval = '1 day', - tsdb.orderby = 'ledger_created_at DESC' + tsdb.orderby = 'ledger_created_at DESC, to_id DESC' ); +SELECT enable_chunk_skipping('transactions', 'to_id'); + CREATE INDEX idx_transactions_hash ON transactions(hash); CREATE INDEX idx_transactions_to_id ON transactions(to_id); @@ -31,9 +33,12 @@ CREATE TABLE transactions_accounts ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', tsdb.chunk_interval = '1 day', - tsdb.orderby = 'ledger_created_at DESC' + tsdb.orderby = 'ledger_created_at DESC, tx_to_id DESC', + tsdb.segmentby = 'account_id' ); +SELECT enable_chunk_skipping('transactions_accounts', 'tx_to_id'); + CREATE INDEX idx_transactions_accounts_tx_to_id ON transactions_accounts(tx_to_id); CREATE INDEX idx_transactions_accounts_account_id ON transactions_accounts(account_id, ledger_created_at DESC); diff --git a/internal/db/migrations/2025-06-10.3-operations.sql b/internal/db/migrations/2025-06-10.3-operations.sql index 380f25c1b..1783503ab 100644 --- a/internal/db/migrations/2025-06-10.3-operations.sql +++ b/internal/db/migrations/2025-06-10.3-operations.sql @@ -27,10 +27,11 @@ CREATE TABLE operations ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', tsdb.chunk_interval = '1 day', - tsdb.segmentby = 'operation_type', - tsdb.orderby = 'ledger_created_at DESC' + tsdb.orderby = 'ledger_created_at DESC, id DESC' ); +SELECT enable_chunk_skipping('operations', 'id'); + CREATE INDEX idx_operations_id ON operations(id); -- Table: operations_accounts (TimescaleDB hypertable for automatic cleanup with retention) @@ -42,9 +43,12 @@ CREATE TABLE operations_accounts ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', tsdb.chunk_interval = '1 day', - tsdb.orderby = 'ledger_created_at DESC' + tsdb.orderby = 'ledger_created_at DESC, operation_id DESC', + tsdb.segmentby = 'account_id' ); +SELECT enable_chunk_skipping('operations_accounts', 'operation_id'); + CREATE INDEX idx_operations_accounts_operation_id ON operations_accounts(operation_id); CREATE INDEX idx_operations_accounts_account_id ON operations_accounts(account_id, ledger_created_at DESC); diff --git a/internal/db/migrations/2025-06-10.4-statechanges.sql b/internal/db/migrations/2025-06-10.4-statechanges.sql index 94bcc69af..9f6d0c7e9 100644 --- a/internal/db/migrations/2025-06-10.4-statechanges.sql +++ b/internal/db/migrations/2025-06-10.4-statechanges.sql @@ -47,9 +47,13 @@ CREATE TABLE state_changes ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', tsdb.chunk_interval = '1 day', - tsdb.orderby = 'ledger_created_at DESC' + tsdb.orderby = 'ledger_created_at DESC, to_id DESC, operation_id DESC, state_change_order DESC', + tsdb.segmentby = 'account_id' ); +SELECT enable_chunk_skipping('state_changes', 'to_id'); +SELECT enable_chunk_skipping('state_changes', 'operation_id'); + CREATE INDEX idx_state_changes_account_id ON state_changes(account_id, ledger_created_at DESC); CREATE INDEX idx_state_changes_to_id ON state_changes(to_id); CREATE INDEX idx_state_changes_operation_id ON state_changes(operation_id); From 7a6334b3a00c4f562e48826d12f3352106a8a328 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 11 Feb 2026 12:33:10 -0500 Subject: [PATCH 33/77] Update normal B-tree indexes --- internal/db/migrations/2025-06-10.2-transactions.sql | 2 +- internal/db/migrations/2025-06-10.3-operations.sql | 2 +- internal/db/migrations/2025-06-10.4-statechanges.sql | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/db/migrations/2025-06-10.2-transactions.sql b/internal/db/migrations/2025-06-10.2-transactions.sql index 05cd88606..3424e7e2c 100644 --- a/internal/db/migrations/2025-06-10.2-transactions.sql +++ b/internal/db/migrations/2025-06-10.2-transactions.sql @@ -40,7 +40,7 @@ CREATE TABLE transactions_accounts ( SELECT enable_chunk_skipping('transactions_accounts', 'tx_to_id'); CREATE INDEX idx_transactions_accounts_tx_to_id ON transactions_accounts(tx_to_id); -CREATE INDEX idx_transactions_accounts_account_id ON transactions_accounts(account_id, ledger_created_at DESC); +CREATE INDEX idx_transactions_accounts_account_id ON transactions_accounts(account_id, ledger_created_at DESC, tx_to_id DESC); -- +migrate Down diff --git a/internal/db/migrations/2025-06-10.3-operations.sql b/internal/db/migrations/2025-06-10.3-operations.sql index 1783503ab..497f9f1f2 100644 --- a/internal/db/migrations/2025-06-10.3-operations.sql +++ b/internal/db/migrations/2025-06-10.3-operations.sql @@ -50,7 +50,7 @@ CREATE TABLE operations_accounts ( SELECT enable_chunk_skipping('operations_accounts', 'operation_id'); CREATE INDEX idx_operations_accounts_operation_id ON operations_accounts(operation_id); -CREATE INDEX idx_operations_accounts_account_id ON operations_accounts(account_id, ledger_created_at DESC); +CREATE INDEX idx_operations_accounts_account_id ON operations_accounts(account_id, ledger_created_at DESC, operation_id DESC); -- +migrate Down diff --git a/internal/db/migrations/2025-06-10.4-statechanges.sql b/internal/db/migrations/2025-06-10.4-statechanges.sql index 9f6d0c7e9..a6b33ea5d 100644 --- a/internal/db/migrations/2025-06-10.4-statechanges.sql +++ b/internal/db/migrations/2025-06-10.4-statechanges.sql @@ -54,8 +54,8 @@ CREATE TABLE state_changes ( SELECT enable_chunk_skipping('state_changes', 'to_id'); SELECT enable_chunk_skipping('state_changes', 'operation_id'); -CREATE INDEX idx_state_changes_account_id ON state_changes(account_id, ledger_created_at DESC); -CREATE INDEX idx_state_changes_to_id ON state_changes(to_id); +CREATE INDEX idx_state_changes_account_id ON state_changes(account_id, ledger_created_at DESC, to_id DESC, operation_id DESC, state_change_order DESC); +CREATE INDEX idx_state_changes_to_id ON state_changes(to_id, operation_id DESC, state_change_order DESC); CREATE INDEX idx_state_changes_operation_id ON state_changes(operation_id); -- +migrate Down From 6768f8d666d8103e225753ce4c12b236380f9e51 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 11 Feb 2026 12:41:16 -0500 Subject: [PATCH 34/77] enable chunk skipping --- docker-compose.yaml | 1 + internal/db/dbtest/dbtest.go | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index f1ac51598..e72c4542e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -8,6 +8,7 @@ services: db: container_name: db image: timescale/timescaledb:2.25.0-pg17 + command: ["postgres", "-c", "timescaledb.enable_chunk_skipping=on"] healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres -d wallet-backend"] interval: 10s diff --git a/internal/db/dbtest/dbtest.go b/internal/db/dbtest/dbtest.go index 2ddfe6eb7..6da1186c0 100644 --- a/internal/db/dbtest/dbtest.go +++ b/internal/db/dbtest/dbtest.go @@ -25,6 +25,12 @@ func Open(t *testing.T) *dbtest.DB { t.Fatal(err) } + // Enable chunk skipping GUC required by enable_chunk_skipping() in migrations + _, err = conn.Exec("SET timescaledb.enable_chunk_skipping = on") + if err != nil { + t.Fatal(err) + } + migrateDirection := schema.MigrateUp m := migrate.HttpFileSystemMigrationSource{FileSystem: http.FS(migrations.FS)} _, err = schema.Migrate(conn.DB, m, migrateDirection, 0) @@ -48,6 +54,12 @@ func OpenWithoutMigrations(t *testing.T) *dbtest.DB { t.Fatal(err) } + // Enable chunk skipping GUC required by enable_chunk_skipping() in migrations + _, err = conn.Exec("SET timescaledb.enable_chunk_skipping = on") + if err != nil { + t.Fatal(err) + } + return db } @@ -64,6 +76,12 @@ func OpenB(b *testing.B) *dbtest.DB { b.Fatal(err) } + // Enable chunk skipping GUC required by enable_chunk_skipping() in migrations + _, err = conn.Exec("SET timescaledb.enable_chunk_skipping = on") + if err != nil { + b.Fatal(err) + } + migrateDirection := schema.MigrateUp m := migrate.HttpFileSystemMigrationSource{FileSystem: http.FS(migrations.FS)} _, err = schema.Migrate(conn.DB, m, migrateDirection, 0) From a4150993a269576c81b404203678587c26db6d2a Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 11 Feb 2026 12:52:59 -0500 Subject: [PATCH 35/77] Update dbtest.go --- internal/db/dbtest/dbtest.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/db/dbtest/dbtest.go b/internal/db/dbtest/dbtest.go index 6da1186c0..67adc9e9b 100644 --- a/internal/db/dbtest/dbtest.go +++ b/internal/db/dbtest/dbtest.go @@ -25,7 +25,7 @@ func Open(t *testing.T) *dbtest.DB { t.Fatal(err) } - // Enable chunk skipping GUC required by enable_chunk_skipping() in migrations + // Also set on the current session since ALTER DATABASE only affects new connections _, err = conn.Exec("SET timescaledb.enable_chunk_skipping = on") if err != nil { t.Fatal(err) @@ -54,8 +54,9 @@ func OpenWithoutMigrations(t *testing.T) *dbtest.DB { t.Fatal(err) } - // Enable chunk skipping GUC required by enable_chunk_skipping() in migrations - _, err = conn.Exec("SET timescaledb.enable_chunk_skipping = on") + // Enable chunk skipping at the database level so all connections (including + // those opened by the Migrate function) inherit this setting. + _, err = conn.Exec("DO $$ BEGIN EXECUTE format('ALTER DATABASE %I SET timescaledb.enable_chunk_skipping = on', current_database()); END $$") if err != nil { t.Fatal(err) } @@ -76,7 +77,7 @@ func OpenB(b *testing.B) *dbtest.DB { b.Fatal(err) } - // Enable chunk skipping GUC required by enable_chunk_skipping() in migrations + // Also set on the current session since ALTER DATABASE only affects new connections _, err = conn.Exec("SET timescaledb.enable_chunk_skipping = on") if err != nil { b.Fatal(err) From d0e76dfcddf4292866e345e0d0da85814313f3c1 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 11 Feb 2026 13:55:18 -0500 Subject: [PATCH 36/77] Update containers.go --- internal/integrationtests/infrastructure/containers.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/integrationtests/infrastructure/containers.go b/internal/integrationtests/infrastructure/containers.go index 80cbfcaa2..9316d87e8 100644 --- a/internal/integrationtests/infrastructure/containers.go +++ b/internal/integrationtests/infrastructure/containers.go @@ -322,6 +322,7 @@ func createWalletDBContainer(ctx context.Context, testNetwork *testcontainers.Do containerRequest := testcontainers.ContainerRequest{ Name: walletBackendDBContainerName, Image: "timescale/timescaledb:latest-pg17", + Cmd: []string{"postgres", "-c", "timescaledb.enable_chunk_skipping=on"}, Labels: map[string]string{ "org.testcontainers.session-id": "wallet-backend-integration-tests", }, From ed7181520d8bbc320e10387e31963edb5f9d0aa1 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 11 Feb 2026 17:11:08 -0500 Subject: [PATCH 37/77] Compress backfill chunks parallelly using goroutine --- internal/services/ingest_backfill.go | 160 ++++++++++++++++++++------- internal/services/ingest_test.go | 99 +++++++++++++++++ 2 files changed, 221 insertions(+), 38 deletions(-) diff --git a/internal/services/ingest_backfill.go b/internal/services/ingest_backfill.go index 00ec1eb46..eee2d15fe 100644 --- a/internal/services/ingest_backfill.go +++ b/internal/services/ingest_backfill.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "maps" + "sync" + "sync/atomic" "time" "github.com/google/uuid" @@ -28,6 +30,8 @@ func (m BackfillMode) isCatchup() bool { return m == BackfillModeCatchup } +const progressiveCompressionInterval = 30 * time.Second + const ( // BackfillModeHistorical fills gaps within already-ingested ledger range. BackfillModeHistorical BackfillMode = iota @@ -331,23 +335,83 @@ func (m *ingestService) splitGapsIntoBatches(gaps []data.LedgerRange) []Backfill } // processBackfillBatchesParallel processes backfill batches in parallel using a worker pool. +// For historical mode, a background goroutine progressively compresses chunks as contiguous +// batches complete, using a watermark to avoid interfering with in-flight writes. func (m *ingestService) processBackfillBatchesParallel(ctx context.Context, mode BackfillMode, batches []BackfillBatch) []BackfillResult { results := make([]BackfillResult, len(batches)) + completed := make([]atomic.Bool, len(batches)) group := m.backfillPool.NewGroupContext(ctx) + // Start background compression goroutine for historical mode + var compressionWg sync.WaitGroup + var done chan struct{} + if mode.isHistorical() && len(batches) > 0 { + done = make(chan struct{}) + compressionWg.Add(1) + go func() { + defer compressionWg.Done() + m.runProgressiveCompression(ctx, results, completed, done) + }() + } + for i, batch := range batches { group.Submit(func() { result := m.processSingleBatch(ctx, mode, batch) results[i] = result + completed[i].Store(true) }) } if err := group.Wait(); err != nil { log.Ctx(ctx).Warnf("Backfill batch group wait returned error: %v", err) } + + // Signal compression goroutine to stop and wait for it to finish + if done != nil { + close(done) + compressionWg.Wait() + } + return results } +// runProgressiveCompression periodically checks for contiguous completed batches +// and compresses their chunks. Only compresses up to the highest contiguous +// completed+successful batch index to avoid interfering with in-flight writes. +func (m *ingestService) runProgressiveCompression(ctx context.Context, results []BackfillResult, completed []atomic.Bool, done chan struct{}) { + watermark := -1 + ticker := time.NewTicker(progressiveCompressionInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-done: + return + case <-ticker.C: + newWatermark := watermark + for i := watermark + 1; i < len(completed); i++ { + if !completed[i].Load() { + break + } + if results[i].Error != nil { + break + } + newWatermark = i + } + + if newWatermark > watermark { + endTime := results[newWatermark].EndTime + if !endTime.IsZero() { + m.compressBackfilledChunks(ctx, time.Time{}, endTime) + } + watermark = newWatermark + } + } + } +} + // processSingleBatch processes a single backfill batch with its own ledger backend. func (m *ingestService) processSingleBatch(ctx context.Context, mode BackfillMode, batch BackfillBatch) BackfillResult { start := time.Now() @@ -616,54 +680,74 @@ func (m *ingestService) processBatchChanges( } // compressBackfilledChunks compresses uncompressed chunks overlapping the backfill range. -// Chunks are compressed sequentially to avoid OOM errors during large backfills. +// Tables are compressed concurrently; chunks within each table stay sequential to avoid OOM. // Skips chunks where range_end >= NOW() to avoid compressing active live ingestion chunks. func (m *ingestService) compressBackfilledChunks(ctx context.Context, startTime, endTime time.Time) { tables := []string{"transactions", "transactions_accounts", "operations", "operations_accounts", "state_changes"} + + var wg sync.WaitGroup + tableCounts := make([]int, len(tables)) + + for i, table := range tables { + wg.Add(1) + go func() { + defer wg.Done() + tableCounts[i] = m.compressTableChunks(ctx, table, startTime, endTime) + }() + } + + wg.Wait() + totalCompressed := 0 + for _, count := range tableCounts { + totalCompressed += count + } + log.Ctx(ctx).Infof("Compressed %d total chunks for time range [%s - %s]", + totalCompressed, startTime.Format(time.RFC3339), endTime.Format(time.RFC3339)) +} - for _, table := range tables { - rows, err := m.models.DB.PgxPool().Query(ctx, - `SELECT chunk_schema || '.' || chunk_name FROM timescaledb_information.chunks - WHERE hypertable_name = $1 AND NOT is_compressed - AND range_start < $2::timestamptz AND range_end > $3::timestamptz - AND range_end < NOW()`, - table, endTime, startTime) - if err != nil { - log.Ctx(ctx).Warnf("Failed to get chunks for %s: %v", table, err) +// compressTableChunks compresses uncompressed chunks for a single hypertable. +// Chunks are compressed sequentially within the table to avoid OOM errors. +func (m *ingestService) compressTableChunks(ctx context.Context, table string, startTime, endTime time.Time) int { + rows, err := m.models.DB.PgxPool().Query(ctx, + `SELECT chunk_schema || '.' || chunk_name FROM timescaledb_information.chunks + WHERE hypertable_name = $1 AND NOT is_compressed + AND range_start < $2::timestamptz AND range_end > $3::timestamptz + AND range_end < NOW()`, + table, endTime, startTime) + if err != nil { + log.Ctx(ctx).Warnf("Failed to get chunks for %s: %v", table, err) + return 0 + } + + var chunks []string + for rows.Next() { + var chunk string + if err := rows.Scan(&chunk); err != nil { continue } + chunks = append(chunks, chunk) + } + rows.Close() - var chunks []string - for rows.Next() { - var chunk string - if err := rows.Scan(&chunk); err != nil { - continue - } - chunks = append(chunks, chunk) + compressed := 0 + for i, chunk := range chunks { + select { + case <-ctx.Done(): + log.Ctx(ctx).Warnf("Compression cancelled for %s after %d chunks", table, compressed) + return compressed + default: } - rows.Close() - - // Compress chunks sequentially to avoid OOM - for i, chunk := range chunks { - select { - case <-ctx.Done(): - log.Ctx(ctx).Warnf("Compression cancelled after %d chunks", totalCompressed) - return - default: - } - _, err := m.models.DB.PgxPool().Exec(ctx, - `CALL convert_to_columnstore($1::regclass, if_not_columnstore => true)`, chunk) - if err != nil { - log.Ctx(ctx).Warnf("Failed to compress chunk %s: %v", chunk, err) - continue - } - totalCompressed++ - log.Ctx(ctx).Debugf("Compressed chunk %d/%d for %s: %s", i+1, len(chunks), table, chunk) + _, err := m.models.DB.PgxPool().Exec(ctx, + `CALL convert_to_columnstore($1::regclass, if_not_columnstore => true)`, chunk) + if err != nil { + log.Ctx(ctx).Warnf("Failed to compress chunk %s: %v", chunk, err) + continue } - log.Ctx(ctx).Infof("Compressed %d chunks for table %s", len(chunks), table) + compressed++ + log.Ctx(ctx).Debugf("Compressed chunk %d/%d for %s: %s", i+1, len(chunks), table, chunk) } - log.Ctx(ctx).Infof("Compressed %d total chunks for time range [%s - %s]", - totalCompressed, startTime.Format(time.RFC3339), endTime.Format(time.RFC3339)) + log.Ctx(ctx).Infof("Compressed %d chunks for table %s", len(chunks), table) + return compressed } diff --git a/internal/services/ingest_test.go b/internal/services/ingest_test.go index 0f906e73b..b565f9b21 100644 --- a/internal/services/ingest_test.go +++ b/internal/services/ingest_test.go @@ -3094,3 +3094,102 @@ func Test_ingestService_startBackfilling_CatchupMode_ProcessesBatchChanges(t *te }) } } + +// Test_ingestService_processBackfillBatchesParallel_ProgressiveCompression verifies +// that historical mode launches a background compression goroutine without deadlocks, +// and that catchup mode skips progressive compression entirely. +func Test_ingestService_processBackfillBatchesParallel_ProgressiveCompression(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + + testCases := []struct { + name string + mode BackfillMode + }{ + { + name: "historical_mode_triggers_progressive_compression", + mode: BackfillModeHistorical, + }, + { + name: "catchup_mode_skips_progressive_compression", + mode: BackfillModeCatchup, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + mockMetricsService := metrics.NewMockMetricsService() + mockMetricsService.On("RegisterPoolMetrics", "ledger_indexer", mock.Anything).Return() + mockMetricsService.On("RegisterPoolMetrics", "backfill", mock.Anything).Return() + mockMetricsService.On("SetOldestLedgerIngested", mock.Anything).Return().Maybe() + mockMetricsService.On("ObserveDBQueryDuration", mock.Anything, mock.Anything, mock.Anything).Return().Maybe() + mockMetricsService.On("IncDBQuery", mock.Anything, mock.Anything).Return().Maybe() + mockMetricsService.On("IncDBTransaction", mock.Anything).Return().Maybe() + mockMetricsService.On("ObserveDBTransactionDuration", mock.Anything, mock.Anything).Return().Maybe() + mockMetricsService.On("ObserveDBBatchSize", mock.Anything, mock.Anything, mock.Anything).Return().Maybe() + mockMetricsService.On("ObserveIngestionParticipantsCount", mock.Anything).Return().Maybe() + mockMetricsService.On("IncStateChanges", mock.Anything, mock.Anything, mock.Anything).Return().Maybe() + defer mockMetricsService.AssertExpectations(t) + + models, modelsErr := data.NewModels(dbConnectionPool, mockMetricsService) + require.NoError(t, modelsErr) + + mockRPCService := &RPCServiceMock{} + mockRPCService.On("NetworkPassphrase").Return(network.TestNetworkPassphrase).Maybe() + + // Factory that returns a backend with minimal valid ledger data + factory := func(ctx context.Context) (ledgerbackend.LedgerBackend, error) { + mockBackend := &LedgerBackendMock{} + mockBackend.On("PrepareRange", mock.Anything, mock.Anything).Return(nil) + mockBackend.On("GetLedger", mock.Anything, mock.Anything).Return(xdr.LedgerCloseMeta{ + V: 0, + V0: &xdr.LedgerCloseMetaV0{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + LedgerSeq: xdr.Uint32(100), + }, + }, + }, + }, nil) + mockBackend.On("Close").Return(nil) + return mockBackend, nil + } + + svc, svcErr := NewIngestService(IngestServiceConfig{ + IngestionMode: IngestionModeBackfill, + Models: models, + LatestLedgerCursorName: "latest_ledger_cursor", + OldestLedgerCursorName: "oldest_ledger_cursor", + AppTracker: &apptracker.MockAppTracker{}, + RPCService: mockRPCService, + LedgerBackend: &LedgerBackendMock{}, + LedgerBackendFactory: factory, + MetricsService: mockMetricsService, + GetLedgersLimit: defaultGetLedgersLimit, + Network: network.TestNetworkPassphrase, + NetworkPassphrase: network.TestNetworkPassphrase, + Archive: &HistoryArchiveMock{}, + BackfillBatchSize: 10, + }) + require.NoError(t, svcErr) + + batches := []BackfillBatch{ + {StartLedger: 100, EndLedger: 100}, + {StartLedger: 101, EndLedger: 101}, + } + + results := svc.processBackfillBatchesParallel(ctx, tc.mode, batches) + + // All batches should succeed + require.Len(t, results, 2) + for i, result := range results { + assert.NoError(t, result.Error, "batch %d should succeed", i) + } + }) + } +} From b9305e5598eca925254c7cebbfb009e3e0af6d8b Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 12 Feb 2026 09:38:53 -0500 Subject: [PATCH 38/77] remove schemas for mainnet and testnet --- docker-compose.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index e72c4542e..a6ad38372 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -270,5 +270,3 @@ configs: postgres_init: content: | CREATE EXTENSION IF NOT EXISTS timescaledb; - CREATE SCHEMA IF NOT EXISTS wallet_backend_mainnet; - CREATE SCHEMA IF NOT EXISTS wallet_backend_testnet; From 5a5ac6e66502787c81590d58ea6da87b467f04d5 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 12 Feb 2026 09:39:08 -0500 Subject: [PATCH 39/77] Enable direct compress and recompression for backfilling --- internal/services/ingest_backfill.go | 101 +++++++-------------------- internal/services/ingest_test.go | 15 ++-- 2 files changed, 31 insertions(+), 85 deletions(-) diff --git a/internal/services/ingest_backfill.go b/internal/services/ingest_backfill.go index eee2d15fe..1d87ab2d3 100644 --- a/internal/services/ingest_backfill.go +++ b/internal/services/ingest_backfill.go @@ -5,7 +5,6 @@ import ( "fmt" "maps" "sync" - "sync/atomic" "time" "github.com/google/uuid" @@ -30,8 +29,6 @@ func (m BackfillMode) isCatchup() bool { return m == BackfillModeCatchup } -const progressiveCompressionInterval = 30 * time.Second - const ( // BackfillModeHistorical fills gaps within already-ingested ledger range. BackfillModeHistorical BackfillMode = iota @@ -335,30 +332,15 @@ func (m *ingestService) splitGapsIntoBatches(gaps []data.LedgerRange) []Backfill } // processBackfillBatchesParallel processes backfill batches in parallel using a worker pool. -// For historical mode, a background goroutine progressively compresses chunks as contiguous -// batches complete, using a watermark to avoid interfering with in-flight writes. +// For historical mode, direct compress handles compression during COPY; a single recompression +// pass runs after all batches complete (in startBackfilling). func (m *ingestService) processBackfillBatchesParallel(ctx context.Context, mode BackfillMode, batches []BackfillBatch) []BackfillResult { results := make([]BackfillResult, len(batches)) - completed := make([]atomic.Bool, len(batches)) group := m.backfillPool.NewGroupContext(ctx) - // Start background compression goroutine for historical mode - var compressionWg sync.WaitGroup - var done chan struct{} - if mode.isHistorical() && len(batches) > 0 { - done = make(chan struct{}) - compressionWg.Add(1) - go func() { - defer compressionWg.Done() - m.runProgressiveCompression(ctx, results, completed, done) - }() - } - for i, batch := range batches { group.Submit(func() { - result := m.processSingleBatch(ctx, mode, batch) - results[i] = result - completed[i].Store(true) + results[i] = m.processSingleBatch(ctx, mode, batch) }) } @@ -366,52 +348,9 @@ func (m *ingestService) processBackfillBatchesParallel(ctx context.Context, mode log.Ctx(ctx).Warnf("Backfill batch group wait returned error: %v", err) } - // Signal compression goroutine to stop and wait for it to finish - if done != nil { - close(done) - compressionWg.Wait() - } - return results } -// runProgressiveCompression periodically checks for contiguous completed batches -// and compresses their chunks. Only compresses up to the highest contiguous -// completed+successful batch index to avoid interfering with in-flight writes. -func (m *ingestService) runProgressiveCompression(ctx context.Context, results []BackfillResult, completed []atomic.Bool, done chan struct{}) { - watermark := -1 - ticker := time.NewTicker(progressiveCompressionInterval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-done: - return - case <-ticker.C: - newWatermark := watermark - for i := watermark + 1; i < len(completed); i++ { - if !completed[i].Load() { - break - } - if results[i].Error != nil { - break - } - newWatermark = i - } - - if newWatermark > watermark { - endTime := results[newWatermark].EndTime - if !endTime.IsZero() { - m.compressBackfilledChunks(ctx, time.Time{}, endTime) - } - watermark = newWatermark - } - } - } -} - // processSingleBatch processes a single backfill batch with its own ledger backend. func (m *ingestService) processSingleBatch(ctx context.Context, mode BackfillMode, batch BackfillBatch) BackfillResult { start := time.Now() @@ -472,7 +411,8 @@ func (m *ingestService) setupBatchBackend(ctx context.Context, batch BackfillBat // flushBatchBufferWithRetry persists buffered data to the database within a transaction. // If updateCursorTo is non-nil, it also updates the oldest cursor atomically. -func (m *ingestService) flushBatchBufferWithRetry(ctx context.Context, buffer *indexer.IndexerBuffer, updateCursorTo *uint32, batchChanges *BatchChanges) error { +// If directCompress is true, enables TimescaleDB direct compress for COPY operations. +func (m *ingestService) flushBatchBufferWithRetry(ctx context.Context, buffer *indexer.IndexerBuffer, updateCursorTo *uint32, batchChanges *BatchChanges, directCompress bool) error { var lastErr error for attempt := 0; attempt < maxIngestProcessedDataRetries; attempt++ { select { @@ -482,6 +422,11 @@ func (m *ingestService) flushBatchBufferWithRetry(ctx context.Context, buffer *i } err := db.RunInPgxTransaction(ctx, m.models.DB, func(dbTx pgx.Tx) error { + if directCompress { + if _, err := dbTx.Exec(ctx, "SET LOCAL timescaledb.enable_direct_compress_copy = on"); err != nil { + return fmt.Errorf("enabling direct compress: %w", err) + } + } filteredData, err := m.filterParticipantData(ctx, dbTx, buffer) if err != nil { return fmt.Errorf("filtering participant data: %w", err) @@ -590,7 +535,7 @@ func (m *ingestService) processLedgersInBatch( // Flush buffer periodically to control memory usage (intermediate flushes, no cursor update) if ledgersInBuffer >= m.backfillDBInsertBatchSize { - if err := m.flushBatchBufferWithRetry(ctx, batchBuffer, nil, batchChanges); err != nil { + if err := m.flushBatchBufferWithRetry(ctx, batchBuffer, nil, batchChanges, mode.isHistorical()); err != nil { return ledgersProcessed, batchChanges, startTime, endTime, err } batchBuffer.Clear() @@ -604,7 +549,7 @@ func (m *ingestService) processLedgersInBatch( if mode.isHistorical() { cursorUpdate = &batch.StartLedger } - if err := m.flushBatchBufferWithRetry(ctx, batchBuffer, cursorUpdate, batchChanges); err != nil { + if err := m.flushBatchBufferWithRetry(ctx, batchBuffer, cursorUpdate, batchChanges, mode.isHistorical()); err != nil { return ledgersProcessed, batchChanges, startTime, endTime, err } } else if mode.isHistorical() { @@ -679,7 +624,8 @@ func (m *ingestService) processBatchChanges( return nil } -// compressBackfilledChunks compresses uncompressed chunks overlapping the backfill range. +// compressBackfilledChunks recompresses already-compressed chunks overlapping the backfill range. +// Direct compress produces compressed chunks during COPY; recompression optimizes compression ratios. // Tables are compressed concurrently; chunks within each table stay sequential to avoid OOM. // Skips chunks where range_end >= NOW() to avoid compressing active live ingestion chunks. func (m *ingestService) compressBackfilledChunks(ctx context.Context, startTime, endTime time.Time) { @@ -702,16 +648,17 @@ func (m *ingestService) compressBackfilledChunks(ctx context.Context, startTime, for _, count := range tableCounts { totalCompressed += count } - log.Ctx(ctx).Infof("Compressed %d total chunks for time range [%s - %s]", + log.Ctx(ctx).Infof("Recompressed %d total chunks for time range [%s - %s]", totalCompressed, startTime.Format(time.RFC3339), endTime.Format(time.RFC3339)) } -// compressTableChunks compresses uncompressed chunks for a single hypertable. -// Chunks are compressed sequentially within the table to avoid OOM errors. +// compressTableChunks recompresses already-compressed chunks for a single hypertable. +// Direct compress produces compressed chunks during COPY; this pass optimizes compression ratios. +// Chunks are recompressed sequentially within the table to avoid OOM errors. func (m *ingestService) compressTableChunks(ctx context.Context, table string, startTime, endTime time.Time) int { rows, err := m.models.DB.PgxPool().Query(ctx, `SELECT chunk_schema || '.' || chunk_name FROM timescaledb_information.chunks - WHERE hypertable_name = $1 AND NOT is_compressed + WHERE hypertable_name = $1 AND is_compressed AND range_start < $2::timestamptz AND range_end > $3::timestamptz AND range_end < NOW()`, table, endTime, startTime) @@ -734,20 +681,20 @@ func (m *ingestService) compressTableChunks(ctx context.Context, table string, s for i, chunk := range chunks { select { case <-ctx.Done(): - log.Ctx(ctx).Warnf("Compression cancelled for %s after %d chunks", table, compressed) + log.Ctx(ctx).Warnf("Recompression cancelled for %s after %d chunks", table, compressed) return compressed default: } _, err := m.models.DB.PgxPool().Exec(ctx, - `CALL convert_to_columnstore($1::regclass, if_not_columnstore => true)`, chunk) + `CALL convert_to_columnstore($1::regclass, if_not_columnstore => true, recompress => true)`, chunk) if err != nil { - log.Ctx(ctx).Warnf("Failed to compress chunk %s: %v", chunk, err) + log.Ctx(ctx).Warnf("Failed to recompress chunk %s: %v", chunk, err) continue } compressed++ - log.Ctx(ctx).Debugf("Compressed chunk %d/%d for %s: %s", i+1, len(chunks), table, chunk) + log.Ctx(ctx).Debugf("Recompressed chunk %d/%d for %s: %s", i+1, len(chunks), table, chunk) } - log.Ctx(ctx).Infof("Compressed %d chunks for table %s", len(chunks), table) + log.Ctx(ctx).Infof("Recompressed %d chunks for table %s", len(chunks), table) return compressed } diff --git a/internal/services/ingest_test.go b/internal/services/ingest_test.go index b565f9b21..b1571ae47 100644 --- a/internal/services/ingest_test.go +++ b/internal/services/ingest_test.go @@ -1317,7 +1317,7 @@ func Test_ingestService_flushBatchBufferWithRetry(t *testing.T) { buffer := tc.setupBuffer() // Call flushBatchBuffer - err = svc.flushBatchBufferWithRetry(ctx, buffer, tc.updateCursorTo, nil) + err = svc.flushBatchBufferWithRetry(ctx, buffer, tc.updateCursorTo, nil, false) require.NoError(t, err) // Verify the cursor value @@ -2820,7 +2820,7 @@ func Test_ingestService_flushBatchBuffer_batchChanges(t *testing.T) { buffer := tc.setupBuffer() - err = svc.flushBatchBufferWithRetry(ctx, buffer, nil, tc.batchChanges) + err = svc.flushBatchBufferWithRetry(ctx, buffer, nil, tc.batchChanges, false) require.NoError(t, err) // Verify collected token changes match expected values @@ -3095,10 +3095,9 @@ func Test_ingestService_startBackfilling_CatchupMode_ProcessesBatchChanges(t *te } } -// Test_ingestService_processBackfillBatchesParallel_ProgressiveCompression verifies -// that historical mode launches a background compression goroutine without deadlocks, -// and that catchup mode skips progressive compression entirely. -func Test_ingestService_processBackfillBatchesParallel_ProgressiveCompression(t *testing.T) { +// Test_ingestService_processBackfillBatchesParallel_BothModes verifies +// that both historical and catchup modes process batches successfully. +func Test_ingestService_processBackfillBatchesParallel_BothModes(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) @@ -3112,11 +3111,11 @@ func Test_ingestService_processBackfillBatchesParallel_ProgressiveCompression(t mode BackfillMode }{ { - name: "historical_mode_triggers_progressive_compression", + name: "historical_mode_processes_batches", mode: BackfillModeHistorical, }, { - name: "catchup_mode_skips_progressive_compression", + name: "catchup_mode_processes_batches", mode: BackfillModeCatchup, }, } From 77dbbc7f8ea243e98fdcae1378ded167f3a0d38e Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 12 Feb 2026 11:17:46 -0500 Subject: [PATCH 40/77] Add command line configuration to set retention policy and chunk size --- cmd/ingest.go | 16 +++ internal/ingest/ingest.go | 10 ++ internal/ingest/timescaledb.go | 57 +++++++++++ internal/ingest/timescaledb_test.go | 152 ++++++++++++++++++++++++++++ 4 files changed, 235 insertions(+) create mode 100644 internal/ingest/timescaledb.go create mode 100644 internal/ingest/timescaledb_test.go diff --git a/cmd/ingest.go b/cmd/ingest.go index e33ce6cce..db039b157 100644 --- a/cmd/ingest.go +++ b/cmd/ingest.go @@ -140,6 +140,22 @@ func (c *ingestCmd) Command() *cobra.Command { FlagDefault: true, Required: false, }, + { + Name: "chunk-interval", + Usage: "TimescaleDB chunk time interval for hypertables. Only affects future chunks. Uses PostgreSQL INTERVAL syntax.", + OptType: types.String, + ConfigKey: &cfg.ChunkInterval, + FlagDefault: "1 day", + Required: false, + }, + { + Name: "retention-period", + Usage: "TimescaleDB data retention period. Chunks older than this are automatically dropped. Empty disables retention. Uses PostgreSQL INTERVAL syntax.", + OptType: types.String, + ConfigKey: &cfg.RetentionPeriod, + FlagDefault: "", + Required: false, + }, } cmd := &cobra.Command{ diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index a6863b3cb..448886b04 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -87,6 +87,12 @@ type Configs struct { // CatchupThreshold is the number of ledgers behind network tip that triggers fast catchup. // Defaults to 100. CatchupThreshold int + // ChunkInterval sets the TimescaleDB chunk time interval for hypertables. + // Only affects future chunks. Uses PostgreSQL INTERVAL syntax (e.g., "1 day", "7 days"). + ChunkInterval string + // RetentionPeriod configures automatic data retention. Chunks older than this are dropped. + // Empty string disables retention. Uses PostgreSQL INTERVAL syntax (e.g., "30 days", "6 months"). + RetentionPeriod string } func Ingest(cfg Configs) error { @@ -135,6 +141,10 @@ func setupDeps(cfg Configs) (services.IngestService, error) { return nil, fmt.Errorf("getting sqlx db: %w", err) } + if err := configureHypertableSettings(ctx, dbConnectionPool, cfg.ChunkInterval, cfg.RetentionPeriod); err != nil { + return nil, fmt.Errorf("configuring hypertable settings: %w", err) + } + metricsService := metrics.NewMetricsService(sqlxDB) models, err := data.NewModels(dbConnectionPool, metricsService) if err != nil { diff --git a/internal/ingest/timescaledb.go b/internal/ingest/timescaledb.go new file mode 100644 index 000000000..d9e2a861f --- /dev/null +++ b/internal/ingest/timescaledb.go @@ -0,0 +1,57 @@ +// Package ingest - configureHypertableSettings applies TimescaleDB chunk interval +// and retention policy settings to hypertables at startup. +package ingest + +import ( + "context" + "fmt" + + "github.com/stellar/go-stellar-sdk/support/log" + + "github.com/stellar/wallet-backend/internal/db" +) + +// hypertables lists all TimescaleDB hypertables managed by the ingestion system. +var hypertables = []string{ + "transactions", + "transactions_accounts", + "operations", + "operations_accounts", + "state_changes", +} + +// configureHypertableSettings applies chunk interval and retention policy settings +// to all hypertables. Chunk interval only affects future chunks. Retention policy +// is idempotent: any existing policy is removed before re-adding. +func configureHypertableSettings(ctx context.Context, pool db.ConnectionPool, chunkInterval, retentionPeriod string) error { + for _, table := range hypertables { + if _, err := pool.ExecContext(ctx, + "SELECT set_chunk_time_interval($1::regclass, $2::interval)", + table, chunkInterval, + ); err != nil { + return fmt.Errorf("setting chunk interval on %s: %w", table, err) + } + log.Ctx(ctx).Infof("Set chunk interval %q on %s", chunkInterval, table) + } + + if retentionPeriod != "" { + for _, table := range hypertables { + if _, err := pool.ExecContext(ctx, + "SELECT remove_retention_policy($1::regclass, if_exists => true)", + table, + ); err != nil { + return fmt.Errorf("removing retention policy on %s: %w", table, err) + } + + if _, err := pool.ExecContext(ctx, + "SELECT add_retention_policy($1::regclass, drop_after => $2::interval)", + table, retentionPeriod, + ); err != nil { + return fmt.Errorf("adding retention policy on %s: %w", table, err) + } + log.Ctx(ctx).Infof("Set retention policy %q on %s", retentionPeriod, table) + } + } + + return nil +} diff --git a/internal/ingest/timescaledb_test.go b/internal/ingest/timescaledb_test.go new file mode 100644 index 000000000..db400a853 --- /dev/null +++ b/internal/ingest/timescaledb_test.go @@ -0,0 +1,152 @@ +// Package ingest - tests for configureHypertableSettings verifying chunk interval +// and retention policy configuration against a real TimescaleDB instance. +package ingest + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stellar/wallet-backend/internal/db" + "github.com/stellar/wallet-backend/internal/db/dbtest" +) + +func TestConfigureHypertableSettings(t *testing.T) { + t.Run("chunk_interval", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + + err = configureHypertableSettings(ctx, dbConnectionPool, "7 days", "") + require.NoError(t, err) + + // Verify chunk interval was updated for all hypertables + for _, table := range hypertables { + var intervalSecs float64 + err := dbConnectionPool.GetContext(ctx, &intervalSecs, + `SELECT EXTRACT(EPOCH FROM d.time_interval) + FROM timescaledb_information.dimensions d + WHERE d.hypertable_name = $1 AND d.column_name = 'ledger_created_at'`, + table, + ) + require.NoError(t, err, "querying dimensions for %s", table) + // 7 days in seconds = 7 * 24 * 60 * 60 + assert.Equal(t, float64(7*24*60*60), intervalSecs, "chunk interval for %s", table) + } + }) + + t.Run("retention_policy", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days") + require.NoError(t, err) + + // Verify retention policy was created for all hypertables + for _, table := range hypertables { + var count int + err := dbConnectionPool.GetContext(ctx, &count, + `SELECT COUNT(*) + FROM timescaledb_information.jobs j + WHERE j.proc_name = 'policy_retention' + AND j.hypertable_name = $1`, + table, + ) + require.NoError(t, err, "querying retention policy for %s", table) + assert.Equal(t, 1, count, "expected exactly 1 retention policy for %s", table) + } + }) + + t.Run("no_retention_when_empty", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "") + require.NoError(t, err) + + // Verify no retention policies were created + var count int + err = dbConnectionPool.GetContext(ctx, &count, + `SELECT COUNT(*) + FROM timescaledb_information.jobs + WHERE proc_name = 'policy_retention'`, + ) + require.NoError(t, err) + assert.Equal(t, 0, count, "expected no retention policies when retention period is empty") + }) + + t.Run("retention_policy_idempotent", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + + // Apply retention policy twice with different values to simulate restarts + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days") + require.NoError(t, err) + + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "90 days") + require.NoError(t, err) + + // Verify exactly 1 retention policy per table (not duplicated) + for _, table := range hypertables { + var count int + err := dbConnectionPool.GetContext(ctx, &count, + `SELECT COUNT(*) + FROM timescaledb_information.jobs j + WHERE j.proc_name = 'policy_retention' + AND j.hypertable_name = $1`, + table, + ) + require.NoError(t, err, "querying retention policy for %s", table) + assert.Equal(t, 1, count, "expected exactly 1 retention policy for %s after re-application", table) + } + }) + + t.Run("invalid_chunk_interval", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + + err = configureHypertableSettings(ctx, dbConnectionPool, "not-an-interval", "") + assert.Error(t, err) + assert.Contains(t, err.Error(), "setting chunk interval") + }) + + t.Run("invalid_retention_period", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "not-an-interval") + assert.Error(t, err) + assert.Contains(t, err.Error(), "adding retention policy") + }) +} From 7ef186643964b324c3e463a132d14e81be3ade8c Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 12 Feb 2026 13:28:18 -0500 Subject: [PATCH 41/77] Update ingest.go --- internal/ingest/ingest.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 448886b04..f41848b1d 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -128,7 +128,7 @@ func setupDeps(cfg Configs) (services.IngestService, error) { log.Ctx(ctx).Warnf("Could not disable FK checks (may require superuser privileges): %v", fkErr) // Continue anyway - other optimizations (async commit, work_mem) still apply } else { - log.Ctx(ctx).Info("Backfill session configured: FK checks disabled, async commit enabled, work_mem=256MB") + log.Ctx(ctx).Info("Backfill session configured: FK checks disabled, async commit enabled") } default: dbConnectionPool, err = db.OpenDBConnectionPool(cfg.DatabaseURL) From 648815581b1205e98e87b5b0219ca31e51f9b52e Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 12 Feb 2026 15:11:25 -0500 Subject: [PATCH 42/77] add batch index tracking to logs --- internal/services/ingest_backfill.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/services/ingest_backfill.go b/internal/services/ingest_backfill.go index 1d87ab2d3..0ee09406a 100644 --- a/internal/services/ingest_backfill.go +++ b/internal/services/ingest_backfill.go @@ -340,7 +340,7 @@ func (m *ingestService) processBackfillBatchesParallel(ctx context.Context, mode for i, batch := range batches { group.Submit(func() { - results[i] = m.processSingleBatch(ctx, mode, batch) + results[i] = m.processSingleBatch(ctx, mode, batch, i, len(batches)) }) } @@ -352,7 +352,7 @@ func (m *ingestService) processBackfillBatchesParallel(ctx context.Context, mode } // processSingleBatch processes a single backfill batch with its own ledger backend. -func (m *ingestService) processSingleBatch(ctx context.Context, mode BackfillMode, batch BackfillBatch) BackfillResult { +func (m *ingestService) processSingleBatch(ctx context.Context, mode BackfillMode, batch BackfillBatch, batchIndex, totalBatches int) BackfillResult { start := time.Now() result := BackfillResult{Batch: batch} @@ -387,8 +387,8 @@ func (m *ingestService) processSingleBatch(ctx context.Context, mode BackfillMod } result.Duration = time.Since(start) - log.Ctx(ctx).Infof("Batch [%d - %d] completed: %d ledgers in %v", - batch.StartLedger, batch.EndLedger, result.LedgersCount, result.Duration) + log.Ctx(ctx).Infof("Batch %d/%d [%d - %d] completed: %d ledgers in %v", + batchIndex+1, totalBatches, batch.StartLedger, batch.EndLedger, result.LedgersCount, result.Duration) return result } From 3db78c898fe9c5bdf9a2e30aeda4944f69c0836d Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 13 Feb 2026 09:40:03 -0500 Subject: [PATCH 43/77] Update ingest_backfill.go --- internal/services/ingest_backfill.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/services/ingest_backfill.go b/internal/services/ingest_backfill.go index 0ee09406a..2ad21cd29 100644 --- a/internal/services/ingest_backfill.go +++ b/internal/services/ingest_backfill.go @@ -687,7 +687,7 @@ func (m *ingestService) compressTableChunks(ctx context.Context, table string, s } _, err := m.models.DB.PgxPool().Exec(ctx, - `CALL convert_to_columnstore($1::regclass, if_not_columnstore => true, recompress => true)`, chunk) + `CALL _timescaledb_functions.rebuild_columnstore($1::regclass)`, chunk) if err != nil { log.Ctx(ctx).Warnf("Failed to recompress chunk %s: %v", chunk, err) continue From 7f578ea343e332966f0ec60419adca74720a0f72 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 13 Feb 2026 11:34:35 -0500 Subject: [PATCH 44/77] update auto-vaccum settings for balances tables --- .../2026-01-12.0-trustline_balances.sql | 25 +++++++++++++++++++ .../2026-01-15.0-native_balances.sql | 20 +++++++++++++++ .../migrations/2026-01-16.0-sac-balances.sql | 20 +++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/internal/db/migrations/2026-01-12.0-trustline_balances.sql b/internal/db/migrations/2026-01-12.0-trustline_balances.sql index a55c20b25..358b8bbfa 100644 --- a/internal/db/migrations/2026-01-12.0-trustline_balances.sql +++ b/internal/db/migrations/2026-01-12.0-trustline_balances.sql @@ -3,6 +3,9 @@ -- +migrate Up +-- Storage parameters tuned for heavy UPSERT/DELETE during ledger ingestion. +-- UPSERTs only modify non-indexed columns (balance, trust_limit, liabilities, flags, +-- last_modified_ledger) while PK columns (account_address, asset_id) are never changed. CREATE TABLE trustline_balances ( account_address TEXT NOT NULL, asset_id UUID NOT NULL, @@ -16,6 +19,28 @@ CREATE TABLE trustline_balances ( CONSTRAINT fk_trustline_asset FOREIGN KEY (asset_id) REFERENCES trustline_assets(id) DEFERRABLE INITIALLY DEFERRED +) WITH ( + -- Reserve 20% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. + -- HOT updates rewrite the row in-place on the same page without creating dead tuples + -- or new index entries, since no indexed column is modified during UPSERTs. + fillfactor = 80, + -- Trigger vacuum when 2% of rows are dead (default 20%). For a 500K-row table, + -- this means vacuum starts at ~10K dead rows instead of waiting for 100K. + autovacuum_vacuum_scale_factor = 0.02, + -- Base dead-row count added to (scale_factor * total_rows). Default is fine here + -- since the scale factor already keeps the threshold low. + autovacuum_vacuum_threshold = 50, + -- Refresh planner statistics at 1% change (default 10%). Balances shift every ledger, + -- so stale stats can cause bad query plans (e.g. on the GetByAccount JOIN). + autovacuum_analyze_scale_factor = 0.01, + autovacuum_analyze_threshold = 50, + -- No sleep between vacuum page-processing cycles (default 2ms). Per-table setting, + -- so only workers on this table run full-speed; other tables are unaffected. + autovacuum_vacuum_cost_delay = 0, + -- 5x the default page-processing budget per cycle (default 200). Combined with + -- cost_delay=0, vacuum finishes quickly. Per-table cost settings exempt this worker + -- from global cost balancing, so other tables' vacuum workers keep their full budget. + autovacuum_vacuum_cost_limit = 1000 ); -- +migrate Down diff --git a/internal/db/migrations/2026-01-15.0-native_balances.sql b/internal/db/migrations/2026-01-15.0-native_balances.sql index 698cece48..eab6566a8 100644 --- a/internal/db/migrations/2026-01-15.0-native_balances.sql +++ b/internal/db/migrations/2026-01-15.0-native_balances.sql @@ -2,6 +2,18 @@ -- Table: native_balances -- Stores native XLM balance data for accounts during ingestion. +-- +-- Storage parameters tuned for heavy UPSERT/DELETE during ledger ingestion: +-- fillfactor=80: Reserves 20% free space per page to enable HOT (Heap-Only Tuple) updates. +-- Since UPSERTs only modify non-indexed columns (balance, minimum_balance, liabilities, +-- last_modified_ledger) while the PK column (account_address) is never changed, +-- PostgreSQL can update rows in-place without creating dead tuples or new index entries. +-- autovacuum_vacuum_scale_factor=0.02: Triggers vacuum at 2% dead rows instead of the default 20%. +-- autovacuum_analyze_scale_factor=0.01: Keeps planner statistics fresh as data changes every ledger. +-- autovacuum_vacuum_cost_delay=0: Runs vacuum at full speed (no throttling). Per-table setting +-- so it only affects workers on this table, not the whole cluster. +-- autovacuum_vacuum_cost_limit=1000: 5x the default budget. Workers with per-table cost settings +-- are exempt from global cost balancing, so this does not starve other tables. CREATE TABLE native_balances ( account_address TEXT PRIMARY KEY, balance BIGINT NOT NULL DEFAULT 0, @@ -9,6 +21,14 @@ CREATE TABLE native_balances ( buying_liabilities BIGINT NOT NULL DEFAULT 0, selling_liabilities BIGINT NOT NULL DEFAULT 0, last_modified_ledger BIGINT NOT NULL DEFAULT 0 +) WITH ( + fillfactor = 80, + autovacuum_vacuum_scale_factor = 0.02, + autovacuum_vacuum_threshold = 50, + autovacuum_analyze_scale_factor = 0.01, + autovacuum_analyze_threshold = 50, + autovacuum_vacuum_cost_delay = 0, + autovacuum_vacuum_cost_limit = 1000 ); -- +migrate Down diff --git a/internal/db/migrations/2026-01-16.0-sac-balances.sql b/internal/db/migrations/2026-01-16.0-sac-balances.sql index 40c03ca8a..133b66a9a 100644 --- a/internal/db/migrations/2026-01-16.0-sac-balances.sql +++ b/internal/db/migrations/2026-01-16.0-sac-balances.sql @@ -3,6 +3,18 @@ -- Table: sac_balances -- Stores SAC (Stellar Asset Contract) balance data for contract addresses (C...) during ingestion. -- Classic Stellar accounts (G...) have SAC balances in their trustlines, so only contract holders are stored here. +-- +-- Storage parameters tuned for heavy UPSERT/DELETE during ledger ingestion: +-- fillfactor=80: Reserves 20% free space per page to enable HOT (Heap-Only Tuple) updates. +-- Since UPSERTs only modify non-indexed columns (balance, is_authorized, is_clawback_enabled, +-- last_modified_ledger) while PK columns (account_address, contract_id) are never changed, +-- PostgreSQL can update rows in-place without creating dead tuples or new index entries. +-- autovacuum_vacuum_scale_factor=0.02: Triggers vacuum at 2% dead rows instead of the default 20%. +-- autovacuum_analyze_scale_factor=0.01: Keeps planner statistics fresh as data changes every ledger. +-- autovacuum_vacuum_cost_delay=0: Runs vacuum at full speed (no throttling). Per-table setting +-- so it only affects workers on this table, not the whole cluster. +-- autovacuum_vacuum_cost_limit=1000: 5x the default budget. Workers with per-table cost settings +-- are exempt from global cost balancing, so this does not starve other tables. CREATE TABLE sac_balances ( account_address TEXT NOT NULL, contract_id UUID NOT NULL, @@ -14,6 +26,14 @@ CREATE TABLE sac_balances ( CONSTRAINT fk_contract_token FOREIGN KEY (contract_id) REFERENCES contract_tokens(id) DEFERRABLE INITIALLY DEFERRED +) WITH ( + fillfactor = 80, + autovacuum_vacuum_scale_factor = 0.02, + autovacuum_vacuum_threshold = 50, + autovacuum_analyze_scale_factor = 0.01, + autovacuum_analyze_threshold = 50, + autovacuum_vacuum_cost_delay = 0, + autovacuum_vacuum_cost_limit = 1000 ); -- +migrate Down From cefebf5efac90c3f51f71762eecbf1d2095cadc1 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 13 Feb 2026 11:35:04 -0500 Subject: [PATCH 45/77] add explanations for the values --- .../2026-01-15.0-native_balances.sql | 29 +++++++++++-------- .../migrations/2026-01-16.0-sac-balances.sql | 29 +++++++++++-------- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/internal/db/migrations/2026-01-15.0-native_balances.sql b/internal/db/migrations/2026-01-15.0-native_balances.sql index eab6566a8..cc393e040 100644 --- a/internal/db/migrations/2026-01-15.0-native_balances.sql +++ b/internal/db/migrations/2026-01-15.0-native_balances.sql @@ -2,18 +2,9 @@ -- Table: native_balances -- Stores native XLM balance data for accounts during ingestion. --- --- Storage parameters tuned for heavy UPSERT/DELETE during ledger ingestion: --- fillfactor=80: Reserves 20% free space per page to enable HOT (Heap-Only Tuple) updates. --- Since UPSERTs only modify non-indexed columns (balance, minimum_balance, liabilities, --- last_modified_ledger) while the PK column (account_address) is never changed, --- PostgreSQL can update rows in-place without creating dead tuples or new index entries. --- autovacuum_vacuum_scale_factor=0.02: Triggers vacuum at 2% dead rows instead of the default 20%. --- autovacuum_analyze_scale_factor=0.01: Keeps planner statistics fresh as data changes every ledger. --- autovacuum_vacuum_cost_delay=0: Runs vacuum at full speed (no throttling). Per-table setting --- so it only affects workers on this table, not the whole cluster. --- autovacuum_vacuum_cost_limit=1000: 5x the default budget. Workers with per-table cost settings --- are exempt from global cost balancing, so this does not starve other tables. +-- Storage parameters tuned for heavy UPSERT/DELETE during ledger ingestion. +-- UPSERTs only modify non-indexed columns (balance, minimum_balance, liabilities, +-- last_modified_ledger) while the PK column (account_address) is never changed. CREATE TABLE native_balances ( account_address TEXT PRIMARY KEY, balance BIGINT NOT NULL DEFAULT 0, @@ -22,12 +13,26 @@ CREATE TABLE native_balances ( selling_liabilities BIGINT NOT NULL DEFAULT 0, last_modified_ledger BIGINT NOT NULL DEFAULT 0 ) WITH ( + -- Reserve 20% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. + -- HOT updates rewrite the row in-place on the same page without creating dead tuples + -- or new index entries, since no indexed column is modified during UPSERTs. fillfactor = 80, + -- Trigger vacuum when 2% of rows are dead (default 20%). For a 500K-row table, + -- this means vacuum starts at ~10K dead rows instead of waiting for 100K. autovacuum_vacuum_scale_factor = 0.02, + -- Base dead-row count added to (scale_factor * total_rows). Default is fine here + -- since the scale factor already keeps the threshold low. autovacuum_vacuum_threshold = 50, + -- Refresh planner statistics at 1% change (default 10%). Balances shift every ledger, + -- so stale stats can cause bad query plans. autovacuum_analyze_scale_factor = 0.01, autovacuum_analyze_threshold = 50, + -- No sleep between vacuum page-processing cycles (default 2ms). Per-table setting, + -- so only workers on this table run full-speed; other tables are unaffected. autovacuum_vacuum_cost_delay = 0, + -- 5x the default page-processing budget per cycle (default 200). Combined with + -- cost_delay=0, vacuum finishes quickly. Per-table cost settings exempt this worker + -- from global cost balancing, so other tables' vacuum workers keep their full budget. autovacuum_vacuum_cost_limit = 1000 ); diff --git a/internal/db/migrations/2026-01-16.0-sac-balances.sql b/internal/db/migrations/2026-01-16.0-sac-balances.sql index 133b66a9a..b3bc5f99a 100644 --- a/internal/db/migrations/2026-01-16.0-sac-balances.sql +++ b/internal/db/migrations/2026-01-16.0-sac-balances.sql @@ -3,18 +3,9 @@ -- Table: sac_balances -- Stores SAC (Stellar Asset Contract) balance data for contract addresses (C...) during ingestion. -- Classic Stellar accounts (G...) have SAC balances in their trustlines, so only contract holders are stored here. --- --- Storage parameters tuned for heavy UPSERT/DELETE during ledger ingestion: --- fillfactor=80: Reserves 20% free space per page to enable HOT (Heap-Only Tuple) updates. --- Since UPSERTs only modify non-indexed columns (balance, is_authorized, is_clawback_enabled, --- last_modified_ledger) while PK columns (account_address, contract_id) are never changed, --- PostgreSQL can update rows in-place without creating dead tuples or new index entries. --- autovacuum_vacuum_scale_factor=0.02: Triggers vacuum at 2% dead rows instead of the default 20%. --- autovacuum_analyze_scale_factor=0.01: Keeps planner statistics fresh as data changes every ledger. --- autovacuum_vacuum_cost_delay=0: Runs vacuum at full speed (no throttling). Per-table setting --- so it only affects workers on this table, not the whole cluster. --- autovacuum_vacuum_cost_limit=1000: 5x the default budget. Workers with per-table cost settings --- are exempt from global cost balancing, so this does not starve other tables. +-- Storage parameters tuned for heavy UPSERT/DELETE during ledger ingestion. +-- UPSERTs only modify non-indexed columns (balance, is_authorized, is_clawback_enabled, +-- last_modified_ledger) while PK columns (account_address, contract_id) are never changed. CREATE TABLE sac_balances ( account_address TEXT NOT NULL, contract_id UUID NOT NULL, @@ -27,12 +18,26 @@ CREATE TABLE sac_balances ( FOREIGN KEY (contract_id) REFERENCES contract_tokens(id) DEFERRABLE INITIALLY DEFERRED ) WITH ( + -- Reserve 20% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. + -- HOT updates rewrite the row in-place on the same page without creating dead tuples + -- or new index entries, since no indexed column is modified during UPSERTs. fillfactor = 80, + -- Trigger vacuum when 2% of rows are dead (default 20%). For a 500K-row table, + -- this means vacuum starts at ~10K dead rows instead of waiting for 100K. autovacuum_vacuum_scale_factor = 0.02, + -- Base dead-row count added to (scale_factor * total_rows). Default is fine here + -- since the scale factor already keeps the threshold low. autovacuum_vacuum_threshold = 50, + -- Refresh planner statistics at 1% change (default 10%). Balances shift every ledger, + -- so stale stats can cause bad query plans. autovacuum_analyze_scale_factor = 0.01, autovacuum_analyze_threshold = 50, + -- No sleep between vacuum page-processing cycles (default 2ms). Per-table setting, + -- so only workers on this table run full-speed; other tables are unaffected. autovacuum_vacuum_cost_delay = 0, + -- 5x the default page-processing budget per cycle (default 200). Combined with + -- cost_delay=0, vacuum finishes quickly. Per-table cost settings exempt this worker + -- from global cost balancing, so other tables' vacuum workers keep their full budget. autovacuum_vacuum_cost_limit = 1000 ); From 22d6f33915178157e2af0b3c5e847bd2b3cd9f2e Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 13 Feb 2026 11:58:54 -0500 Subject: [PATCH 46/77] Disable FK checks during checkpoint population --- internal/data/native_balances.go | 14 ++++++-------- internal/services/token_ingestion.go | 11 ++++++++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/internal/data/native_balances.go b/internal/data/native_balances.go index 32b480087..a1b5091c7 100644 --- a/internal/data/native_balances.go +++ b/internal/data/native_balances.go @@ -129,23 +129,21 @@ func (m *NativeBalanceModel) BatchCopy(ctx context.Context, dbTx pgx.Tx, balance start := time.Now() - rows := make([][]any, len(balances)) - for i, nb := range balances { - rows[i] = []any{nb.AccountAddress, nb.Balance, nb.MinimumBalance, nb.BuyingLiabilities, nb.SellingLiabilities, nb.LedgerNumber} - } - copyCount, err := dbTx.CopyFrom( ctx, pgx.Identifier{"native_balances"}, []string{"account_address", "balance", "minimum_balance", "buying_liabilities", "selling_liabilities", "last_modified_ledger"}, - pgx.CopyFromRows(rows), + pgx.CopyFromSlice(len(balances), func(i int) ([]any, error) { + nb := balances[i] + return []any{nb.AccountAddress, nb.Balance, nb.MinimumBalance, nb.BuyingLiabilities, nb.SellingLiabilities, nb.LedgerNumber}, nil + }), ) if err != nil { return fmt.Errorf("bulk inserting native balances via COPY: %w", err) } - if int(copyCount) != len(rows) { - return fmt.Errorf("expected %d rows copied, got %d", len(rows), copyCount) + if int(copyCount) != len(balances) { + return fmt.Errorf("expected %d rows copied, got %d", len(balances), copyCount) } m.MetricsService.ObserveDBQueryDuration("BatchCopy", "native_balances", time.Since(start).Seconds()) diff --git a/internal/services/token_ingestion.go b/internal/services/token_ingestion.go index 1402beea5..b868ee591 100644 --- a/internal/services/token_ingestion.go +++ b/internal/services/token_ingestion.go @@ -28,7 +28,7 @@ import ( const ( // FlushBatchSize is the number of entries to buffer before flushing to DB. - flushBatchSize = 100_000 + flushBatchSize = 250_000 ) // checkpointData holds all data collected from processing a checkpoint ledger. @@ -268,6 +268,15 @@ func (s *tokenIngestionService) PopulateAccountTokens(ctx context.Context, check return fmt.Errorf("setting synchronous_commit=off: %w", txErr) } + // Disable FK constraint checking for this transaction only. Data integrity is + // guaranteed by the code: trustline assets are collected in uniqueAssets and SAC + // contracts get minimal stubs created when balance entries are encountered (even + // when contract instance entries are missing from the checkpoint). All parent rows + // are inserted via storeTokensInDB before commit. Requires superuser privileges. + if _, txErr := dbTx.Exec(ctx, "SET LOCAL session_replication_role = 'replica'"); txErr != nil { + log.Ctx(ctx).Warnf("Could not disable FK checks for checkpoint population (may require superuser): %v", txErr) + } + // Stream trustlines and collect contracts from checkpoint cpData, txErr := s.streamCheckpointData(ctx, dbTx, reader, checkpointLedger) if txErr != nil { From 4c7afbac4d6557a9ad6b93307e0a800e20943e16 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 13 Feb 2026 12:00:50 -0500 Subject: [PATCH 47/77] Update ingest_live.go --- internal/services/ingest_live.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/services/ingest_live.go b/internal/services/ingest_live.go index 4b3fe2077..3b03be1f8 100644 --- a/internal/services/ingest_live.go +++ b/internal/services/ingest_live.go @@ -124,6 +124,8 @@ func (m *ingestService) startLiveIngestion(ctx context.Context) error { if err != nil { return fmt.Errorf("populating account tokens and initializing cursors: %w", err) } + m.metricsService.SetLatestLedgerIngested(float64(startLedger)) + m.metricsService.SetOldestLedgerIngested(float64(startLedger)) } else { // If we already have data in the DB, we will do an optimized catchup by parallely backfilling the ledgers. health, err := m.rpcService.GetHealth() From 88586d1cf7494295af71436152e51980f93191ac Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 13 Feb 2026 15:13:54 -0500 Subject: [PATCH 48/77] Update ingest_live.go --- internal/services/ingest_live.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/services/ingest_live.go b/internal/services/ingest_live.go index 3b03be1f8..6b94243e3 100644 --- a/internal/services/ingest_live.go +++ b/internal/services/ingest_live.go @@ -127,6 +127,14 @@ func (m *ingestService) startLiveIngestion(ctx context.Context) error { m.metricsService.SetLatestLedgerIngested(float64(startLedger)) m.metricsService.SetOldestLedgerIngested(float64(startLedger)) } else { + // Initialize metrics from DB state so Prometheus reflects backfill progress after restart + oldestIngestedLedger, oldestErr := m.models.IngestStore.Get(ctx, m.oldestLedgerCursorName) + if oldestErr != nil { + return fmt.Errorf("getting oldest ledger cursor: %w", oldestErr) + } + m.metricsService.SetOldestLedgerIngested(float64(oldestIngestedLedger)) + m.metricsService.SetLatestLedgerIngested(float64(latestIngestedLedger)) + // If we already have data in the DB, we will do an optimized catchup by parallely backfilling the ledgers. health, err := m.rpcService.GetHealth() if err != nil { From 5896323aa69f947783358cac277cfdbfa63b84a0 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 13 Feb 2026 15:58:44 -0500 Subject: [PATCH 49/77] Update ingest_live.go --- internal/services/ingest_live.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/services/ingest_live.go b/internal/services/ingest_live.go index 6b94243e3..d3c608d80 100644 --- a/internal/services/ingest_live.go +++ b/internal/services/ingest_live.go @@ -20,6 +20,7 @@ import ( const ( maxIngestProcessedDataRetries = 5 maxIngestProcessedDataRetryBackoff = 10 * time.Second + oldestLedgerSyncInterval = 100 ) // PersistLedgerData persists processed ledger data to the database in a single atomic transaction. @@ -204,6 +205,12 @@ func (m *ingestService) ingestLiveLedgers(ctx context.Context, startLedger uint3 m.metricsService.IncIngestionOperationsProcessed(numOperationProcessed) m.metricsService.IncIngestionLedgersProcessed(1) m.metricsService.SetLatestLedgerIngested(float64(currentLedger)) + // Periodically sync oldest ledger metric from DB (picks up changes from backfill jobs) + if currentLedger%oldestLedgerSyncInterval == 0 { + if oldest, syncErr := m.models.IngestStore.Get(ctx, m.oldestLedgerCursorName); syncErr == nil { + m.metricsService.SetOldestLedgerIngested(float64(oldest)) + } + } log.Ctx(ctx).Infof("Ingested ledger %d in %.4fs", currentLedger, totalIngestionDuration) currentLedger++ From b4e4abe3056f997abcffa1742705f1b0c4ca103d Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 13 Feb 2026 16:11:00 -0500 Subject: [PATCH 50/77] Update ingest_backfill.go --- internal/services/ingest_backfill.go | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/internal/services/ingest_backfill.go b/internal/services/ingest_backfill.go index 2ad21cd29..4ec697f9b 100644 --- a/internal/services/ingest_backfill.go +++ b/internal/services/ingest_backfill.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "maps" - "sync" "time" "github.com/google/uuid" @@ -193,7 +192,7 @@ func (m *ingestService) startBackfilling(ctx context.Context, startLedger, endLe } } if !minTime.IsZero() { - m.compressBackfilledChunks(ctx, minTime, maxTime) + m.recompressBackfilledChunks(ctx, minTime, maxTime) } } @@ -624,26 +623,19 @@ func (m *ingestService) processBatchChanges( return nil } -// compressBackfilledChunks recompresses already-compressed chunks overlapping the backfill range. +// recompressBackfilledChunks recompresses already-compressed chunks overlapping the backfill range. // Direct compress produces compressed chunks during COPY; recompression optimizes compression ratios. -// Tables are compressed concurrently; chunks within each table stay sequential to avoid OOM. +// Tables are compressed sequentially to avoid CPU spikes; chunks within each table also stay sequential to avoid OOM. // Skips chunks where range_end >= NOW() to avoid compressing active live ingestion chunks. -func (m *ingestService) compressBackfilledChunks(ctx context.Context, startTime, endTime time.Time) { +func (m *ingestService) recompressBackfilledChunks(ctx context.Context, startTime, endTime time.Time) { tables := []string{"transactions", "transactions_accounts", "operations", "operations_accounts", "state_changes"} - var wg sync.WaitGroup tableCounts := make([]int, len(tables)) for i, table := range tables { - wg.Add(1) - go func() { - defer wg.Done() - tableCounts[i] = m.compressTableChunks(ctx, table, startTime, endTime) - }() + tableCounts[i] = m.compressTableChunks(ctx, table, startTime, endTime) } - wg.Wait() - totalCompressed := 0 for _, count := range tableCounts { totalCompressed += count From 0183370b99897618d9f3ebb81212aa519d5661cf Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 13 Feb 2026 17:11:16 -0500 Subject: [PATCH 51/77] fix ledger_created_at bug --- internal/indexer/processors/participants.go | 1 + internal/indexer/processors/participants_test.go | 1 + 2 files changed, 2 insertions(+) diff --git a/internal/indexer/processors/participants.go b/internal/indexer/processors/participants.go index c94626ff3..fc293d98a 100644 --- a/internal/indexer/processors/participants.go +++ b/internal/indexer/processors/participants.go @@ -134,6 +134,7 @@ func (p *ParticipantsProcessor) GetOperationsParticipants(transaction ingest.Led Operation: xdrOp, LedgerSequence: ledgerSequence, Network: p.networkPassphrase, + LedgerClosed: transaction.Ledger.ClosedAt(), } opID := op.ID() diff --git a/internal/indexer/processors/participants_test.go b/internal/indexer/processors/participants_test.go index 3d6e63beb..b535df0ac 100644 --- a/internal/indexer/processors/participants_test.go +++ b/internal/indexer/processors/participants_test.go @@ -615,6 +615,7 @@ func TestParticipantsProcessor_GetOperationsParticipants(t *testing.T) { Network: network.TestNetworkPassphrase, Transaction: ingestTx, LedgerSequence: 4873, + LedgerClosed: ingestTx.Ledger.ClosedAt(), } assert.Equal(t, tc.wantParticipantsFn(t, opWrapper), gotParticipants) }) From 5ccac7f9995a0130416e885d9eab99f38eb85288 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 13 Feb 2026 17:49:38 -0500 Subject: [PATCH 52/77] update oldest ledger using timescaledb job --- internal/ingest/ingest.go | 2 +- internal/ingest/timescaledb.go | 43 +++++++++++++- internal/ingest/timescaledb_test.go | 87 ++++++++++++++++++++++++++--- 3 files changed, 122 insertions(+), 10 deletions(-) diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index f41848b1d..b205fb5ef 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -141,7 +141,7 @@ func setupDeps(cfg Configs) (services.IngestService, error) { return nil, fmt.Errorf("getting sqlx db: %w", err) } - if err := configureHypertableSettings(ctx, dbConnectionPool, cfg.ChunkInterval, cfg.RetentionPeriod); err != nil { + if err := configureHypertableSettings(ctx, dbConnectionPool, cfg.ChunkInterval, cfg.RetentionPeriod, cfg.OldestLedgerCursorName); err != nil { return nil, fmt.Errorf("configuring hypertable settings: %w", err) } diff --git a/internal/ingest/timescaledb.go b/internal/ingest/timescaledb.go index d9e2a861f..65c371b5c 100644 --- a/internal/ingest/timescaledb.go +++ b/internal/ingest/timescaledb.go @@ -22,8 +22,10 @@ var hypertables = []string{ // configureHypertableSettings applies chunk interval and retention policy settings // to all hypertables. Chunk interval only affects future chunks. Retention policy -// is idempotent: any existing policy is removed before re-adding. -func configureHypertableSettings(ctx context.Context, pool db.ConnectionPool, chunkInterval, retentionPeriod string) error { +// is idempotent: any existing policy is removed before re-adding. When retention +// is enabled, a reconciliation job keeps oldest_ingest_ledger in sync with the +// actual minimum ledger remaining after chunk drops. +func configureHypertableSettings(ctx context.Context, pool db.ConnectionPool, chunkInterval, retentionPeriod, oldestCursorName string) error { for _, table := range hypertables { if _, err := pool.ExecContext(ctx, "SELECT set_chunk_time_interval($1::regclass, $2::interval)", @@ -51,6 +53,43 @@ func configureHypertableSettings(ctx context.Context, pool db.ConnectionPool, ch } log.Ctx(ctx).Infof("Set retention policy %q on %s", retentionPeriod, table) } + + // Reconciliation job: keeps oldestCursorName in sync after retention drops chunks. + // Remove any existing job first (idempotent re-registration on every restart). + if _, err := pool.ExecContext(ctx, + "SELECT delete_job(job_id) FROM timescaledb_information.jobs WHERE proc_name = 'reconcile_oldest_cursor'", + ); err != nil { + return fmt.Errorf("removing existing reconciliation job: %w", err) + } + + // Create or replace the PL/pgSQL function that advances the cursor. + if _, err := pool.ExecContext(ctx, ` + CREATE OR REPLACE FUNCTION reconcile_oldest_cursor(job_id INT, config JSONB) + RETURNS VOID LANGUAGE plpgsql AS $$ + DECLARE + actual_min INTEGER; + stored INTEGER; + BEGIN + SELECT ledger_number INTO actual_min FROM transactions + ORDER BY ledger_created_at ASC, to_id ASC LIMIT 1; + IF actual_min IS NULL THEN RETURN; END IF; + SELECT value::integer INTO stored FROM ingest_store WHERE key = config->>'cursor_name'; + IF stored IS NULL OR actual_min <= stored THEN RETURN; END IF; + UPDATE ingest_store SET value = actual_min::text WHERE key = config->>'cursor_name'; + RAISE LOG 'reconcile_oldest_cursor: advanced % from % to %', config->>'cursor_name', stored, actual_min; + END $$; + `); err != nil { + return fmt.Errorf("creating reconcile_oldest_cursor function: %w", err) + } + + // Schedule the reconciliation job with the same cadence as the chunk interval. + if _, err := pool.ExecContext(ctx, + "SELECT add_job('reconcile_oldest_cursor', $1::interval, config => $2::jsonb)", + chunkInterval, fmt.Sprintf(`{"cursor_name":"%s"}`, oldestCursorName), + ); err != nil { + return fmt.Errorf("scheduling reconciliation job: %w", err) + } + log.Ctx(ctx).Infof("Scheduled reconcile_oldest_cursor job every %s for cursor %q", chunkInterval, oldestCursorName) } return nil diff --git a/internal/ingest/timescaledb_test.go b/internal/ingest/timescaledb_test.go index db400a853..4dc165d52 100644 --- a/internal/ingest/timescaledb_test.go +++ b/internal/ingest/timescaledb_test.go @@ -23,7 +23,7 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "7 days", "") + err = configureHypertableSettings(ctx, dbConnectionPool, "7 days", "", "oldest_ledger_cursor") require.NoError(t, err) // Verify chunk interval was updated for all hypertables @@ -50,7 +50,7 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor") require.NoError(t, err) // Verify retention policy was created for all hypertables @@ -77,7 +77,7 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "", "oldest_ledger_cursor") require.NoError(t, err) // Verify no retention policies were created @@ -101,10 +101,10 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() // Apply retention policy twice with different values to simulate restarts - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor") require.NoError(t, err) - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "90 days") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "90 days", "oldest_ledger_cursor") require.NoError(t, err) // Verify exactly 1 retention policy per table (not duplicated) @@ -122,6 +122,79 @@ func TestConfigureHypertableSettings(t *testing.T) { } }) + t.Run("reconciliation_job_created", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor") + require.NoError(t, err) + + // Verify reconciliation job was created + var count int + err = dbConnectionPool.GetContext(ctx, &count, + `SELECT COUNT(*) + FROM timescaledb_information.jobs + WHERE proc_name = 'reconcile_oldest_cursor'`, + ) + require.NoError(t, err) + assert.Equal(t, 1, count, "expected exactly 1 reconciliation job") + }) + + t.Run("reconciliation_job_idempotent", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + + // Apply twice to simulate restarts + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor") + require.NoError(t, err) + + err = configureHypertableSettings(ctx, dbConnectionPool, "7 days", "90 days", "oldest_ledger_cursor") + require.NoError(t, err) + + // Verify exactly 1 reconciliation job (not duplicated) + var count int + err = dbConnectionPool.GetContext(ctx, &count, + `SELECT COUNT(*) + FROM timescaledb_information.jobs + WHERE proc_name = 'reconcile_oldest_cursor'`, + ) + require.NoError(t, err) + assert.Equal(t, 1, count, "expected exactly 1 reconciliation job after re-application") + }) + + t.Run("no_reconciliation_job_without_retention", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "", "oldest_ledger_cursor") + require.NoError(t, err) + + // Verify no reconciliation job was created + var count int + err = dbConnectionPool.GetContext(ctx, &count, + `SELECT COUNT(*) + FROM timescaledb_information.jobs + WHERE proc_name = 'reconcile_oldest_cursor'`, + ) + require.NoError(t, err) + assert.Equal(t, 0, count, "expected no reconciliation job when retention is disabled") + }) + t.Run("invalid_chunk_interval", func(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -131,7 +204,7 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "not-an-interval", "") + err = configureHypertableSettings(ctx, dbConnectionPool, "not-an-interval", "", "oldest_ledger_cursor") assert.Error(t, err) assert.Contains(t, err.Error(), "setting chunk interval") }) @@ -145,7 +218,7 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "not-an-interval") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "not-an-interval", "oldest_ledger_cursor") assert.Error(t, err) assert.Contains(t, err.Error(), "adding retention policy") }) From 89b11ae17c8ecbe2c6134422833a79af4fd613bf Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Sat, 14 Feb 2026 19:45:45 -0500 Subject: [PATCH 53/77] Use ledger_created_at in the cursor --- internal/data/operations.go | 100 +++++++++++++----- internal/data/operations_test.go | 12 +-- internal/data/query_utils.go | 79 -------------- internal/data/statechanges.go | 51 ++++----- internal/data/transactions.go | 92 ++++++++++++---- internal/data/transactions_test.go | 16 +-- internal/indexer/types/types.go | 19 +++- .../graphql/resolvers/account.resolvers.go | 20 ++-- .../graphql/resolvers/operation.resolvers.go | 4 +- .../graphql/resolvers/queries.resolvers.go | 20 ++-- .../serve/graphql/resolvers/test_utils.go | 9 +- .../resolvers/transaction.resolvers.go | 8 +- internal/serve/graphql/resolvers/utils.go | 80 ++++++++++++-- 13 files changed, 301 insertions(+), 209 deletions(-) diff --git a/internal/data/operations.go b/internal/data/operations.go index ad4ddda34..b812f59f1 100644 --- a/internal/data/operations.go +++ b/internal/data/operations.go @@ -38,23 +38,23 @@ func (m *OperationModel) GetByID(ctx context.Context, id int64, columns string) return &operation, nil } -func (m *OperationModel) GetAll(ctx context.Context, columns string, limit *int32, cursor *int64, sortOrder SortOrder) ([]*types.OperationWithCursor, error) { +func (m *OperationModel) GetAll(ctx context.Context, columns string, limit *int32, cursor *types.CompositeCursor, sortOrder SortOrder) ([]*types.OperationWithCursor, error) { columns = prepareColumnsWithID(columns, types.Operation{}, "", "id") queryBuilder := strings.Builder{} - queryBuilder.WriteString(fmt.Sprintf(`SELECT %s, id as cursor FROM operations`, columns)) + queryBuilder.WriteString(fmt.Sprintf(`SELECT %s, ledger_created_at as "cursor.cursor_ledger_created_at", id as "cursor.cursor_id" FROM operations`, columns)) if cursor != nil { if sortOrder == DESC { - queryBuilder.WriteString(fmt.Sprintf(" WHERE id < %d", *cursor)) + queryBuilder.WriteString(fmt.Sprintf(` WHERE (ledger_created_at, id) < ('%s', %d)`, cursor.LedgerCreatedAt.Format(time.RFC3339Nano), cursor.ID)) } else { - queryBuilder.WriteString(fmt.Sprintf(" WHERE id > %d", *cursor)) + queryBuilder.WriteString(fmt.Sprintf(` WHERE (ledger_created_at, id) > ('%s', %d)`, cursor.LedgerCreatedAt.Format(time.RFC3339Nano), cursor.ID)) } } if sortOrder == DESC { - queryBuilder.WriteString(" ORDER BY id DESC") + queryBuilder.WriteString(" ORDER BY ledger_created_at DESC, id DESC") } else { - queryBuilder.WriteString(" ORDER BY id ASC") + queryBuilder.WriteString(" ORDER BY ledger_created_at ASC, id ASC") } if limit != nil { @@ -62,7 +62,7 @@ func (m *OperationModel) GetAll(ctx context.Context, columns string, limit *int3 } query := queryBuilder.String() if sortOrder == DESC { - query = fmt.Sprintf(`SELECT * FROM (%s) AS operations ORDER BY cursor ASC`, query) + query = fmt.Sprintf(`SELECT * FROM (%s) AS operations ORDER BY operations."cursor.cursor_ledger_created_at" ASC, operations."cursor.cursor_id" ASC`, query) } var operations []*types.OperationWithCursor @@ -131,7 +131,7 @@ func (m *OperationModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, col JOIN inputs i ON o.id > i.to_id AND o.id < i.to_id + 4096 ) - SELECT %s, id as cursor FROM ranked_operations_per_to_id + SELECT %s, ledger_created_at as "cursor.cursor_ledger_created_at", id as "cursor.cursor_id" FROM ranked_operations_per_to_id ` queryBuilder.WriteString(fmt.Sprintf(query, sortOrder, columns)) if limit != nil { @@ -139,7 +139,7 @@ func (m *OperationModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, col } query = queryBuilder.String() if sortOrder == DESC { - query = fmt.Sprintf(`SELECT * FROM (%s) AS operations ORDER BY cursor ASC`, query) + query = fmt.Sprintf(`SELECT * FROM (%s) AS operations ORDER BY operations."cursor.cursor_ledger_created_at" ASC, operations."cursor.cursor_id" ASC`, query) } var operations []*types.OperationWithCursor @@ -162,7 +162,7 @@ func (m *OperationModel) BatchGetByToID(ctx context.Context, toID int64, columns columns = prepareColumnsWithID(columns, types.Operation{}, "", "id") queryBuilder := strings.Builder{} // Operations for a tx_to_id are in range (tx_to_id, tx_to_id + 4096) based on TOID encoding. - queryBuilder.WriteString(fmt.Sprintf(`SELECT %s, id as cursor FROM operations WHERE id > $1 AND id < $1 + 4096`, columns)) + queryBuilder.WriteString(fmt.Sprintf(`SELECT %s, ledger_created_at as "cursor.cursor_ledger_created_at", id as "cursor.cursor_id" FROM operations WHERE id > $1 AND id < $1 + 4096`, columns)) args := []interface{}{toID} argIndex := 2 @@ -190,7 +190,7 @@ func (m *OperationModel) BatchGetByToID(ctx context.Context, toID int64, columns query := queryBuilder.String() if sortOrder == DESC { - query = fmt.Sprintf(`SELECT * FROM (%s) AS operations ORDER BY cursor ASC`, query) + query = fmt.Sprintf(`SELECT * FROM (%s) AS operations ORDER BY operations."cursor.cursor_ledger_created_at" ASC, operations."cursor.cursor_id" ASC`, query) } var operations []*types.OperationWithCursor @@ -207,21 +207,71 @@ func (m *OperationModel) BatchGetByToID(ctx context.Context, toID int64, columns } // BatchGetByAccountAddress gets the operations that are associated with a single account address. -func (m *OperationModel) BatchGetByAccountAddress(ctx context.Context, accountAddress string, columns string, limit *int32, cursor *int64, orderBy SortOrder) ([]*types.OperationWithCursor, error) { - columns = prepareColumnsWithID(columns, types.Operation{}, "operations", "id") +// Uses a MATERIALIZED CTE + LATERAL join pattern to allow TimescaleDB ChunkAppend optimization +// on the operations_accounts hypertable by ordering on ledger_created_at first. +func (m *OperationModel) BatchGetByAccountAddress(ctx context.Context, accountAddress string, columns string, limit *int32, cursor *types.CompositeCursor, orderBy SortOrder) ([]*types.OperationWithCursor, error) { + columns = prepareColumnsWithID(columns, types.Operation{}, "o", "id") + + var queryBuilder strings.Builder + args := []interface{}{types.AddressBytea(accountAddress)} + argIndex := 2 + + // MATERIALIZED CTE scans operations_accounts with ledger_created_at leading the ORDER BY, + // enabling TimescaleDB ChunkAppend on the hypertable. + queryBuilder.WriteString(` + WITH account_ops AS MATERIALIZED ( + SELECT operation_id, ledger_created_at + FROM operations_accounts + WHERE account_id = $1`) + + // Add cursor-based pagination on the CTE + if cursor != nil { + if orderBy == DESC { + queryBuilder.WriteString(fmt.Sprintf(` + AND (ledger_created_at, operation_id) < ($%d, $%d)`, argIndex, argIndex+1)) + } else { + queryBuilder.WriteString(fmt.Sprintf(` + AND (ledger_created_at, operation_id) > ($%d, $%d)`, argIndex, argIndex+1)) + } + args = append(args, cursor.LedgerCreatedAt, cursor.ID) + argIndex += 2 + } + + if orderBy == DESC { + queryBuilder.WriteString(` + ORDER BY ledger_created_at DESC, operation_id DESC`) + } else { + queryBuilder.WriteString(` + ORDER BY ledger_created_at ASC, operation_id ASC`) + } - // Build paginated query using shared utility - query, args := buildGetByAccountAddressQuery(paginatedQueryConfig{ - TableName: "operations", - CursorColumn: "id", - JoinTable: "operations_accounts", - JoinCondition: "operations_accounts.operation_id = operations.id", - Columns: columns, - AccountAddress: accountAddress, - Limit: limit, - Cursor: cursor, - OrderBy: orderBy, - }) + if limit != nil { + queryBuilder.WriteString(fmt.Sprintf(` LIMIT $%d`, argIndex)) + args = append(args, *limit) + argIndex++ + } + + // Close CTE and LATERAL join to fetch full operation rows + queryBuilder.WriteString(fmt.Sprintf(` + ) + SELECT %s, o.ledger_created_at as "cursor.cursor_ledger_created_at", o.id as "cursor.cursor_id" + FROM account_ops ao, + LATERAL (SELECT * FROM operations o WHERE o.id = ao.operation_id AND o.ledger_created_at = ao.ledger_created_at LIMIT 1) o`, columns)) + + if orderBy == DESC { + queryBuilder.WriteString(` + ORDER BY o.ledger_created_at DESC, o.id DESC`) + } else { + queryBuilder.WriteString(` + ORDER BY o.ledger_created_at ASC, o.id ASC`) + } + + query := queryBuilder.String() + + // For backward pagination, wrap query to reverse the final order + if orderBy == DESC { + query = fmt.Sprintf(`SELECT * FROM (%s) AS operations ORDER BY operations."cursor.cursor_ledger_created_at" ASC, operations."cursor.cursor_id" ASC`, query) + } var operations []*types.OperationWithCursor start := time.Now() diff --git a/internal/data/operations_test.go b/internal/data/operations_test.go index fe27139d0..be09b5667 100644 --- a/internal/data/operations_test.go +++ b/internal/data/operations_test.go @@ -267,17 +267,17 @@ func TestOperationModel_GetAll(t *testing.T) { operations, err := m.GetAll(ctx, "", nil, nil, ASC) require.NoError(t, err) assert.Len(t, operations, 3) - assert.Equal(t, int64(2), operations[0].Cursor) - assert.Equal(t, int64(4098), operations[1].Cursor) - assert.Equal(t, int64(8194), operations[2].Cursor) + assert.Equal(t, int64(2), operations[0].Cursor.ID) + assert.Equal(t, int64(4098), operations[1].Cursor.ID) + assert.Equal(t, int64(8194), operations[2].Cursor.ID) // Test GetAll with smaller limit limit := int32(2) operations, err = m.GetAll(ctx, "", &limit, nil, ASC) require.NoError(t, err) assert.Len(t, operations, 2) - assert.Equal(t, int64(2), operations[0].Cursor) - assert.Equal(t, int64(4098), operations[1].Cursor) + assert.Equal(t, int64(2), operations[0].Cursor.ID) + assert.Equal(t, int64(4098), operations[1].Cursor.ID) } func TestOperationModel_BatchGetByToIDs(t *testing.T) { @@ -589,7 +589,7 @@ func TestOperationModel_BatchGetByAccountAddresses(t *testing.T) { require.NoError(t, err) // Test BatchGetByAccount - operations, err := m.BatchGetByAccountAddress(ctx, address1, "", nil, nil, "ASC") + operations, err := m.BatchGetByAccountAddress(ctx, address1, "", nil, nil, ASC) require.NoError(t, err) assert.Len(t, operations, 2) assert.Equal(t, int64(4097), operations[0].Operation.ID) diff --git a/internal/data/query_utils.go b/internal/data/query_utils.go index 35fbd70d4..4610d8d5a 100644 --- a/internal/data/query_utils.go +++ b/internal/data/query_utils.go @@ -19,24 +19,6 @@ const ( DESC SortOrder = "DESC" ) -// PaginatedQueryConfig contains configuration for building paginated queries -type paginatedQueryConfig struct { - // Base table configuration - TableName string // e.g., "operations" or "transactions" - CursorColumn string // e.g., "id" or "to_id" - - // Join configuration - JoinTable string // e.g., "operations_accounts" or "transactions_accounts" - JoinCondition string // e.g., "operations_accounts.operation_id = operations.id" - - // Query parameters - Columns string - AccountAddress string - Limit *int32 - Cursor *int64 - OrderBy SortOrder -} - // pgtypeTextFromNullString converts sql.NullString to pgtype.Text for efficient binary COPY. func pgtypeTextFromNullString(ns sql.NullString) pgtype.Text { return pgtype.Text{String: ns.String, Valid: ns.Valid} @@ -80,67 +62,6 @@ func pgtypeBytesFromNullAddressBytea(na types.NullAddressBytea) ([]byte, error) return val.([]byte), nil } -// BuildPaginatedQuery constructs a paginated SQL query with cursor-based pagination -func buildGetByAccountAddressQuery(config paginatedQueryConfig) (string, []any) { - var queryBuilder strings.Builder - var args []any - argIndex := 1 - - // Base query with join - queryBuilder.WriteString(fmt.Sprintf(` - SELECT %s, %s.%s as cursor - FROM %s - INNER JOIN %s - ON %s - WHERE %s.account_id = $%d`, - config.Columns, - config.TableName, - config.CursorColumn, - config.TableName, - config.JoinTable, - config.JoinCondition, - config.JoinTable, - argIndex)) - args = append(args, types.AddressBytea(config.AccountAddress)) - argIndex++ - - // Add cursor condition if provided - if config.Cursor != nil { - // When paginating in descending order, we are going from greater cursor id to smaller cursor id - if config.OrderBy == DESC { - queryBuilder.WriteString(fmt.Sprintf(` AND %s.%s < $%d`, config.TableName, config.CursorColumn, argIndex)) - } else { - queryBuilder.WriteString(fmt.Sprintf(` AND %s.%s > $%d`, config.TableName, config.CursorColumn, argIndex)) - } - args = append(args, *config.Cursor) - argIndex++ - } - - // Add ordering - if config.OrderBy == DESC { - queryBuilder.WriteString(fmt.Sprintf(" ORDER BY %s.%s DESC", config.TableName, config.CursorColumn)) - } else { - queryBuilder.WriteString(fmt.Sprintf(" ORDER BY %s.%s ASC", config.TableName, config.CursorColumn)) - } - - // Add limit if provided - if config.Limit != nil { - queryBuilder.WriteString(fmt.Sprintf(` LIMIT $%d`, argIndex)) - args = append(args, *config.Limit) - } - - query := queryBuilder.String() - - // For backward pagination, wrap query to reverse the final order - // This ensures we always display the oldest items first in the output - if config.OrderBy == DESC { - query = fmt.Sprintf(`SELECT * FROM (%s) AS %s ORDER BY %s.cursor ASC`, - query, config.TableName, config.TableName) - } - - return query, args -} - func getDBColumns(model any) set.Set[string] { modelType := reflect.TypeOf(model) dbColumns := set.NewSet[string]() diff --git a/internal/data/statechanges.go b/internal/data/statechanges.go index 0560e0b44..a298cd5f9 100644 --- a/internal/data/statechanges.go +++ b/internal/data/statechanges.go @@ -30,7 +30,7 @@ func (m *StateChangeModel) BatchGetByAccountAddress(ctx context.Context, account argIndex := 2 queryBuilder.WriteString(fmt.Sprintf(` - SELECT %s, to_id as "cursor.cursor_to_id", operation_id as "cursor.cursor_operation_id", state_change_order as "cursor.cursor_state_change_order" + SELECT %s, ledger_created_at as "cursor.cursor_ledger_created_at", to_id as "cursor.cursor_to_id", operation_id as "cursor.cursor_operation_id", state_change_order as "cursor.cursor_state_change_order" FROM state_changes WHERE account_id = $1 `, columns)) @@ -63,29 +63,29 @@ func (m *StateChangeModel) BatchGetByAccountAddress(ctx context.Context, account argIndex++ } - // Add cursor-based pagination using 3-column comparison (to_id, operation_id, state_change_order) + // Add cursor-based pagination using 4-column comparison with ledger_created_at as the + // leading column. This enables TimescaleDB ChunkAppend optimization on hypertables. if cursor != nil { if sortOrder == DESC { queryBuilder.WriteString(fmt.Sprintf(` - AND (to_id, operation_id, state_change_order) < ($%d, $%d, $%d) - `, argIndex, argIndex+1, argIndex+2)) - args = append(args, cursor.ToID, cursor.OperationID, cursor.StateChangeOrder) - argIndex += 3 + AND (ledger_created_at, to_id, operation_id, state_change_order) < ($%d, $%d, $%d, $%d) + `, argIndex, argIndex+1, argIndex+2, argIndex+3)) + args = append(args, cursor.LedgerCreatedAt, cursor.ToID, cursor.OperationID, cursor.StateChangeOrder) + argIndex += 4 } else { queryBuilder.WriteString(fmt.Sprintf(` - AND (to_id, operation_id, state_change_order) > ($%d, $%d, $%d) - `, argIndex, argIndex+1, argIndex+2)) - args = append(args, cursor.ToID, cursor.OperationID, cursor.StateChangeOrder) - argIndex += 3 + AND (ledger_created_at, to_id, operation_id, state_change_order) > ($%d, $%d, $%d, $%d) + `, argIndex, argIndex+1, argIndex+2, argIndex+3)) + args = append(args, cursor.LedgerCreatedAt, cursor.ToID, cursor.OperationID, cursor.StateChangeOrder) + argIndex += 4 } } - // TODO: Extract the ordering code to separate function in utils and use everywhere - // Add ordering + // Add ordering with ledger_created_at as leading column for TimescaleDB ChunkAppend if sortOrder == DESC { - queryBuilder.WriteString(" ORDER BY to_id DESC, operation_id DESC, state_change_order DESC") + queryBuilder.WriteString(" ORDER BY ledger_created_at DESC, to_id DESC, operation_id DESC, state_change_order DESC") } else { - queryBuilder.WriteString(" ORDER BY to_id ASC, operation_id ASC, state_change_order ASC") + queryBuilder.WriteString(" ORDER BY ledger_created_at ASC, to_id ASC, operation_id ASC, state_change_order ASC") } // Add limit using parameterized query @@ -97,10 +97,10 @@ func (m *StateChangeModel) BatchGetByAccountAddress(ctx context.Context, account query := queryBuilder.String() // For backward pagination, wrap query to reverse the final order. - // We use cursor alias columns (e.g., "cursor.cursor_to_id") in ORDER BY to avoid + // We use cursor alias columns (e.g., "cursor.cursor_ledger_created_at") in ORDER BY to avoid // ambiguity since the inner SELECT includes both original columns and cursor aliases. if sortOrder == DESC { - query = fmt.Sprintf(`SELECT * FROM (%s) AS statechanges ORDER BY statechanges."cursor.cursor_to_id" ASC, statechanges."cursor.cursor_operation_id" ASC, statechanges."cursor.cursor_state_change_order" ASC`, query) + query = fmt.Sprintf(`SELECT * FROM (%s) AS statechanges ORDER BY statechanges."cursor.cursor_ledger_created_at" ASC, statechanges."cursor.cursor_to_id" ASC, statechanges."cursor.cursor_operation_id" ASC, statechanges."cursor.cursor_state_change_order" ASC`, query) } var stateChanges []*types.StateChangeWithCursor @@ -120,26 +120,27 @@ func (m *StateChangeModel) GetAll(ctx context.Context, columns string, limit *in columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_order") var queryBuilder strings.Builder queryBuilder.WriteString(fmt.Sprintf(` - SELECT %s, to_id as "cursor.cursor_to_id", operation_id as "cursor.cursor_operation_id", state_change_order as "cursor.cursor_state_change_order" + SELECT %s, ledger_created_at as "cursor.cursor_ledger_created_at", to_id as "cursor.cursor_to_id", operation_id as "cursor.cursor_operation_id", state_change_order as "cursor.cursor_state_change_order" FROM state_changes `, columns)) if cursor != nil { if sortOrder == DESC { queryBuilder.WriteString(fmt.Sprintf(` - WHERE (to_id, operation_id, state_change_order) < (%d, %d, %d) - `, cursor.ToID, cursor.OperationID, cursor.StateChangeOrder)) + WHERE (ledger_created_at, to_id, operation_id, state_change_order) < ('%s', %d, %d, %d) + `, cursor.LedgerCreatedAt.Format(time.RFC3339Nano), cursor.ToID, cursor.OperationID, cursor.StateChangeOrder)) } else { queryBuilder.WriteString(fmt.Sprintf(` - WHERE (to_id, operation_id, state_change_order) > (%d, %d, %d) - `, cursor.ToID, cursor.OperationID, cursor.StateChangeOrder)) + WHERE (ledger_created_at, to_id, operation_id, state_change_order) > ('%s', %d, %d, %d) + `, cursor.LedgerCreatedAt.Format(time.RFC3339Nano), cursor.ToID, cursor.OperationID, cursor.StateChangeOrder)) } } + // Order with ledger_created_at as leading column for TimescaleDB ChunkAppend if sortOrder == DESC { - queryBuilder.WriteString(" ORDER BY to_id DESC, operation_id DESC, state_change_order DESC") + queryBuilder.WriteString(" ORDER BY ledger_created_at DESC, to_id DESC, operation_id DESC, state_change_order DESC") } else { - queryBuilder.WriteString(" ORDER BY to_id ASC, operation_id ASC, state_change_order ASC") + queryBuilder.WriteString(" ORDER BY ledger_created_at ASC, to_id ASC, operation_id ASC, state_change_order ASC") } if limit != nil && *limit > 0 { @@ -149,10 +150,10 @@ func (m *StateChangeModel) GetAll(ctx context.Context, columns string, limit *in query := queryBuilder.String() // For backward pagination, wrap query to reverse the final order. - // We use cursor alias columns (e.g., "cursor.cursor_to_id") in ORDER BY to avoid + // We use cursor alias columns (e.g., "cursor.cursor_ledger_created_at") in ORDER BY to avoid // ambiguity since the inner SELECT includes both original columns and cursor aliases. if sortOrder == DESC { - query = fmt.Sprintf(`SELECT * FROM (%s) AS statechanges ORDER BY statechanges."cursor.cursor_to_id" ASC, statechanges."cursor.cursor_operation_id" ASC, statechanges."cursor.cursor_state_change_order" ASC`, query) + query = fmt.Sprintf(`SELECT * FROM (%s) AS statechanges ORDER BY statechanges."cursor.cursor_ledger_created_at" ASC, statechanges."cursor.cursor_to_id" ASC, statechanges."cursor.cursor_operation_id" ASC, statechanges."cursor.cursor_state_change_order" ASC`, query) } var stateChanges []*types.StateChangeWithCursor diff --git a/internal/data/transactions.go b/internal/data/transactions.go index 5209ffcfc..8f525d208 100644 --- a/internal/data/transactions.go +++ b/internal/data/transactions.go @@ -39,23 +39,23 @@ func (m *TransactionModel) GetByHash(ctx context.Context, hash string, columns s return &transaction, nil } -func (m *TransactionModel) GetAll(ctx context.Context, columns string, limit *int32, cursor *int64, sortOrder SortOrder) ([]*types.TransactionWithCursor, error) { +func (m *TransactionModel) GetAll(ctx context.Context, columns string, limit *int32, cursor *types.CompositeCursor, sortOrder SortOrder) ([]*types.TransactionWithCursor, error) { columns = prepareColumnsWithID(columns, types.Transaction{}, "", "to_id") queryBuilder := strings.Builder{} - queryBuilder.WriteString(fmt.Sprintf(`SELECT %s, to_id as cursor FROM transactions`, columns)) + queryBuilder.WriteString(fmt.Sprintf(`SELECT %s, ledger_created_at as "cursor.cursor_ledger_created_at", to_id as "cursor.cursor_id" FROM transactions`, columns)) if cursor != nil { if sortOrder == DESC { - queryBuilder.WriteString(fmt.Sprintf(" WHERE to_id < %d", *cursor)) + queryBuilder.WriteString(fmt.Sprintf(` WHERE (ledger_created_at, to_id) < ('%s', %d)`, cursor.LedgerCreatedAt.Format(time.RFC3339Nano), cursor.ID)) } else { - queryBuilder.WriteString(fmt.Sprintf(" WHERE to_id > %d", *cursor)) + queryBuilder.WriteString(fmt.Sprintf(` WHERE (ledger_created_at, to_id) > ('%s', %d)`, cursor.LedgerCreatedAt.Format(time.RFC3339Nano), cursor.ID)) } } if sortOrder == DESC { - queryBuilder.WriteString(" ORDER BY to_id DESC") + queryBuilder.WriteString(" ORDER BY ledger_created_at DESC, to_id DESC") } else { - queryBuilder.WriteString(" ORDER BY to_id ASC") + queryBuilder.WriteString(" ORDER BY ledger_created_at ASC, to_id ASC") } if limit != nil { @@ -64,7 +64,7 @@ func (m *TransactionModel) GetAll(ctx context.Context, columns string, limit *in query := queryBuilder.String() if sortOrder == DESC { - query = fmt.Sprintf(`SELECT * FROM (%s) AS transactions ORDER BY cursor ASC`, query) + query = fmt.Sprintf(`SELECT * FROM (%s) AS transactions ORDER BY transactions."cursor.cursor_ledger_created_at" ASC, transactions."cursor.cursor_id" ASC`, query) } var transactions []*types.TransactionWithCursor @@ -81,21 +81,71 @@ func (m *TransactionModel) GetAll(ctx context.Context, columns string, limit *in } // BatchGetByAccountAddress gets the transactions that are associated with a single account address. -func (m *TransactionModel) BatchGetByAccountAddress(ctx context.Context, accountAddress string, columns string, limit *int32, cursor *int64, orderBy SortOrder) ([]*types.TransactionWithCursor, error) { - columns = prepareColumnsWithID(columns, types.Transaction{}, "transactions", "to_id") +// Uses a MATERIALIZED CTE + LATERAL join pattern to allow TimescaleDB ChunkAppend optimization +// on the transactions_accounts hypertable by ordering on ledger_created_at first. +func (m *TransactionModel) BatchGetByAccountAddress(ctx context.Context, accountAddress string, columns string, limit *int32, cursor *types.CompositeCursor, orderBy SortOrder) ([]*types.TransactionWithCursor, error) { + columns = prepareColumnsWithID(columns, types.Transaction{}, "t", "to_id") + + var queryBuilder strings.Builder + args := []interface{}{types.AddressBytea(accountAddress)} + argIndex := 2 + + // MATERIALIZED CTE scans transactions_accounts with ledger_created_at leading the ORDER BY, + // enabling TimescaleDB ChunkAppend on the hypertable. + queryBuilder.WriteString(fmt.Sprintf(` + WITH account_txns AS MATERIALIZED ( + SELECT tx_to_id, ledger_created_at + FROM transactions_accounts + WHERE account_id = $1`)) + + // Add cursor-based pagination on the CTE + if cursor != nil { + if orderBy == DESC { + queryBuilder.WriteString(fmt.Sprintf(` + AND (ledger_created_at, tx_to_id) < ($%d, $%d)`, argIndex, argIndex+1)) + } else { + queryBuilder.WriteString(fmt.Sprintf(` + AND (ledger_created_at, tx_to_id) > ($%d, $%d)`, argIndex, argIndex+1)) + } + args = append(args, cursor.LedgerCreatedAt, cursor.ID) + argIndex += 2 + } + + if orderBy == DESC { + queryBuilder.WriteString(` + ORDER BY ledger_created_at DESC, tx_to_id DESC`) + } else { + queryBuilder.WriteString(` + ORDER BY ledger_created_at ASC, tx_to_id ASC`) + } - // Build paginated query using shared utility - query, args := buildGetByAccountAddressQuery(paginatedQueryConfig{ - TableName: "transactions", - CursorColumn: "to_id", - JoinTable: "transactions_accounts", - JoinCondition: "transactions_accounts.tx_to_id = transactions.to_id", - Columns: columns, - AccountAddress: accountAddress, - Limit: limit, - Cursor: cursor, - OrderBy: orderBy, - }) + if limit != nil { + queryBuilder.WriteString(fmt.Sprintf(` LIMIT $%d`, argIndex)) + args = append(args, *limit) + argIndex++ + } + + // Close CTE and LATERAL join to fetch full transaction rows + queryBuilder.WriteString(fmt.Sprintf(` + ) + SELECT %s, t.ledger_created_at as "cursor.cursor_ledger_created_at", t.to_id as "cursor.cursor_id" + FROM account_txns ta, + LATERAL (SELECT * FROM transactions t WHERE t.to_id = ta.tx_to_id AND t.ledger_created_at = ta.ledger_created_at LIMIT 1) t`, columns)) + + if orderBy == DESC { + queryBuilder.WriteString(` + ORDER BY t.ledger_created_at DESC, t.to_id DESC`) + } else { + queryBuilder.WriteString(` + ORDER BY t.ledger_created_at ASC, t.to_id ASC`) + } + + query := queryBuilder.String() + + // For backward pagination, wrap query to reverse the final order + if orderBy == DESC { + query = fmt.Sprintf(`SELECT * FROM (%s) AS transactions ORDER BY transactions."cursor.cursor_ledger_created_at" ASC, transactions."cursor.cursor_id" ASC`, query) + } var transactions []*types.TransactionWithCursor start := time.Now() diff --git a/internal/data/transactions_test.go b/internal/data/transactions_test.go index 3e1b952c6..5e6580456 100644 --- a/internal/data/transactions_test.go +++ b/internal/data/transactions_test.go @@ -295,17 +295,17 @@ func TestTransactionModel_GetAll(t *testing.T) { transactions, err := m.GetAll(ctx, "", nil, nil, ASC) require.NoError(t, err) assert.Len(t, transactions, 3) - assert.Equal(t, int64(1), transactions[0].Cursor) - assert.Equal(t, int64(2), transactions[1].Cursor) - assert.Equal(t, int64(3), transactions[2].Cursor) + assert.Equal(t, int64(1), transactions[0].Cursor.ID) + assert.Equal(t, int64(2), transactions[1].Cursor.ID) + assert.Equal(t, int64(3), transactions[2].Cursor.ID) // Test GetAll with smaller limit limit := int32(2) transactions, err = m.GetAll(ctx, "", &limit, nil, ASC) require.NoError(t, err) assert.Len(t, transactions, 2) - assert.Equal(t, int64(1), transactions[0].Cursor) - assert.Equal(t, int64(2), transactions[1].Cursor) + assert.Equal(t, int64(1), transactions[0].Cursor.ID) + assert.Equal(t, int64(2), transactions[1].Cursor.ID) } func TestTransactionModel_BatchGetByAccountAddress(t *testing.T) { @@ -359,12 +359,12 @@ func TestTransactionModel_BatchGetByAccountAddress(t *testing.T) { require.NoError(t, err) // Test BatchGetByAccount - transactions, err := m.BatchGetByAccountAddress(ctx, address1, "", nil, nil, "ASC") + transactions, err := m.BatchGetByAccountAddress(ctx, address1, "", nil, nil, ASC) require.NoError(t, err) assert.Len(t, transactions, 2) - assert.Equal(t, int64(1), transactions[0].Cursor) - assert.Equal(t, int64(2), transactions[1].Cursor) + assert.Equal(t, int64(1), transactions[0].Cursor.ID) + assert.Equal(t, int64(2), transactions[1].Cursor.ID) } func TestTransactionModel_BatchGetByOperationIDs(t *testing.T) { diff --git a/internal/indexer/types/types.go b/internal/indexer/types/types.go index ae6d2344b..20717e916 100644 --- a/internal/indexer/types/types.go +++ b/internal/indexer/types/types.go @@ -312,9 +312,17 @@ type Transaction struct { InnerTransactionHash string `json:"innerTransactionHash,omitempty" db:"-"` } +// CompositeCursor encodes both ledger_created_at and an entity ID for TimescaleDB-friendly +// cursor-based pagination. Using ledger_created_at as the leading sort column allows +// TimescaleDB to use ChunkAppend optimization on hypertables. +type CompositeCursor struct { + LedgerCreatedAt time.Time `db:"cursor_ledger_created_at"` + ID int64 `db:"cursor_id"` +} + type TransactionWithCursor struct { Transaction - Cursor int64 `json:"cursor,omitempty" db:"cursor"` + Cursor CompositeCursor `json:"cursor,omitempty" db:"cursor"` } type TransactionWithStateChangeID struct { @@ -420,7 +428,7 @@ type Operation struct { type OperationWithCursor struct { Operation - Cursor int64 `json:"cursor,omitempty" db:"cursor"` + Cursor CompositeCursor `json:"cursor,omitempty" db:"cursor"` } type OperationWithStateChangeID struct { @@ -617,9 +625,10 @@ type StateChangeWithCursor struct { } type StateChangeCursor struct { - ToID int64 `db:"cursor_to_id"` - OperationID int64 `db:"cursor_operation_id"` - StateChangeOrder int64 `db:"cursor_state_change_order"` + LedgerCreatedAt time.Time `db:"cursor_ledger_created_at"` + ToID int64 `db:"cursor_to_id"` + OperationID int64 `db:"cursor_operation_id"` + StateChangeOrder int64 `db:"cursor_state_change_order"` } type StateChangeCursorGetter interface { diff --git a/internal/serve/graphql/resolvers/account.resolvers.go b/internal/serve/graphql/resolvers/account.resolvers.go index 42cce7eb3..ddec6224b 100644 --- a/internal/serve/graphql/resolvers/account.resolvers.go +++ b/internal/serve/graphql/resolvers/account.resolvers.go @@ -23,20 +23,20 @@ func (r *accountResolver) Address(ctx context.Context, obj *types.Account) (stri // gqlgen calls this when a GraphQL query requests the transactions field on an Account // Field resolvers receive the parent object (Account) and return the field value func (r *accountResolver) Transactions(ctx context.Context, obj *types.Account, first *int32, after *string, last *int32, before *string) (*graphql1.TransactionConnection, error) { - params, err := parsePaginationParams(first, after, last, before, false) + params, err := parsePaginationParams(first, after, last, before, CursorTypeComposite) if err != nil { return nil, fmt.Errorf("parsing pagination params: %w", err) } queryLimit := *params.Limit + 1 // +1 to check if there is a next page dbColumns := GetDBColumnsForFields(ctx, types.Transaction{}) - transactions, err := r.models.Transactions.BatchGetByAccountAddress(ctx, string(obj.StellarAddress), strings.Join(dbColumns, ", "), &queryLimit, params.Cursor, params.SortOrder) + transactions, err := r.models.Transactions.BatchGetByAccountAddress(ctx, string(obj.StellarAddress), strings.Join(dbColumns, ", "), &queryLimit, params.CompositeCursor, params.SortOrder) if err != nil { return nil, fmt.Errorf("getting transactions from db for account %s: %w", obj.StellarAddress, err) } - conn := NewConnectionWithRelayPagination(transactions, params, func(tx *types.TransactionWithCursor) int64 { - return tx.Cursor + conn := NewConnectionWithRelayPagination(transactions, params, func(tx *types.TransactionWithCursor) string { + return fmt.Sprintf("%d:%d", tx.Cursor.LedgerCreatedAt.UnixNano(), tx.Cursor.ID) }) edges := make([]*graphql1.TransactionEdge, len(conn.Edges)) @@ -56,20 +56,20 @@ func (r *accountResolver) Transactions(ctx context.Context, obj *types.Account, // Operations is the resolver for the operations field. // This field resolver handles the "operations" field on an Account object func (r *accountResolver) Operations(ctx context.Context, obj *types.Account, first *int32, after *string, last *int32, before *string) (*graphql1.OperationConnection, error) { - params, err := parsePaginationParams(first, after, last, before, false) + params, err := parsePaginationParams(first, after, last, before, CursorTypeComposite) if err != nil { return nil, fmt.Errorf("parsing pagination params: %w", err) } queryLimit := *params.Limit + 1 // +1 to check if there is a next page dbColumns := GetDBColumnsForFields(ctx, types.Operation{}) - operations, err := r.models.Operations.BatchGetByAccountAddress(ctx, string(obj.StellarAddress), strings.Join(dbColumns, ", "), &queryLimit, params.Cursor, params.SortOrder) + operations, err := r.models.Operations.BatchGetByAccountAddress(ctx, string(obj.StellarAddress), strings.Join(dbColumns, ", "), &queryLimit, params.CompositeCursor, params.SortOrder) if err != nil { return nil, fmt.Errorf("getting operations from db for account %s: %w", obj.StellarAddress, err) } - conn := NewConnectionWithRelayPagination(operations, params, func(op *types.OperationWithCursor) int64 { - return op.Cursor + conn := NewConnectionWithRelayPagination(operations, params, func(op *types.OperationWithCursor) string { + return fmt.Sprintf("%d:%d", op.Cursor.LedgerCreatedAt.UnixNano(), op.Cursor.ID) }) edges := make([]*graphql1.OperationEdge, len(conn.Edges)) @@ -88,7 +88,7 @@ func (r *accountResolver) Operations(ctx context.Context, obj *types.Account, fi // StateChanges is the resolver for the stateChanges field. func (r *accountResolver) StateChanges(ctx context.Context, obj *types.Account, filter *graphql1.AccountStateChangeFilterInput, first *int32, after *string, last *int32, before *string) (*graphql1.StateChangeConnection, error) { - params, err := parsePaginationParams(first, after, last, before, true) + params, err := parsePaginationParams(first, after, last, before, CursorTypeStateChange) if err != nil { return nil, fmt.Errorf("parsing pagination params: %w", err) } @@ -122,7 +122,7 @@ func (r *accountResolver) StateChanges(ctx context.Context, obj *types.Account, convertedStateChanges := convertStateChangeToBaseStateChange(stateChanges) conn := NewConnectionWithRelayPagination(convertedStateChanges, params, func(sc *baseStateChangeWithCursor) string { - return fmt.Sprintf("%d:%d:%d", sc.cursor.ToID, sc.cursor.OperationID, sc.cursor.StateChangeOrder) + return fmt.Sprintf("%d:%d:%d:%d", sc.cursor.LedgerCreatedAt.UnixNano(), sc.cursor.ToID, sc.cursor.OperationID, sc.cursor.StateChangeOrder) }) edges := make([]*graphql1.StateChangeEdge, len(conn.Edges)) diff --git a/internal/serve/graphql/resolvers/operation.resolvers.go b/internal/serve/graphql/resolvers/operation.resolvers.go index 4097f59c9..ca994d780 100644 --- a/internal/serve/graphql/resolvers/operation.resolvers.go +++ b/internal/serve/graphql/resolvers/operation.resolvers.go @@ -73,7 +73,7 @@ func (r *operationResolver) Accounts(ctx context.Context, obj *types.Operation) // Field resolvers receive the parent object (Operation) and return the field value func (r *operationResolver) StateChanges(ctx context.Context, obj *types.Operation, first *int32, after *string, last *int32, before *string) (*graphql1.StateChangeConnection, error) { dbColumns := GetDBColumnsForFields(ctx, types.StateChange{}) - params, err := parsePaginationParams(first, after, last, before, true) + params, err := parsePaginationParams(first, after, last, before, CursorTypeStateChange) if err != nil { return nil, fmt.Errorf("parsing pagination params: %w", err) } @@ -95,7 +95,7 @@ func (r *operationResolver) StateChanges(ctx context.Context, obj *types.Operati convertedStateChanges := convertStateChangeToBaseStateChange(stateChanges) conn := NewConnectionWithRelayPagination(convertedStateChanges, params, func(sc *baseStateChangeWithCursor) string { - return fmt.Sprintf("%d:%d:%d", sc.cursor.ToID, sc.cursor.OperationID, sc.cursor.StateChangeOrder) + return fmt.Sprintf("%d:%d:%d:%d", sc.cursor.LedgerCreatedAt.UnixNano(), sc.cursor.ToID, sc.cursor.OperationID, sc.cursor.StateChangeOrder) }) edges := make([]*graphql1.StateChangeEdge, len(conn.Edges)) diff --git a/internal/serve/graphql/resolvers/queries.resolvers.go b/internal/serve/graphql/resolvers/queries.resolvers.go index 3677303ab..de0dc5bb8 100644 --- a/internal/serve/graphql/resolvers/queries.resolvers.go +++ b/internal/serve/graphql/resolvers/queries.resolvers.go @@ -34,20 +34,20 @@ func (r *queryResolver) TransactionByHash(ctx context.Context, hash string) (*ty // This resolver handles the "transactions" query. // It demonstrates handling optional arguments (limit can be nil) func (r *queryResolver) Transactions(ctx context.Context, first *int32, after *string, last *int32, before *string) (*graphql1.TransactionConnection, error) { - params, err := parsePaginationParams(first, after, last, before, false) + params, err := parsePaginationParams(first, after, last, before, CursorTypeComposite) if err != nil { return nil, fmt.Errorf("parsing pagination params: %w", err) } queryLimit := *params.Limit + 1 // +1 to check if there is a next page dbColumns := GetDBColumnsForFields(ctx, types.Transaction{}) - transactions, err := r.models.Transactions.GetAll(ctx, strings.Join(dbColumns, ", "), &queryLimit, params.Cursor, params.SortOrder) + transactions, err := r.models.Transactions.GetAll(ctx, strings.Join(dbColumns, ", "), &queryLimit, params.CompositeCursor, params.SortOrder) if err != nil { return nil, fmt.Errorf("getting transactions from db: %w", err) } - conn := NewConnectionWithRelayPagination(transactions, params, func(t *types.TransactionWithCursor) int64 { - return t.Cursor + conn := NewConnectionWithRelayPagination(transactions, params, func(t *types.TransactionWithCursor) string { + return fmt.Sprintf("%d:%d", t.Cursor.LedgerCreatedAt.UnixNano(), t.Cursor.ID) }) edges := make([]*graphql1.TransactionEdge, len(conn.Edges)) @@ -88,20 +88,20 @@ func (r *queryResolver) AccountByAddress(ctx context.Context, address string) (* // Operations is the resolver for the operations field. // This resolver handles the "operations" query. func (r *queryResolver) Operations(ctx context.Context, first *int32, after *string, last *int32, before *string) (*graphql1.OperationConnection, error) { - params, err := parsePaginationParams(first, after, last, before, false) + params, err := parsePaginationParams(first, after, last, before, CursorTypeComposite) if err != nil { return nil, fmt.Errorf("parsing pagination params: %w", err) } queryLimit := *params.Limit + 1 // +1 to check if there is a next page dbColumns := GetDBColumnsForFields(ctx, types.Operation{}) - operations, err := r.models.Operations.GetAll(ctx, strings.Join(dbColumns, ", "), &queryLimit, params.Cursor, params.SortOrder) + operations, err := r.models.Operations.GetAll(ctx, strings.Join(dbColumns, ", "), &queryLimit, params.CompositeCursor, params.SortOrder) if err != nil { return nil, fmt.Errorf("getting operations from db: %w", err) } - conn := NewConnectionWithRelayPagination(operations, params, func(o *types.OperationWithCursor) int64 { - return o.Cursor + conn := NewConnectionWithRelayPagination(operations, params, func(o *types.OperationWithCursor) string { + return fmt.Sprintf("%d:%d", o.Cursor.LedgerCreatedAt.UnixNano(), o.Cursor.ID) }) edges := make([]*graphql1.OperationEdge, len(conn.Edges)) @@ -126,7 +126,7 @@ func (r *queryResolver) OperationByID(ctx context.Context, id int64) (*types.Ope // StateChanges is the resolver for the stateChanges field. func (r *queryResolver) StateChanges(ctx context.Context, first *int32, after *string, last *int32, before *string) (*graphql1.StateChangeConnection, error) { - params, err := parsePaginationParams(first, after, last, before, true) + params, err := parsePaginationParams(first, after, last, before, CursorTypeStateChange) if err != nil { return nil, fmt.Errorf("parsing pagination params: %w", err) } @@ -140,7 +140,7 @@ func (r *queryResolver) StateChanges(ctx context.Context, first *int32, after *s convertedStateChanges := convertStateChangeToBaseStateChange(stateChanges) conn := NewConnectionWithRelayPagination(convertedStateChanges, params, func(sc *baseStateChangeWithCursor) string { - return fmt.Sprintf("%d:%d:%d", sc.cursor.ToID, sc.cursor.OperationID, sc.cursor.StateChangeOrder) + return fmt.Sprintf("%d:%d:%d:%d", sc.cursor.LedgerCreatedAt.UnixNano(), sc.cursor.ToID, sc.cursor.OperationID, sc.cursor.StateChangeOrder) }) edges := make([]*graphql1.StateChangeEdge, len(conn.Edges)) diff --git a/internal/serve/graphql/resolvers/test_utils.go b/internal/serve/graphql/resolvers/test_utils.go index bed41d477..b5897ce8b 100644 --- a/internal/serve/graphql/resolvers/test_utils.go +++ b/internal/serve/graphql/resolvers/test_utils.go @@ -65,6 +65,7 @@ var ( func setupDB(ctx context.Context, t *testing.T, dbConnectionPool db.ConnectionPool) { testLedger := int32(1000) + now := time.Now() parentAccount := &types.Account{StellarAddress: types.AddressBytea(sharedTestAccountAddress)} txns := make([]*types.Transaction, 0, 4) ops := make([]*types.Operation, 0, 8) @@ -78,7 +79,7 @@ func setupDB(ctx context.Context, t *testing.T, dbConnectionPool db.ConnectionPo ResultCode: "TransactionResultCodeTxSuccess", MetaXDR: ptr(fmt.Sprintf("meta%d", i+1)), LedgerNumber: 1, - LedgerCreatedAt: time.Now(), + LedgerCreatedAt: now, IsFeeBump: false, } txns = append(txns, txn) @@ -92,7 +93,7 @@ func setupDB(ctx context.Context, t *testing.T, dbConnectionPool db.ConnectionPo ResultCode: "op_success", Successful: true, LedgerNumber: 1, - LedgerCreatedAt: time.Now(), + LedgerCreatedAt: now, }) opIdx++ } @@ -121,7 +122,7 @@ func setupDB(ctx context.Context, t *testing.T, dbConnectionPool db.ConnectionPo StateChangeReason: reason, OperationID: op.ID, AccountID: parentAccount.StellarAddress, - LedgerCreatedAt: time.Now(), + LedgerCreatedAt: now, LedgerNumber: 1, }) } @@ -135,7 +136,7 @@ func setupDB(ctx context.Context, t *testing.T, dbConnectionPool db.ConnectionPo StateChangeCategory: types.StateChangeCategoryBalance, StateChangeReason: &debitReason, AccountID: parentAccount.StellarAddress, - LedgerCreatedAt: time.Now(), + LedgerCreatedAt: now, LedgerNumber: 1000, }) } diff --git a/internal/serve/graphql/resolvers/transaction.resolvers.go b/internal/serve/graphql/resolvers/transaction.resolvers.go index eeb7dec2b..8542283bc 100644 --- a/internal/serve/graphql/resolvers/transaction.resolvers.go +++ b/internal/serve/graphql/resolvers/transaction.resolvers.go @@ -25,7 +25,7 @@ func (r *transactionResolver) Hash(ctx context.Context, obj *types.Transaction) // It's called when a GraphQL query requests the operations within a transaction func (r *transactionResolver) Operations(ctx context.Context, obj *types.Transaction, first *int32, after *string, last *int32, before *string) (*graphql1.OperationConnection, error) { dbColumns := GetDBColumnsForFields(ctx, types.Operation{}) - params, err := parsePaginationParams(first, after, last, before, false) + params, err := parsePaginationParams(first, after, last, before, CursorTypeInt64) if err != nil { return nil, fmt.Errorf("parsing pagination params: %w", err) } @@ -46,7 +46,7 @@ func (r *transactionResolver) Operations(ctx context.Context, obj *types.Transac } conn := NewConnectionWithRelayPagination(operations, params, func(o *types.OperationWithCursor) int64 { - return o.Cursor + return o.Cursor.ID }) edges := make([]*graphql1.OperationEdge, len(conn.Edges)) @@ -89,7 +89,7 @@ func (r *transactionResolver) Accounts(ctx context.Context, obj *types.Transacti // It's called when a GraphQL query requests the state changes within a transaction func (r *transactionResolver) StateChanges(ctx context.Context, obj *types.Transaction, first *int32, after *string, last *int32, before *string) (*graphql1.StateChangeConnection, error) { dbColumns := GetDBColumnsForFields(ctx, types.StateChange{}) - params, err := parsePaginationParams(first, after, last, before, true) + params, err := parsePaginationParams(first, after, last, before, CursorTypeStateChange) if err != nil { return nil, fmt.Errorf("parsing pagination params: %w", err) } @@ -111,7 +111,7 @@ func (r *transactionResolver) StateChanges(ctx context.Context, obj *types.Trans convertedStateChanges := convertStateChangeToBaseStateChange(stateChanges) conn := NewConnectionWithRelayPagination(convertedStateChanges, params, func(sc *baseStateChangeWithCursor) string { - return fmt.Sprintf("%d:%d:%d", sc.cursor.ToID, sc.cursor.OperationID, sc.cursor.StateChangeOrder) + return fmt.Sprintf("%d:%d:%d:%d", sc.cursor.LedgerCreatedAt.UnixNano(), sc.cursor.ToID, sc.cursor.OperationID, sc.cursor.StateChangeOrder) }) edges := make([]*graphql1.StateChangeEdge, len(conn.Edges)) diff --git a/internal/serve/graphql/resolvers/utils.go b/internal/serve/graphql/resolvers/utils.go index c7b1d407c..9813c7496 100644 --- a/internal/serve/graphql/resolvers/utils.go +++ b/internal/serve/graphql/resolvers/utils.go @@ -7,6 +7,7 @@ import ( "reflect" "strconv" "strings" + "time" "github.com/99designs/gqlgen/graphql" @@ -31,9 +32,22 @@ type GenericConnection[T any] struct { PageInfo *generated.PageInfo } +// CursorType determines how pagination cursors are parsed and interpreted. +type CursorType int + +const ( + // CursorTypeInt64 is used for within-transaction nested resolvers (e.g., operations by ToID) + CursorTypeInt64 CursorType = iota + // CursorTypeComposite is used for account-level and root tx/ops queries (ledger_created_at:id format) + CursorTypeComposite + // CursorTypeStateChange is used for state change queries (ledger_created_at:to_id:op_id:sc_order format) + CursorTypeStateChange +) + type PaginationParams struct { Limit *int32 Cursor *int64 + CompositeCursor *types.CompositeCursor StateChangeCursor *types.StateChangeCursor ForwardPagination bool SortOrder data.SortOrder @@ -49,19 +63,21 @@ func NewConnectionWithRelayPagination[T any, C int64 | string](nodes []T, params hasNextPage := false hasPreviousPage := false + hasCursor := params.Cursor != nil || params.CompositeCursor != nil || params.StateChangeCursor != nil + if params.ForwardPagination { if int32(len(nodes)) > *params.Limit { hasNextPage = true nodes = nodes[:*params.Limit] } - hasPreviousPage = (params.Cursor != nil || params.StateChangeCursor != nil) + hasPreviousPage = hasCursor } else { if int32(len(nodes)) > *params.Limit { hasPreviousPage = true nodes = nodes[1:] } // In backward pagination, presence of a before-cursor implies there may be newer items (a "next page") - hasNextPage = (params.Cursor != nil || params.StateChangeCursor != nil) + hasNextPage = hasCursor } edges := make([]*GenericEdge[T], len(nodes)) @@ -273,7 +289,7 @@ func getColumnMap(model any) map[string]string { return fieldToColumnMap } -func parsePaginationParams(first *int32, after *string, last *int32, before *string, isStateChange bool) (PaginationParams, error) { +func parsePaginationParams(first *int32, after *string, last *int32, before *string, cursorType CursorType) (PaginationParams, error) { err := validatePaginationParams(first, after, last, before) if err != nil { return PaginationParams{}, fmt.Errorf("validating pagination params: %w", err) @@ -299,13 +315,20 @@ func parsePaginationParams(first *int32, after *string, last *int32, before *str ForwardPagination: forwardPagination, } - if isStateChange { + switch cursorType { + case CursorTypeStateChange: stateChangeCursor, err := parseStateChangeCursor(cursor) if err != nil { return PaginationParams{}, fmt.Errorf("parsing state change cursor: %w", err) } paginationParams.StateChangeCursor = stateChangeCursor - } else { + case CursorTypeComposite: + compositeCursor, err := parseCompositeCursor(cursor) + if err != nil { + return PaginationParams{}, fmt.Errorf("parsing composite cursor: %w", err) + } + paginationParams.CompositeCursor = compositeCursor + default: decodedCursor, err := decodeInt64Cursor(cursor) if err != nil { return PaginationParams{}, fmt.Errorf("decoding cursor: %w", err) @@ -327,32 +350,69 @@ func parseStateChangeCursor(s *string) (*types.StateChangeCursor, error) { } parts := strings.Split(*decodedCursor, ":") - if len(parts) != 3 { - return nil, fmt.Errorf("invalid cursor format: %s (expected format: to_id:operation_id:state_change_order)", *s) + if len(parts) != 4 { + return nil, fmt.Errorf("invalid cursor format: %s (expected format: ledger_created_at_nano:to_id:operation_id:state_change_order)", *s) } - toID, err := strconv.ParseInt(parts[0], 10, 64) + nanos, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return nil, fmt.Errorf("parsing ledger_created_at: %w", err) + } + + toID, err := strconv.ParseInt(parts[1], 10, 64) if err != nil { return nil, fmt.Errorf("parsing to_id: %w", err) } - operationID, err := strconv.ParseInt(parts[1], 10, 64) + operationID, err := strconv.ParseInt(parts[2], 10, 64) if err != nil { return nil, fmt.Errorf("parsing operation_id: %w", err) } - stateChangeOrder, err := strconv.ParseInt(parts[2], 10, 64) + stateChangeOrder, err := strconv.ParseInt(parts[3], 10, 64) if err != nil { return nil, fmt.Errorf("parsing state_change_order: %w", err) } return &types.StateChangeCursor{ + LedgerCreatedAt: time.Unix(0, nanos), ToID: toID, OperationID: operationID, StateChangeOrder: stateChangeOrder, }, nil } +func parseCompositeCursor(s *string) (*types.CompositeCursor, error) { + if s == nil { + return nil, nil + } + + decodedCursor, err := decodeStringCursor(s) + if err != nil { + return nil, fmt.Errorf("decoding cursor: %w", err) + } + + parts := strings.Split(*decodedCursor, ":") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid cursor format: %s (expected format: ledger_created_at_nano:id)", *s) + } + + nanos, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return nil, fmt.Errorf("parsing ledger_created_at: %w", err) + } + + id, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("parsing id: %w", err) + } + + return &types.CompositeCursor{ + LedgerCreatedAt: time.Unix(0, nanos), + ID: id, + }, nil +} + func validatePaginationParams(first *int32, after *string, last *int32, before *string) error { if first != nil && last != nil { return fmt.Errorf("first and last cannot be used together") From 2d614a18b735ce1af6411e1279ccaf2618135113 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Sat, 14 Feb 2026 21:00:52 -0500 Subject: [PATCH 54/77] Decompose cursor comparison into individual OR clauses --- internal/data/operations.go | 42 +++++----- internal/data/query_utils.go | 55 +++++++++++++ internal/data/query_utils_test.go | 130 ++++++++++++++++++++++++++++++ internal/data/statechanges.go | 92 +++++++++++---------- internal/data/transactions.go | 46 ++++++----- 5 files changed, 283 insertions(+), 82 deletions(-) create mode 100644 internal/data/query_utils_test.go diff --git a/internal/data/operations.go b/internal/data/operations.go index b812f59f1..995e1db41 100644 --- a/internal/data/operations.go +++ b/internal/data/operations.go @@ -41,14 +41,21 @@ func (m *OperationModel) GetByID(ctx context.Context, id int64, columns string) func (m *OperationModel) GetAll(ctx context.Context, columns string, limit *int32, cursor *types.CompositeCursor, sortOrder SortOrder) ([]*types.OperationWithCursor, error) { columns = prepareColumnsWithID(columns, types.Operation{}, "", "id") queryBuilder := strings.Builder{} + var args []interface{} + argIndex := 1 + queryBuilder.WriteString(fmt.Sprintf(`SELECT %s, ledger_created_at as "cursor.cursor_ledger_created_at", id as "cursor.cursor_id" FROM operations`, columns)) + // Decomposed cursor pagination: expands ROW() tuple comparison into OR clauses so + // TimescaleDB ColumnarScan can push filters into vectorized batch processing. if cursor != nil { - if sortOrder == DESC { - queryBuilder.WriteString(fmt.Sprintf(` WHERE (ledger_created_at, id) < ('%s', %d)`, cursor.LedgerCreatedAt.Format(time.RFC3339Nano), cursor.ID)) - } else { - queryBuilder.WriteString(fmt.Sprintf(` WHERE (ledger_created_at, id) > ('%s', %d)`, cursor.LedgerCreatedAt.Format(time.RFC3339Nano), cursor.ID)) - } + clause, cursorArgs, nextIdx := buildDecomposedCursorCondition([]CursorColumn{ + {Name: "ledger_created_at", Value: cursor.LedgerCreatedAt}, + {Name: "id", Value: cursor.ID}, + }, sortOrder, argIndex) + queryBuilder.WriteString(" WHERE " + clause) + args = append(args, cursorArgs...) + argIndex = nextIdx } if sortOrder == DESC { @@ -58,7 +65,8 @@ func (m *OperationModel) GetAll(ctx context.Context, columns string, limit *int3 } if limit != nil { - queryBuilder.WriteString(fmt.Sprintf(" LIMIT %d", *limit)) + queryBuilder.WriteString(fmt.Sprintf(" LIMIT $%d", argIndex)) + args = append(args, *limit) } query := queryBuilder.String() if sortOrder == DESC { @@ -67,7 +75,7 @@ func (m *OperationModel) GetAll(ctx context.Context, columns string, limit *int3 var operations []*types.OperationWithCursor start := time.Now() - err := m.DB.SelectContext(ctx, &operations, query) + err := m.DB.SelectContext(ctx, &operations, query, args...) duration := time.Since(start).Seconds() m.MetricsService.ObserveDBQueryDuration("GetAll", "operations", duration) if err != nil { @@ -224,17 +232,16 @@ func (m *OperationModel) BatchGetByAccountAddress(ctx context.Context, accountAd FROM operations_accounts WHERE account_id = $1`) - // Add cursor-based pagination on the CTE + // Decomposed cursor pagination: expands ROW() tuple comparison into OR clauses so + // TimescaleDB ColumnarScan can push filters into vectorized batch processing. if cursor != nil { - if orderBy == DESC { - queryBuilder.WriteString(fmt.Sprintf(` - AND (ledger_created_at, operation_id) < ($%d, $%d)`, argIndex, argIndex+1)) - } else { - queryBuilder.WriteString(fmt.Sprintf(` - AND (ledger_created_at, operation_id) > ($%d, $%d)`, argIndex, argIndex+1)) - } - args = append(args, cursor.LedgerCreatedAt, cursor.ID) - argIndex += 2 + clause, cursorArgs, nextIdx := buildDecomposedCursorCondition([]CursorColumn{ + {Name: "ledger_created_at", Value: cursor.LedgerCreatedAt}, + {Name: "operation_id", Value: cursor.ID}, + }, orderBy, argIndex) + queryBuilder.WriteString("\n\t\t\tAND " + clause) + args = append(args, cursorArgs...) + argIndex = nextIdx } if orderBy == DESC { @@ -248,7 +255,6 @@ func (m *OperationModel) BatchGetByAccountAddress(ctx context.Context, accountAd if limit != nil { queryBuilder.WriteString(fmt.Sprintf(` LIMIT $%d`, argIndex)) args = append(args, *limit) - argIndex++ } // Close CTE and LATERAL join to fetch full operation rows diff --git a/internal/data/query_utils.go b/internal/data/query_utils.go index 4610d8d5a..148649e40 100644 --- a/internal/data/query_utils.go +++ b/internal/data/query_utils.go @@ -62,6 +62,61 @@ func pgtypeBytesFromNullAddressBytea(na types.NullAddressBytea) ([]byte, error) return val.([]byte), nil } +// CursorColumn represents a column name and its cursor value for decomposed pagination. +type CursorColumn struct { + Name string + Value interface{} +} + +// BuildDecomposedCursorCondition decomposes a ROW() tuple comparison into an equivalent +// OR clause that TimescaleDB's ColumnarScan can push into vectorized filters. +// +// For example, (a, b, c) < ($1, $2, $3) becomes: +// +// (a < $1 OR (a = $1 AND b < $2) OR (a = $1 AND b = $2 AND c < $3)) +// +// DESC uses "<", ASC uses ">". Returns the clause string, args slice, and next arg index. +func buildDecomposedCursorCondition(columns []CursorColumn, sortOrder SortOrder, startArgIndex int) (string, []interface{}, int) { + if len(columns) == 0 { + return "", nil, startArgIndex + } + + op := "<" + if sortOrder == ASC { + op = ">" + } + + argIdx := startArgIndex + var args []interface{} + var orParts []string + + for i := range columns { + var parts []string + // Add equality conditions for all preceding columns + for j := 0; j < i; j++ { + parts = append(parts, fmt.Sprintf("%s = $%d", columns[j].Name, argIdx)) + args = append(args, columns[j].Value) + argIdx++ + } + // Add the comparison condition for the current column + parts = append(parts, fmt.Sprintf("%s %s $%d", columns[i].Name, op, argIdx)) + args = append(args, columns[i].Value) + argIdx++ + + orParts = append(orParts, strings.Join(parts, " AND ")) + } + + // Wrap each OR branch in parens if it has multiple conditions + for i, part := range orParts { + if i > 0 { + orParts[i] = "(" + part + ")" + } + } + + clause := "(" + strings.Join(orParts, " OR ") + ")" + return clause, args, argIdx +} + func getDBColumns(model any) set.Set[string] { modelType := reflect.TypeOf(model) dbColumns := set.NewSet[string]() diff --git a/internal/data/query_utils_test.go b/internal/data/query_utils_test.go new file mode 100644 index 000000000..cf75f5faf --- /dev/null +++ b/internal/data/query_utils_test.go @@ -0,0 +1,130 @@ +// Tests for buildDecomposedCursorCondition which decomposes ROW() tuple comparisons +// into OR clauses for TimescaleDB vectorized filtering on compressed hypertables. +package data + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func Test_buildDecomposedCursorCondition(t *testing.T) { + testTime := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC) + + tests := []struct { + name string + columns []CursorColumn + sortOrder SortOrder + startArgIndex int + wantClause string + wantArgs []interface{} + wantNextIndex int + }{ + { + name: "empty columns returns empty", + columns: []CursorColumn{}, + sortOrder: DESC, + startArgIndex: 1, + wantClause: "", + wantArgs: nil, + wantNextIndex: 1, + }, + { + name: "single column DESC", + columns: []CursorColumn{ + {Name: "ledger_created_at", Value: testTime}, + }, + sortOrder: DESC, + startArgIndex: 1, + wantClause: "(ledger_created_at < $1)", + wantArgs: []interface{}{testTime}, + wantNextIndex: 2, + }, + { + name: "single column ASC", + columns: []CursorColumn{ + {Name: "id", Value: int64(42)}, + }, + sortOrder: ASC, + startArgIndex: 3, + wantClause: "(id > $3)", + wantArgs: []interface{}{int64(42)}, + wantNextIndex: 4, + }, + { + name: "two columns DESC", + columns: []CursorColumn{ + {Name: "ledger_created_at", Value: testTime}, + {Name: "to_id", Value: int64(100)}, + }, + sortOrder: DESC, + startArgIndex: 1, + wantClause: "(ledger_created_at < $1 OR (ledger_created_at = $2 AND to_id < $3))", + wantArgs: []interface{}{testTime, testTime, int64(100)}, + wantNextIndex: 4, + }, + { + name: "two columns ASC", + columns: []CursorColumn{ + {Name: "ledger_created_at", Value: testTime}, + {Name: "id", Value: int64(50)}, + }, + sortOrder: ASC, + startArgIndex: 2, + wantClause: "(ledger_created_at > $2 OR (ledger_created_at = $3 AND id > $4))", + wantArgs: []interface{}{testTime, testTime, int64(50)}, + wantNextIndex: 5, + }, + { + name: "three columns DESC", + columns: []CursorColumn{ + {Name: "to_id", Value: int64(10)}, + {Name: "operation_id", Value: int64(20)}, + {Name: "state_change_order", Value: int64(30)}, + }, + sortOrder: DESC, + startArgIndex: 2, + wantClause: "(to_id < $2 OR (to_id = $3 AND operation_id < $4) OR (to_id = $5 AND operation_id = $6 AND state_change_order < $7))", + wantArgs: []interface{}{int64(10), int64(10), int64(20), int64(10), int64(20), int64(30)}, + wantNextIndex: 8, + }, + { + name: "four columns DESC", + columns: []CursorColumn{ + {Name: "ledger_created_at", Value: testTime}, + {Name: "to_id", Value: int64(10)}, + {Name: "operation_id", Value: int64(20)}, + {Name: "state_change_order", Value: int64(30)}, + }, + sortOrder: DESC, + startArgIndex: 1, + wantClause: "(ledger_created_at < $1 OR (ledger_created_at = $2 AND to_id < $3) OR (ledger_created_at = $4 AND to_id = $5 AND operation_id < $6) OR (ledger_created_at = $7 AND to_id = $8 AND operation_id = $9 AND state_change_order < $10))", + wantArgs: []interface{}{testTime, testTime, int64(10), testTime, int64(10), int64(20), testTime, int64(10), int64(20), int64(30)}, + wantNextIndex: 11, + }, + { + name: "four columns ASC", + columns: []CursorColumn{ + {Name: "ledger_created_at", Value: testTime}, + {Name: "to_id", Value: int64(10)}, + {Name: "operation_id", Value: int64(20)}, + {Name: "state_change_order", Value: int64(30)}, + }, + sortOrder: ASC, + startArgIndex: 5, + wantClause: "(ledger_created_at > $5 OR (ledger_created_at = $6 AND to_id > $7) OR (ledger_created_at = $8 AND to_id = $9 AND operation_id > $10) OR (ledger_created_at = $11 AND to_id = $12 AND operation_id = $13 AND state_change_order > $14))", + wantArgs: []interface{}{testTime, testTime, int64(10), testTime, int64(10), int64(20), testTime, int64(10), int64(20), int64(30)}, + wantNextIndex: 15, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotClause, gotArgs, gotNextIndex := buildDecomposedCursorCondition(tt.columns, tt.sortOrder, tt.startArgIndex) + assert.Equal(t, tt.wantClause, gotClause) + assert.Equal(t, tt.wantArgs, gotArgs) + assert.Equal(t, tt.wantNextIndex, gotNextIndex) + }) + } +} diff --git a/internal/data/statechanges.go b/internal/data/statechanges.go index a298cd5f9..543f72a1f 100644 --- a/internal/data/statechanges.go +++ b/internal/data/statechanges.go @@ -63,22 +63,18 @@ func (m *StateChangeModel) BatchGetByAccountAddress(ctx context.Context, account argIndex++ } - // Add cursor-based pagination using 4-column comparison with ledger_created_at as the - // leading column. This enables TimescaleDB ChunkAppend optimization on hypertables. + // Decomposed cursor pagination: expands ROW() tuple comparison into OR clauses so + // TimescaleDB ColumnarScan can push filters into vectorized batch processing. if cursor != nil { - if sortOrder == DESC { - queryBuilder.WriteString(fmt.Sprintf(` - AND (ledger_created_at, to_id, operation_id, state_change_order) < ($%d, $%d, $%d, $%d) - `, argIndex, argIndex+1, argIndex+2, argIndex+3)) - args = append(args, cursor.LedgerCreatedAt, cursor.ToID, cursor.OperationID, cursor.StateChangeOrder) - argIndex += 4 - } else { - queryBuilder.WriteString(fmt.Sprintf(` - AND (ledger_created_at, to_id, operation_id, state_change_order) > ($%d, $%d, $%d, $%d) - `, argIndex, argIndex+1, argIndex+2, argIndex+3)) - args = append(args, cursor.LedgerCreatedAt, cursor.ToID, cursor.OperationID, cursor.StateChangeOrder) - argIndex += 4 - } + clause, cursorArgs, nextIdx := buildDecomposedCursorCondition([]CursorColumn{ + {Name: "ledger_created_at", Value: cursor.LedgerCreatedAt}, + {Name: "to_id", Value: cursor.ToID}, + {Name: "operation_id", Value: cursor.OperationID}, + {Name: "state_change_order", Value: cursor.StateChangeOrder}, + }, sortOrder, argIndex) + queryBuilder.WriteString(" AND " + clause) + args = append(args, cursorArgs...) + argIndex = nextIdx } // Add ordering with ledger_created_at as leading column for TimescaleDB ChunkAppend @@ -119,21 +115,26 @@ func (m *StateChangeModel) BatchGetByAccountAddress(ctx context.Context, account func (m *StateChangeModel) GetAll(ctx context.Context, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_order") var queryBuilder strings.Builder + var args []interface{} + argIndex := 1 + queryBuilder.WriteString(fmt.Sprintf(` SELECT %s, ledger_created_at as "cursor.cursor_ledger_created_at", to_id as "cursor.cursor_to_id", operation_id as "cursor.cursor_operation_id", state_change_order as "cursor.cursor_state_change_order" FROM state_changes `, columns)) + // Decomposed cursor pagination: expands ROW() tuple comparison into OR clauses so + // TimescaleDB ColumnarScan can push filters into vectorized batch processing. if cursor != nil { - if sortOrder == DESC { - queryBuilder.WriteString(fmt.Sprintf(` - WHERE (ledger_created_at, to_id, operation_id, state_change_order) < ('%s', %d, %d, %d) - `, cursor.LedgerCreatedAt.Format(time.RFC3339Nano), cursor.ToID, cursor.OperationID, cursor.StateChangeOrder)) - } else { - queryBuilder.WriteString(fmt.Sprintf(` - WHERE (ledger_created_at, to_id, operation_id, state_change_order) > ('%s', %d, %d, %d) - `, cursor.LedgerCreatedAt.Format(time.RFC3339Nano), cursor.ToID, cursor.OperationID, cursor.StateChangeOrder)) - } + clause, cursorArgs, nextIdx := buildDecomposedCursorCondition([]CursorColumn{ + {Name: "ledger_created_at", Value: cursor.LedgerCreatedAt}, + {Name: "to_id", Value: cursor.ToID}, + {Name: "operation_id", Value: cursor.OperationID}, + {Name: "state_change_order", Value: cursor.StateChangeOrder}, + }, sortOrder, argIndex) + queryBuilder.WriteString(" WHERE " + clause) + args = append(args, cursorArgs...) + argIndex = nextIdx } // Order with ledger_created_at as leading column for TimescaleDB ChunkAppend @@ -144,7 +145,8 @@ func (m *StateChangeModel) GetAll(ctx context.Context, columns string, limit *in } if limit != nil && *limit > 0 { - queryBuilder.WriteString(fmt.Sprintf(" LIMIT %d", *limit)) + queryBuilder.WriteString(fmt.Sprintf(" LIMIT $%d", argIndex)) + args = append(args, *limit) } query := queryBuilder.String() @@ -158,7 +160,7 @@ func (m *StateChangeModel) GetAll(ctx context.Context, columns string, limit *in var stateChanges []*types.StateChangeWithCursor start := time.Now() - err := m.DB.SelectContext(ctx, &stateChanges, query) + err := m.DB.SelectContext(ctx, &stateChanges, query, args...) duration := time.Since(start).Seconds() m.MetricsService.ObserveDBQueryDuration("GetAll", "state_changes", duration) if err != nil { @@ -295,16 +297,17 @@ func (m *StateChangeModel) BatchGetByToID(ctx context.Context, toID int64, colum args := []interface{}{toID} argIndex := 2 + // Decomposed cursor pagination: expands ROW() tuple comparison into OR clauses so + // TimescaleDB ColumnarScan can push filters into vectorized batch processing. if cursor != nil { - if sortOrder == DESC { - queryBuilder.WriteString(fmt.Sprintf(` - AND (to_id, operation_id, state_change_order) < (%d, %d, %d) - `, cursor.ToID, cursor.OperationID, cursor.StateChangeOrder)) - } else { - queryBuilder.WriteString(fmt.Sprintf(` - AND (to_id, operation_id, state_change_order) > (%d, %d, %d) - `, cursor.ToID, cursor.OperationID, cursor.StateChangeOrder)) - } + clause, cursorArgs, nextIdx := buildDecomposedCursorCondition([]CursorColumn{ + {Name: "to_id", Value: cursor.ToID}, + {Name: "operation_id", Value: cursor.OperationID}, + {Name: "state_change_order", Value: cursor.StateChangeOrder}, + }, sortOrder, argIndex) + queryBuilder.WriteString(" AND " + clause) + args = append(args, cursorArgs...) + argIndex = nextIdx } if sortOrder == DESC { @@ -405,16 +408,17 @@ func (m *StateChangeModel) BatchGetByOperationID(ctx context.Context, operationI args := []interface{}{operationID} argIndex := 2 + // Decomposed cursor pagination: expands ROW() tuple comparison into OR clauses so + // TimescaleDB ColumnarScan can push filters into vectorized batch processing. if cursor != nil { - if sortOrder == DESC { - queryBuilder.WriteString(fmt.Sprintf(` - AND (to_id, operation_id, state_change_order) < (%d, %d, %d) - `, cursor.ToID, cursor.OperationID, cursor.StateChangeOrder)) - } else { - queryBuilder.WriteString(fmt.Sprintf(` - AND (to_id, operation_id, state_change_order) > (%d, %d, %d) - `, cursor.ToID, cursor.OperationID, cursor.StateChangeOrder)) - } + clause, cursorArgs, nextIdx := buildDecomposedCursorCondition([]CursorColumn{ + {Name: "to_id", Value: cursor.ToID}, + {Name: "operation_id", Value: cursor.OperationID}, + {Name: "state_change_order", Value: cursor.StateChangeOrder}, + }, sortOrder, argIndex) + queryBuilder.WriteString(" AND " + clause) + args = append(args, cursorArgs...) + argIndex = nextIdx } if sortOrder == DESC { diff --git a/internal/data/transactions.go b/internal/data/transactions.go index 8f525d208..d8b8fbfcb 100644 --- a/internal/data/transactions.go +++ b/internal/data/transactions.go @@ -42,14 +42,21 @@ func (m *TransactionModel) GetByHash(ctx context.Context, hash string, columns s func (m *TransactionModel) GetAll(ctx context.Context, columns string, limit *int32, cursor *types.CompositeCursor, sortOrder SortOrder) ([]*types.TransactionWithCursor, error) { columns = prepareColumnsWithID(columns, types.Transaction{}, "", "to_id") queryBuilder := strings.Builder{} + var args []interface{} + argIndex := 1 + queryBuilder.WriteString(fmt.Sprintf(`SELECT %s, ledger_created_at as "cursor.cursor_ledger_created_at", to_id as "cursor.cursor_id" FROM transactions`, columns)) + // Decomposed cursor pagination: expands ROW() tuple comparison into OR clauses so + // TimescaleDB ColumnarScan can push filters into vectorized batch processing. if cursor != nil { - if sortOrder == DESC { - queryBuilder.WriteString(fmt.Sprintf(` WHERE (ledger_created_at, to_id) < ('%s', %d)`, cursor.LedgerCreatedAt.Format(time.RFC3339Nano), cursor.ID)) - } else { - queryBuilder.WriteString(fmt.Sprintf(` WHERE (ledger_created_at, to_id) > ('%s', %d)`, cursor.LedgerCreatedAt.Format(time.RFC3339Nano), cursor.ID)) - } + clause, cursorArgs, nextIdx := buildDecomposedCursorCondition([]CursorColumn{ + {Name: "ledger_created_at", Value: cursor.LedgerCreatedAt}, + {Name: "to_id", Value: cursor.ID}, + }, sortOrder, argIndex) + queryBuilder.WriteString(" WHERE " + clause) + args = append(args, cursorArgs...) + argIndex = nextIdx } if sortOrder == DESC { @@ -59,7 +66,8 @@ func (m *TransactionModel) GetAll(ctx context.Context, columns string, limit *in } if limit != nil { - queryBuilder.WriteString(fmt.Sprintf(" LIMIT %d", *limit)) + queryBuilder.WriteString(fmt.Sprintf(" LIMIT $%d", argIndex)) + args = append(args, *limit) } query := queryBuilder.String() @@ -69,7 +77,7 @@ func (m *TransactionModel) GetAll(ctx context.Context, columns string, limit *in var transactions []*types.TransactionWithCursor start := time.Now() - err := m.DB.SelectContext(ctx, &transactions, query) + err := m.DB.SelectContext(ctx, &transactions, query, args...) duration := time.Since(start).Seconds() m.MetricsService.ObserveDBQueryDuration("GetAll", "transactions", duration) if err != nil { @@ -92,23 +100,22 @@ func (m *TransactionModel) BatchGetByAccountAddress(ctx context.Context, account // MATERIALIZED CTE scans transactions_accounts with ledger_created_at leading the ORDER BY, // enabling TimescaleDB ChunkAppend on the hypertable. - queryBuilder.WriteString(fmt.Sprintf(` + queryBuilder.WriteString(` WITH account_txns AS MATERIALIZED ( SELECT tx_to_id, ledger_created_at FROM transactions_accounts - WHERE account_id = $1`)) + WHERE account_id = $1`) - // Add cursor-based pagination on the CTE + // Decomposed cursor pagination: expands ROW() tuple comparison into OR clauses so + // TimescaleDB ColumnarScan can push filters into vectorized batch processing. if cursor != nil { - if orderBy == DESC { - queryBuilder.WriteString(fmt.Sprintf(` - AND (ledger_created_at, tx_to_id) < ($%d, $%d)`, argIndex, argIndex+1)) - } else { - queryBuilder.WriteString(fmt.Sprintf(` - AND (ledger_created_at, tx_to_id) > ($%d, $%d)`, argIndex, argIndex+1)) - } - args = append(args, cursor.LedgerCreatedAt, cursor.ID) - argIndex += 2 + clause, cursorArgs, nextIdx := buildDecomposedCursorCondition([]CursorColumn{ + {Name: "ledger_created_at", Value: cursor.LedgerCreatedAt}, + {Name: "tx_to_id", Value: cursor.ID}, + }, orderBy, argIndex) + queryBuilder.WriteString("\n\t\t\tAND " + clause) + args = append(args, cursorArgs...) + argIndex = nextIdx } if orderBy == DESC { @@ -122,7 +129,6 @@ func (m *TransactionModel) BatchGetByAccountAddress(ctx context.Context, account if limit != nil { queryBuilder.WriteString(fmt.Sprintf(` LIMIT $%d`, argIndex)) args = append(args, *limit) - argIndex++ } // Close CTE and LATERAL join to fetch full transaction rows From 98929609571cecd9b1fe6d48e97e2a2dc23c3850 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Sun, 15 Feb 2026 10:09:50 -0500 Subject: [PATCH 55/77] Add since/until time range params to Account GraphQL schema Add optional since: Time and until: Time parameters to transactions, operations, and stateChanges fields on the Account type. These enable TimescaleDB chunk pruning on the ledger_created_at hypertable partition column for significant query performance improvements. --- internal/serve/graphql/schema/account.graphqls | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/serve/graphql/schema/account.graphqls b/internal/serve/graphql/schema/account.graphqls index cc2eba2a7..818eba2dd 100644 --- a/internal/serve/graphql/schema/account.graphqls +++ b/internal/serve/graphql/schema/account.graphqls @@ -5,18 +5,22 @@ type Account{ # GraphQL Relationships - these fields use resolvers for data fetching # Each relationship resolver will be called when the field is requested - + # All transactions associated with this account - transactions(first: Int, after: String, last: Int, before: String): TransactionConnection - + # Optional since/until params enable TimescaleDB chunk pruning on ledger_created_at + transactions(since: Time, until: Time, first: Int, after: String, last: Int, before: String): TransactionConnection + # All operations associated with this account - operations(first: Int, after: String, last: Int, before: String): OperationConnection - + # Optional since/until params enable TimescaleDB chunk pruning on ledger_created_at + operations(since: Time, until: Time, first: Int, after: String, last: Int, before: String): OperationConnection + # All state changes associated with this account # Uses resolver to fetch related state changes # Optional filter parameter allows filtering by transaction hash and/or operation ID + # Optional since/until params enable TimescaleDB chunk pruning on ledger_created_at stateChanges( filter: AccountStateChangeFilterInput + since: Time, until: Time first: Int, after: String, last: Int, before: String ): StateChangeConnection } From 52f06d38355198afa8af8ef900f9e476391abec7 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Sun, 15 Feb 2026 10:10:27 -0500 Subject: [PATCH 56/77] Regenerate gqlgen code with since/until resolver params gqlgen automatically updated resolver signatures to include since *time.Time and until *time.Time parameters. --- internal/serve/graphql/generated/generated.go | 217 ++++++++++++++---- .../graphql/resolvers/account.resolvers.go | 7 +- 2 files changed, 177 insertions(+), 47 deletions(-) diff --git a/internal/serve/graphql/generated/generated.go b/internal/serve/graphql/generated/generated.go index 836693bd5..4b88427a7 100644 --- a/internal/serve/graphql/generated/generated.go +++ b/internal/serve/graphql/generated/generated.go @@ -14,11 +14,10 @@ import ( "github.com/99designs/gqlgen/graphql" "github.com/99designs/gqlgen/graphql/introspection" - gqlparser "github.com/vektah/gqlparser/v2" - "github.com/vektah/gqlparser/v2/ast" - "github.com/stellar/wallet-backend/internal/indexer/types" "github.com/stellar/wallet-backend/internal/serve/graphql/scalars" + gqlparser "github.com/vektah/gqlparser/v2" + "github.com/vektah/gqlparser/v2/ast" ) // region ************************** generated!.gotpl ************************** @@ -63,9 +62,9 @@ type DirectiveRoot struct { type ComplexityRoot struct { Account struct { Address func(childComplexity int) int - Operations func(childComplexity int, first *int32, after *string, last *int32, before *string) int - StateChanges func(childComplexity int, filter *AccountStateChangeFilterInput, first *int32, after *string, last *int32, before *string) int - Transactions func(childComplexity int, first *int32, after *string, last *int32, before *string) int + Operations func(childComplexity int, since *time.Time, until *time.Time, first *int32, after *string, last *int32, before *string) int + StateChanges func(childComplexity int, filter *AccountStateChangeFilterInput, since *time.Time, until *time.Time, first *int32, after *string, last *int32, before *string) int + Transactions func(childComplexity int, since *time.Time, until *time.Time, first *int32, after *string, last *int32, before *string) int } AccountBalances struct { @@ -346,9 +345,9 @@ type ComplexityRoot struct { type AccountResolver interface { Address(ctx context.Context, obj *types.Account) (string, error) - Transactions(ctx context.Context, obj *types.Account, first *int32, after *string, last *int32, before *string) (*TransactionConnection, error) - Operations(ctx context.Context, obj *types.Account, first *int32, after *string, last *int32, before *string) (*OperationConnection, error) - StateChanges(ctx context.Context, obj *types.Account, filter *AccountStateChangeFilterInput, first *int32, after *string, last *int32, before *string) (*StateChangeConnection, error) + Transactions(ctx context.Context, obj *types.Account, since *time.Time, until *time.Time, first *int32, after *string, last *int32, before *string) (*TransactionConnection, error) + Operations(ctx context.Context, obj *types.Account, since *time.Time, until *time.Time, first *int32, after *string, last *int32, before *string) (*OperationConnection, error) + StateChanges(ctx context.Context, obj *types.Account, filter *AccountStateChangeFilterInput, since *time.Time, until *time.Time, first *int32, after *string, last *int32, before *string) (*StateChangeConnection, error) } type AccountChangeResolver interface { Type(ctx context.Context, obj *types.AccountStateChangeModel) (types.StateChangeCategory, error) @@ -509,7 +508,7 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Account.Operations(childComplexity, args["first"].(*int32), args["after"].(*string), args["last"].(*int32), args["before"].(*string)), true + return e.complexity.Account.Operations(childComplexity, args["since"].(*time.Time), args["until"].(*time.Time), args["first"].(*int32), args["after"].(*string), args["last"].(*int32), args["before"].(*string)), true case "Account.stateChanges": if e.complexity.Account.StateChanges == nil { @@ -521,7 +520,7 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Account.StateChanges(childComplexity, args["filter"].(*AccountStateChangeFilterInput), args["first"].(*int32), args["after"].(*string), args["last"].(*int32), args["before"].(*string)), true + return e.complexity.Account.StateChanges(childComplexity, args["filter"].(*AccountStateChangeFilterInput), args["since"].(*time.Time), args["until"].(*time.Time), args["first"].(*int32), args["after"].(*string), args["last"].(*int32), args["before"].(*string)), true case "Account.transactions": if e.complexity.Account.Transactions == nil { @@ -533,7 +532,7 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin return 0, false } - return e.complexity.Account.Transactions(childComplexity, args["first"].(*int32), args["after"].(*string), args["last"].(*int32), args["before"].(*string)), true + return e.complexity.Account.Transactions(childComplexity, args["since"].(*time.Time), args["until"].(*time.Time), args["first"].(*int32), args["after"].(*string), args["last"].(*int32), args["before"].(*string)), true case "AccountBalances.address": if e.complexity.AccountBalances.Address == nil { @@ -2044,18 +2043,22 @@ type Account{ # GraphQL Relationships - these fields use resolvers for data fetching # Each relationship resolver will be called when the field is requested - + # All transactions associated with this account - transactions(first: Int, after: String, last: Int, before: String): TransactionConnection - + # Optional since/until params enable TimescaleDB chunk pruning on ledger_created_at + transactions(since: Time, until: Time, first: Int, after: String, last: Int, before: String): TransactionConnection + # All operations associated with this account - operations(first: Int, after: String, last: Int, before: String): OperationConnection - + # Optional since/until params enable TimescaleDB chunk pruning on ledger_created_at + operations(since: Time, until: Time, first: Int, after: String, last: Int, before: String): OperationConnection + # All state changes associated with this account # Uses resolver to fetch related state changes # Optional filter parameter allows filtering by transaction hash and/or operation ID + # Optional since/until params enable TimescaleDB chunk pruning on ledger_created_at stateChanges( filter: AccountStateChangeFilterInput + since: Time, until: Time first: Int, after: String, last: Int, before: String ): StateChangeConnection } @@ -2587,28 +2590,64 @@ var parsedSchema = gqlparser.MustLoadSchema(sources...) func (ec *executionContext) field_Account_operations_args(ctx context.Context, rawArgs map[string]any) (map[string]any, error) { var err error args := map[string]any{} - arg0, err := ec.field_Account_operations_argsFirst(ctx, rawArgs) + arg0, err := ec.field_Account_operations_argsSince(ctx, rawArgs) if err != nil { return nil, err } - args["first"] = arg0 - arg1, err := ec.field_Account_operations_argsAfter(ctx, rawArgs) + args["since"] = arg0 + arg1, err := ec.field_Account_operations_argsUntil(ctx, rawArgs) if err != nil { return nil, err } - args["after"] = arg1 - arg2, err := ec.field_Account_operations_argsLast(ctx, rawArgs) + args["until"] = arg1 + arg2, err := ec.field_Account_operations_argsFirst(ctx, rawArgs) if err != nil { return nil, err } - args["last"] = arg2 - arg3, err := ec.field_Account_operations_argsBefore(ctx, rawArgs) + args["first"] = arg2 + arg3, err := ec.field_Account_operations_argsAfter(ctx, rawArgs) if err != nil { return nil, err } - args["before"] = arg3 + args["after"] = arg3 + arg4, err := ec.field_Account_operations_argsLast(ctx, rawArgs) + if err != nil { + return nil, err + } + args["last"] = arg4 + arg5, err := ec.field_Account_operations_argsBefore(ctx, rawArgs) + if err != nil { + return nil, err + } + args["before"] = arg5 return args, nil } +func (ec *executionContext) field_Account_operations_argsSince( + ctx context.Context, + rawArgs map[string]any, +) (*time.Time, error) { + ctx = graphql.WithPathContext(ctx, graphql.NewPathWithField("since")) + if tmp, ok := rawArgs["since"]; ok { + return ec.unmarshalOTime2ᚖtimeᚐTime(ctx, tmp) + } + + var zeroVal *time.Time + return zeroVal, nil +} + +func (ec *executionContext) field_Account_operations_argsUntil( + ctx context.Context, + rawArgs map[string]any, +) (*time.Time, error) { + ctx = graphql.WithPathContext(ctx, graphql.NewPathWithField("until")) + if tmp, ok := rawArgs["until"]; ok { + return ec.unmarshalOTime2ᚖtimeᚐTime(ctx, tmp) + } + + var zeroVal *time.Time + return zeroVal, nil +} + func (ec *executionContext) field_Account_operations_argsFirst( ctx context.Context, rawArgs map[string]any, @@ -2669,26 +2708,36 @@ func (ec *executionContext) field_Account_stateChanges_args(ctx context.Context, return nil, err } args["filter"] = arg0 - arg1, err := ec.field_Account_stateChanges_argsFirst(ctx, rawArgs) + arg1, err := ec.field_Account_stateChanges_argsSince(ctx, rawArgs) if err != nil { return nil, err } - args["first"] = arg1 - arg2, err := ec.field_Account_stateChanges_argsAfter(ctx, rawArgs) + args["since"] = arg1 + arg2, err := ec.field_Account_stateChanges_argsUntil(ctx, rawArgs) if err != nil { return nil, err } - args["after"] = arg2 - arg3, err := ec.field_Account_stateChanges_argsLast(ctx, rawArgs) + args["until"] = arg2 + arg3, err := ec.field_Account_stateChanges_argsFirst(ctx, rawArgs) if err != nil { return nil, err } - args["last"] = arg3 - arg4, err := ec.field_Account_stateChanges_argsBefore(ctx, rawArgs) + args["first"] = arg3 + arg4, err := ec.field_Account_stateChanges_argsAfter(ctx, rawArgs) if err != nil { return nil, err } - args["before"] = arg4 + args["after"] = arg4 + arg5, err := ec.field_Account_stateChanges_argsLast(ctx, rawArgs) + if err != nil { + return nil, err + } + args["last"] = arg5 + arg6, err := ec.field_Account_stateChanges_argsBefore(ctx, rawArgs) + if err != nil { + return nil, err + } + args["before"] = arg6 return args, nil } func (ec *executionContext) field_Account_stateChanges_argsFilter( @@ -2704,6 +2753,32 @@ func (ec *executionContext) field_Account_stateChanges_argsFilter( return zeroVal, nil } +func (ec *executionContext) field_Account_stateChanges_argsSince( + ctx context.Context, + rawArgs map[string]any, +) (*time.Time, error) { + ctx = graphql.WithPathContext(ctx, graphql.NewPathWithField("since")) + if tmp, ok := rawArgs["since"]; ok { + return ec.unmarshalOTime2ᚖtimeᚐTime(ctx, tmp) + } + + var zeroVal *time.Time + return zeroVal, nil +} + +func (ec *executionContext) field_Account_stateChanges_argsUntil( + ctx context.Context, + rawArgs map[string]any, +) (*time.Time, error) { + ctx = graphql.WithPathContext(ctx, graphql.NewPathWithField("until")) + if tmp, ok := rawArgs["until"]; ok { + return ec.unmarshalOTime2ᚖtimeᚐTime(ctx, tmp) + } + + var zeroVal *time.Time + return zeroVal, nil +} + func (ec *executionContext) field_Account_stateChanges_argsFirst( ctx context.Context, rawArgs map[string]any, @@ -2759,28 +2834,64 @@ func (ec *executionContext) field_Account_stateChanges_argsBefore( func (ec *executionContext) field_Account_transactions_args(ctx context.Context, rawArgs map[string]any) (map[string]any, error) { var err error args := map[string]any{} - arg0, err := ec.field_Account_transactions_argsFirst(ctx, rawArgs) + arg0, err := ec.field_Account_transactions_argsSince(ctx, rawArgs) if err != nil { return nil, err } - args["first"] = arg0 - arg1, err := ec.field_Account_transactions_argsAfter(ctx, rawArgs) + args["since"] = arg0 + arg1, err := ec.field_Account_transactions_argsUntil(ctx, rawArgs) if err != nil { return nil, err } - args["after"] = arg1 - arg2, err := ec.field_Account_transactions_argsLast(ctx, rawArgs) + args["until"] = arg1 + arg2, err := ec.field_Account_transactions_argsFirst(ctx, rawArgs) if err != nil { return nil, err } - args["last"] = arg2 - arg3, err := ec.field_Account_transactions_argsBefore(ctx, rawArgs) + args["first"] = arg2 + arg3, err := ec.field_Account_transactions_argsAfter(ctx, rawArgs) if err != nil { return nil, err } - args["before"] = arg3 + args["after"] = arg3 + arg4, err := ec.field_Account_transactions_argsLast(ctx, rawArgs) + if err != nil { + return nil, err + } + args["last"] = arg4 + arg5, err := ec.field_Account_transactions_argsBefore(ctx, rawArgs) + if err != nil { + return nil, err + } + args["before"] = arg5 return args, nil } +func (ec *executionContext) field_Account_transactions_argsSince( + ctx context.Context, + rawArgs map[string]any, +) (*time.Time, error) { + ctx = graphql.WithPathContext(ctx, graphql.NewPathWithField("since")) + if tmp, ok := rawArgs["since"]; ok { + return ec.unmarshalOTime2ᚖtimeᚐTime(ctx, tmp) + } + + var zeroVal *time.Time + return zeroVal, nil +} + +func (ec *executionContext) field_Account_transactions_argsUntil( + ctx context.Context, + rawArgs map[string]any, +) (*time.Time, error) { + ctx = graphql.WithPathContext(ctx, graphql.NewPathWithField("until")) + if tmp, ok := rawArgs["until"]; ok { + return ec.unmarshalOTime2ᚖtimeᚐTime(ctx, tmp) + } + + var zeroVal *time.Time + return zeroVal, nil +} + func (ec *executionContext) field_Account_transactions_argsFirst( ctx context.Context, rawArgs map[string]any, @@ -3683,7 +3794,7 @@ func (ec *executionContext) _Account_transactions(ctx context.Context, field gra }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { ctx = rctx // use context from middleware stack in children - return ec.resolvers.Account().Transactions(rctx, obj, fc.Args["first"].(*int32), fc.Args["after"].(*string), fc.Args["last"].(*int32), fc.Args["before"].(*string)) + return ec.resolvers.Account().Transactions(rctx, obj, fc.Args["since"].(*time.Time), fc.Args["until"].(*time.Time), fc.Args["first"].(*int32), fc.Args["after"].(*string), fc.Args["last"].(*int32), fc.Args["before"].(*string)) }) if err != nil { ec.Error(ctx, err) @@ -3741,7 +3852,7 @@ func (ec *executionContext) _Account_operations(ctx context.Context, field graph }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { ctx = rctx // use context from middleware stack in children - return ec.resolvers.Account().Operations(rctx, obj, fc.Args["first"].(*int32), fc.Args["after"].(*string), fc.Args["last"].(*int32), fc.Args["before"].(*string)) + return ec.resolvers.Account().Operations(rctx, obj, fc.Args["since"].(*time.Time), fc.Args["until"].(*time.Time), fc.Args["first"].(*int32), fc.Args["after"].(*string), fc.Args["last"].(*int32), fc.Args["before"].(*string)) }) if err != nil { ec.Error(ctx, err) @@ -3799,7 +3910,7 @@ func (ec *executionContext) _Account_stateChanges(ctx context.Context, field gra }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (any, error) { ctx = rctx // use context from middleware stack in children - return ec.resolvers.Account().StateChanges(rctx, obj, fc.Args["filter"].(*AccountStateChangeFilterInput), fc.Args["first"].(*int32), fc.Args["after"].(*string), fc.Args["last"].(*int32), fc.Args["before"].(*string)) + return ec.resolvers.Account().StateChanges(rctx, obj, fc.Args["filter"].(*AccountStateChangeFilterInput), fc.Args["since"].(*time.Time), fc.Args["until"].(*time.Time), fc.Args["first"].(*int32), fc.Args["after"].(*string), fc.Args["last"].(*int32), fc.Args["before"].(*string)) }) if err != nil { ec.Error(ctx, err) @@ -21120,6 +21231,24 @@ func (ec *executionContext) marshalOString2ᚖstring(ctx context.Context, sel as return res } +func (ec *executionContext) unmarshalOTime2ᚖtimeᚐTime(ctx context.Context, v any) (*time.Time, error) { + if v == nil { + return nil, nil + } + res, err := graphql.UnmarshalTime(v) + return &res, graphql.ErrorOnPath(ctx, err) +} + +func (ec *executionContext) marshalOTime2ᚖtimeᚐTime(ctx context.Context, sel ast.SelectionSet, v *time.Time) graphql.Marshaler { + if v == nil { + return graphql.Null + } + _ = sel + _ = ctx + res := graphql.MarshalTime(*v) + return res +} + func (ec *executionContext) marshalOTransaction2ᚖgithubᚗcomᚋstellarᚋwalletᚑbackendᚋinternalᚋindexerᚋtypesᚐTransaction(ctx context.Context, sel ast.SelectionSet, v *types.Transaction) graphql.Marshaler { if v == nil { return graphql.Null diff --git a/internal/serve/graphql/resolvers/account.resolvers.go b/internal/serve/graphql/resolvers/account.resolvers.go index ddec6224b..5298f89ea 100644 --- a/internal/serve/graphql/resolvers/account.resolvers.go +++ b/internal/serve/graphql/resolvers/account.resolvers.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/stellar/wallet-backend/internal/indexer/types" graphql1 "github.com/stellar/wallet-backend/internal/serve/graphql/generated" @@ -22,7 +23,7 @@ func (r *accountResolver) Address(ctx context.Context, obj *types.Account) (stri // This is a field resolver - it resolves the "transactions" field on an Account object // gqlgen calls this when a GraphQL query requests the transactions field on an Account // Field resolvers receive the parent object (Account) and return the field value -func (r *accountResolver) Transactions(ctx context.Context, obj *types.Account, first *int32, after *string, last *int32, before *string) (*graphql1.TransactionConnection, error) { +func (r *accountResolver) Transactions(ctx context.Context, obj *types.Account, since *time.Time, until *time.Time, first *int32, after *string, last *int32, before *string) (*graphql1.TransactionConnection, error) { params, err := parsePaginationParams(first, after, last, before, CursorTypeComposite) if err != nil { return nil, fmt.Errorf("parsing pagination params: %w", err) @@ -55,7 +56,7 @@ func (r *accountResolver) Transactions(ctx context.Context, obj *types.Account, // Operations is the resolver for the operations field. // This field resolver handles the "operations" field on an Account object -func (r *accountResolver) Operations(ctx context.Context, obj *types.Account, first *int32, after *string, last *int32, before *string) (*graphql1.OperationConnection, error) { +func (r *accountResolver) Operations(ctx context.Context, obj *types.Account, since *time.Time, until *time.Time, first *int32, after *string, last *int32, before *string) (*graphql1.OperationConnection, error) { params, err := parsePaginationParams(first, after, last, before, CursorTypeComposite) if err != nil { return nil, fmt.Errorf("parsing pagination params: %w", err) @@ -87,7 +88,7 @@ func (r *accountResolver) Operations(ctx context.Context, obj *types.Account, fi } // StateChanges is the resolver for the stateChanges field. -func (r *accountResolver) StateChanges(ctx context.Context, obj *types.Account, filter *graphql1.AccountStateChangeFilterInput, first *int32, after *string, last *int32, before *string) (*graphql1.StateChangeConnection, error) { +func (r *accountResolver) StateChanges(ctx context.Context, obj *types.Account, filter *graphql1.AccountStateChangeFilterInput, since *time.Time, until *time.Time, first *int32, after *string, last *int32, before *string) (*graphql1.StateChangeConnection, error) { params, err := parsePaginationParams(first, after, last, before, CursorTypeStateChange) if err != nil { return nil, fmt.Errorf("parsing pagination params: %w", err) From 0dac37917ef449ef7cc2a418f34047114530c479 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Sun, 15 Feb 2026 10:10:51 -0500 Subject: [PATCH 57/77] Add TimeRange struct and appendTimeRangeConditions helper Shared helper for appending ledger_created_at >= / <= conditions to query builders. Accepts a column name parameter for table-qualified columns in CTEs. Used by all three BatchGetByAccountAddress methods. --- internal/data/query_utils.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/internal/data/query_utils.go b/internal/data/query_utils.go index 148649e40..d61dd4ec8 100644 --- a/internal/data/query_utils.go +++ b/internal/data/query_utils.go @@ -5,6 +5,7 @@ import ( "fmt" "reflect" "strings" + "time" set "github.com/deckarep/golang-set/v2" "github.com/jackc/pgx/v5/pgtype" @@ -12,6 +13,34 @@ import ( "github.com/stellar/wallet-backend/internal/indexer/types" ) +// TimeRange represents an optional time window for filtering queries by ledger_created_at. +// Both fields are optional: omit both for all data, use Since alone for "from this point", +// use Until alone for "up to this point", or both for a bounded window. +type TimeRange struct { + Since *time.Time + Until *time.Time +} + +// appendTimeRangeConditions appends ledger_created_at >= and/or <= conditions to the query builder. +// Returns the updated args slice and next arg index. The column parameter allows specifying +// a table-qualified column name (e.g., "ta.ledger_created_at" for CTEs). +func appendTimeRangeConditions(qb *strings.Builder, column string, timeRange *TimeRange, args []interface{}, argIndex int) ([]interface{}, int) { + if timeRange == nil { + return args, argIndex + } + if timeRange.Since != nil { + qb.WriteString(fmt.Sprintf(" AND %s >= $%d", column, argIndex)) + args = append(args, *timeRange.Since) + argIndex++ + } + if timeRange.Until != nil { + qb.WriteString(fmt.Sprintf(" AND %s <= $%d", column, argIndex)) + args = append(args, *timeRange.Until) + argIndex++ + } + return args, argIndex +} + type SortOrder string const ( From 8414f543a2c02c5b809e89c39d2267d91d316648 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Sun, 15 Feb 2026 10:12:10 -0500 Subject: [PATCH 58/77] Add timeRange parameter to BatchGetByAccountAddress methods Add *TimeRange parameter to transactions, operations, and state changes BatchGetByAccountAddress. Inserts ledger_created_at >= / <= conditions inside the MATERIALIZED CTE (transactions/operations) or after the account_id WHERE clause (state changes) for early chunk pruning. --- internal/data/operations.go | 5 ++++- internal/data/operations_test.go | 2 +- internal/data/statechanges.go | 5 ++++- internal/data/statechanges_test.go | 28 ++++++++++++++-------------- internal/data/transactions.go | 5 ++++- internal/data/transactions_test.go | 2 +- 6 files changed, 28 insertions(+), 19 deletions(-) diff --git a/internal/data/operations.go b/internal/data/operations.go index 995e1db41..4af74f5e6 100644 --- a/internal/data/operations.go +++ b/internal/data/operations.go @@ -217,7 +217,7 @@ func (m *OperationModel) BatchGetByToID(ctx context.Context, toID int64, columns // BatchGetByAccountAddress gets the operations that are associated with a single account address. // Uses a MATERIALIZED CTE + LATERAL join pattern to allow TimescaleDB ChunkAppend optimization // on the operations_accounts hypertable by ordering on ledger_created_at first. -func (m *OperationModel) BatchGetByAccountAddress(ctx context.Context, accountAddress string, columns string, limit *int32, cursor *types.CompositeCursor, orderBy SortOrder) ([]*types.OperationWithCursor, error) { +func (m *OperationModel) BatchGetByAccountAddress(ctx context.Context, accountAddress string, columns string, limit *int32, cursor *types.CompositeCursor, orderBy SortOrder, timeRange *TimeRange) ([]*types.OperationWithCursor, error) { columns = prepareColumnsWithID(columns, types.Operation{}, "o", "id") var queryBuilder strings.Builder @@ -232,6 +232,9 @@ func (m *OperationModel) BatchGetByAccountAddress(ctx context.Context, accountAd FROM operations_accounts WHERE account_id = $1`) + // Time range filter: enables TimescaleDB chunk pruning at the earliest query stage + args, argIndex = appendTimeRangeConditions(&queryBuilder, "ledger_created_at", timeRange, args, argIndex) + // Decomposed cursor pagination: expands ROW() tuple comparison into OR clauses so // TimescaleDB ColumnarScan can push filters into vectorized batch processing. if cursor != nil { diff --git a/internal/data/operations_test.go b/internal/data/operations_test.go index be09b5667..4368514da 100644 --- a/internal/data/operations_test.go +++ b/internal/data/operations_test.go @@ -589,7 +589,7 @@ func TestOperationModel_BatchGetByAccountAddresses(t *testing.T) { require.NoError(t, err) // Test BatchGetByAccount - operations, err := m.BatchGetByAccountAddress(ctx, address1, "", nil, nil, ASC) + operations, err := m.BatchGetByAccountAddress(ctx, address1, "", nil, nil, ASC, nil) require.NoError(t, err) assert.Len(t, operations, 2) assert.Equal(t, int64(4097), operations[0].Operation.ID) diff --git a/internal/data/statechanges.go b/internal/data/statechanges.go index 543f72a1f..b58c57c52 100644 --- a/internal/data/statechanges.go +++ b/internal/data/statechanges.go @@ -23,7 +23,7 @@ type StateChangeModel struct { // BatchGetByAccountAddress gets the state changes that are associated with the given account address. // Optional filters: txHash, operationID, category, and reason can be used to further filter results. -func (m *StateChangeModel) BatchGetByAccountAddress(ctx context.Context, accountAddress string, txHash *string, operationID *int64, category *string, reason *string, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { +func (m *StateChangeModel) BatchGetByAccountAddress(ctx context.Context, accountAddress string, txHash *string, operationID *int64, category *string, reason *string, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder, timeRange *TimeRange) ([]*types.StateChangeWithCursor, error) { columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_order") var queryBuilder strings.Builder args := []interface{}{types.AddressBytea(accountAddress)} @@ -35,6 +35,9 @@ func (m *StateChangeModel) BatchGetByAccountAddress(ctx context.Context, account WHERE account_id = $1 `, columns)) + // Time range filter: enables TimescaleDB chunk pruning on the state_changes hypertable + args, argIndex = appendTimeRangeConditions(&queryBuilder, "ledger_created_at", timeRange, args, argIndex) + // Add transaction hash filter if provided (uses subquery to find to_id by hash) if txHash != nil { queryBuilder.WriteString(fmt.Sprintf(" AND to_id = (SELECT to_id FROM transactions WHERE hash = $%d)", argIndex)) diff --git a/internal/data/statechanges_test.go b/internal/data/statechanges_test.go index e2cf539fc..4777f07f6 100644 --- a/internal/data/statechanges_test.go +++ b/internal/data/statechanges_test.go @@ -287,7 +287,7 @@ func TestStateChangeModel_BatchGetByAccountAddress(t *testing.T) { } // Test BatchGetByAccount for address1 - stateChanges, err := m.BatchGetByAccountAddress(ctx, address1, nil, nil, nil, nil, "", nil, nil, ASC) + stateChanges, err := m.BatchGetByAccountAddress(ctx, address1, nil, nil, nil, nil, "", nil, nil, ASC, nil) require.NoError(t, err) assert.Len(t, stateChanges, 2) for _, sc := range stateChanges { @@ -295,7 +295,7 @@ func TestStateChangeModel_BatchGetByAccountAddress(t *testing.T) { } // Test BatchGetByAccount for address2 - stateChanges, err = m.BatchGetByAccountAddress(ctx, address2, nil, nil, nil, nil, "", nil, nil, ASC) + stateChanges, err = m.BatchGetByAccountAddress(ctx, address2, nil, nil, nil, nil, "", nil, nil, ASC, nil) require.NoError(t, err) assert.Len(t, stateChanges, 1) for _, sc := range stateChanges { @@ -357,7 +357,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { } txHash := testHash1 - stateChanges, err := m.BatchGetByAccountAddress(ctx, address, &txHash, nil, nil, nil, "", nil, nil, ASC) + stateChanges, err := m.BatchGetByAccountAddress(ctx, address, &txHash, nil, nil, nil, "", nil, nil, ASC, nil) require.NoError(t, err) // tx1 has to_id=1, so we get state changes where to_id=1 (2 state changes now) assert.Len(t, stateChanges, 2) @@ -379,7 +379,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { } operationID := int64(123) - stateChanges, err := m.BatchGetByAccountAddress(ctx, address, nil, &operationID, nil, nil, "", nil, nil, ASC) + stateChanges, err := m.BatchGetByAccountAddress(ctx, address, nil, &operationID, nil, nil, "", nil, nil, ASC, nil) require.NoError(t, err) // Only 1 state change has operation_id=123 (the first one with to_id=1) assert.Len(t, stateChanges, 1) @@ -402,7 +402,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { txHash := testHash1 operationID := int64(123) - stateChanges, err := m.BatchGetByAccountAddress(ctx, address, &txHash, &operationID, nil, nil, "", nil, nil, ASC) + stateChanges, err := m.BatchGetByAccountAddress(ctx, address, &txHash, &operationID, nil, nil, "", nil, nil, ASC, nil) require.NoError(t, err) // Should get only state changes that match BOTH filters (to_id=1 from tx1 hash, operation_id=123) assert.Len(t, stateChanges, 1) @@ -425,7 +425,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { } category := "BALANCE" - stateChanges, err := m.BatchGetByAccountAddress(ctx, address, nil, nil, &category, nil, "", nil, nil, ASC) + stateChanges, err := m.BatchGetByAccountAddress(ctx, address, nil, nil, &category, nil, "", nil, nil, ASC, nil) require.NoError(t, err) assert.Len(t, stateChanges, 3) for _, sc := range stateChanges { @@ -446,7 +446,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { } reason := "ADD" - stateChanges, err := m.BatchGetByAccountAddress(ctx, address, nil, nil, nil, &reason, "", nil, nil, ASC) + stateChanges, err := m.BatchGetByAccountAddress(ctx, address, nil, nil, nil, &reason, "", nil, nil, ASC, nil) require.NoError(t, err) assert.Len(t, stateChanges, 2) for _, sc := range stateChanges { @@ -468,7 +468,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { category := "SIGNER" reason := "ADD" - stateChanges, err := m.BatchGetByAccountAddress(ctx, address, nil, nil, &category, &reason, "", nil, nil, ASC) + stateChanges, err := m.BatchGetByAccountAddress(ctx, address, nil, nil, &category, &reason, "", nil, nil, ASC, nil) require.NoError(t, err) assert.Len(t, stateChanges, 2) for _, sc := range stateChanges { @@ -493,7 +493,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { operationID := int64(123) category := "BALANCE" reason := "CREDIT" - stateChanges, err := m.BatchGetByAccountAddress(ctx, address, &txHash, &operationID, &category, &reason, "", nil, nil, ASC) + stateChanges, err := m.BatchGetByAccountAddress(ctx, address, &txHash, &operationID, &category, &reason, "", nil, nil, ASC, nil) require.NoError(t, err) assert.Len(t, stateChanges, 1) for _, sc := range stateChanges { @@ -517,7 +517,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { } txHash := testHashNonExistent - stateChanges, err := m.BatchGetByAccountAddress(ctx, address, &txHash, nil, nil, nil, "", nil, nil, ASC) + stateChanges, err := m.BatchGetByAccountAddress(ctx, address, &txHash, nil, nil, nil, "", nil, nil, ASC, nil) require.NoError(t, err) assert.Empty(t, stateChanges) }) @@ -535,7 +535,7 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { txHash := testHash1 limit := int32(1) - stateChanges, err := m.BatchGetByAccountAddress(ctx, address, &txHash, nil, nil, nil, "", &limit, nil, ASC) + stateChanges, err := m.BatchGetByAccountAddress(ctx, address, &txHash, nil, nil, nil, "", &limit, nil, ASC, nil) require.NoError(t, err) assert.Len(t, stateChanges, 1) assert.Equal(t, int64(1), stateChanges[0].ToID) @@ -862,7 +862,7 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { require.NoError(t, err) t.Run("get all state changes for single to_id", func(t *testing.T) { - stateChanges, err := m.BatchGetByToID(ctx, 1, "", nil, nil, ASC) + stateChanges, err := m.BatchGetByToID(ctx, 1, "", nil, nil, ASC, nil) require.NoError(t, err) assert.Len(t, stateChanges, 3) @@ -879,7 +879,7 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { t.Run("get state changes with pagination - first", func(t *testing.T) { limit := int32(2) - stateChanges, err := m.BatchGetByToID(ctx, 1, "", &limit, nil, ASC) + stateChanges, err := m.BatchGetByToID(ctx, 1, "", &limit, nil, ASC, nil) require.NoError(t, err) assert.Len(t, stateChanges, 2) @@ -911,7 +911,7 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { }) t.Run("no state changes for non-existent to_id", func(t *testing.T) { - stateChanges, err := m.BatchGetByToID(ctx, 999, "", nil, nil, ASC) + stateChanges, err := m.BatchGetByToID(ctx, 999, "", nil, nil, ASC, nil) require.NoError(t, err) assert.Empty(t, stateChanges) }) diff --git a/internal/data/transactions.go b/internal/data/transactions.go index d8b8fbfcb..0546e71c5 100644 --- a/internal/data/transactions.go +++ b/internal/data/transactions.go @@ -91,7 +91,7 @@ func (m *TransactionModel) GetAll(ctx context.Context, columns string, limit *in // BatchGetByAccountAddress gets the transactions that are associated with a single account address. // Uses a MATERIALIZED CTE + LATERAL join pattern to allow TimescaleDB ChunkAppend optimization // on the transactions_accounts hypertable by ordering on ledger_created_at first. -func (m *TransactionModel) BatchGetByAccountAddress(ctx context.Context, accountAddress string, columns string, limit *int32, cursor *types.CompositeCursor, orderBy SortOrder) ([]*types.TransactionWithCursor, error) { +func (m *TransactionModel) BatchGetByAccountAddress(ctx context.Context, accountAddress string, columns string, limit *int32, cursor *types.CompositeCursor, orderBy SortOrder, timeRange *TimeRange) ([]*types.TransactionWithCursor, error) { columns = prepareColumnsWithID(columns, types.Transaction{}, "t", "to_id") var queryBuilder strings.Builder @@ -106,6 +106,9 @@ func (m *TransactionModel) BatchGetByAccountAddress(ctx context.Context, account FROM transactions_accounts WHERE account_id = $1`) + // Time range filter: enables TimescaleDB chunk pruning at the earliest query stage + args, argIndex = appendTimeRangeConditions(&queryBuilder, "ledger_created_at", timeRange, args, argIndex) + // Decomposed cursor pagination: expands ROW() tuple comparison into OR clauses so // TimescaleDB ColumnarScan can push filters into vectorized batch processing. if cursor != nil { diff --git a/internal/data/transactions_test.go b/internal/data/transactions_test.go index 5e6580456..aa413dc20 100644 --- a/internal/data/transactions_test.go +++ b/internal/data/transactions_test.go @@ -359,7 +359,7 @@ func TestTransactionModel_BatchGetByAccountAddress(t *testing.T) { require.NoError(t, err) // Test BatchGetByAccount - transactions, err := m.BatchGetByAccountAddress(ctx, address1, "", nil, nil, ASC) + transactions, err := m.BatchGetByAccountAddress(ctx, address1, "", nil, nil, ASC, nil) require.NoError(t, err) assert.Len(t, transactions, 2) From 2be7686c666bcc720805cb80c356b451649abff0 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Sun, 15 Feb 2026 10:13:36 -0500 Subject: [PATCH 59/77] Wire since/until through resolvers to data layer Add buildTimeRange helper that validates since <= until and constructs a *data.TimeRange. All three account field resolvers (Transactions, Operations, StateChanges) now build and pass time range to the data layer. --- .../graphql/resolvers/account.resolvers.go | 21 ++++++++++++++++--- internal/serve/graphql/resolvers/utils.go | 13 ++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/internal/serve/graphql/resolvers/account.resolvers.go b/internal/serve/graphql/resolvers/account.resolvers.go index 5298f89ea..f185bd6dd 100644 --- a/internal/serve/graphql/resolvers/account.resolvers.go +++ b/internal/serve/graphql/resolvers/account.resolvers.go @@ -30,8 +30,13 @@ func (r *accountResolver) Transactions(ctx context.Context, obj *types.Account, } queryLimit := *params.Limit + 1 // +1 to check if there is a next page + timeRange, err := buildTimeRange(since, until) + if err != nil { + return nil, err + } + dbColumns := GetDBColumnsForFields(ctx, types.Transaction{}) - transactions, err := r.models.Transactions.BatchGetByAccountAddress(ctx, string(obj.StellarAddress), strings.Join(dbColumns, ", "), &queryLimit, params.CompositeCursor, params.SortOrder) + transactions, err := r.models.Transactions.BatchGetByAccountAddress(ctx, string(obj.StellarAddress), strings.Join(dbColumns, ", "), &queryLimit, params.CompositeCursor, params.SortOrder, timeRange) if err != nil { return nil, fmt.Errorf("getting transactions from db for account %s: %w", obj.StellarAddress, err) } @@ -63,8 +68,13 @@ func (r *accountResolver) Operations(ctx context.Context, obj *types.Account, si } queryLimit := *params.Limit + 1 // +1 to check if there is a next page + timeRange, err := buildTimeRange(since, until) + if err != nil { + return nil, err + } + dbColumns := GetDBColumnsForFields(ctx, types.Operation{}) - operations, err := r.models.Operations.BatchGetByAccountAddress(ctx, string(obj.StellarAddress), strings.Join(dbColumns, ", "), &queryLimit, params.CompositeCursor, params.SortOrder) + operations, err := r.models.Operations.BatchGetByAccountAddress(ctx, string(obj.StellarAddress), strings.Join(dbColumns, ", "), &queryLimit, params.CompositeCursor, params.SortOrder, timeRange) if err != nil { return nil, fmt.Errorf("getting operations from db for account %s: %w", obj.StellarAddress, err) } @@ -115,8 +125,13 @@ func (r *accountResolver) StateChanges(ctx context.Context, obj *types.Account, } } + timeRange, err := buildTimeRange(since, until) + if err != nil { + return nil, err + } + dbColumns := GetDBColumnsForFields(ctx, types.StateChange{}) - stateChanges, err := r.models.StateChanges.BatchGetByAccountAddress(ctx, string(obj.StellarAddress), txHash, operationID, category, reason, strings.Join(dbColumns, ", "), &queryLimit, params.StateChangeCursor, params.SortOrder) + stateChanges, err := r.models.StateChanges.BatchGetByAccountAddress(ctx, string(obj.StellarAddress), txHash, operationID, category, reason, strings.Join(dbColumns, ", "), &queryLimit, params.StateChangeCursor, params.SortOrder, timeRange) if err != nil { return nil, fmt.Errorf("getting state changes from db for account %s: %w", obj.StellarAddress, err) } diff --git a/internal/serve/graphql/resolvers/utils.go b/internal/serve/graphql/resolvers/utils.go index 9813c7496..0cdfada32 100644 --- a/internal/serve/graphql/resolvers/utils.go +++ b/internal/serve/graphql/resolvers/utils.go @@ -413,6 +413,19 @@ func parseCompositeCursor(s *string) (*types.CompositeCursor, error) { }, nil } +// buildTimeRange constructs a *data.TimeRange from optional since/until params. +// Returns an error if both are provided and until is before since. +// Returns nil if both are nil (no time range filtering). +func buildTimeRange(since *time.Time, until *time.Time) (*data.TimeRange, error) { + if since == nil && until == nil { + return nil, nil + } + if since != nil && until != nil && until.Before(*since) { + return nil, fmt.Errorf("until must not be before since") + } + return &data.TimeRange{Since: since, Until: until}, nil +} + func validatePaginationParams(first *int32, after *string, last *int32, before *string) error { if first != nil && last != nil { return fmt.Errorf("first and last cannot be used together") From e75d828bfcf6b62ea0866d19557c781ec5a93f58 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Sun, 15 Feb 2026 10:16:09 -0500 Subject: [PATCH 60/77] Add since/until time range params to wbclient methods Update GetAccountTransactions, GetAccountOperations, and GetAccountStateChanges client methods with since/until *time.Time parameters. Update corresponding GraphQL query builders to declare $since: Time, $until: Time variables and pass them through. Fix integration test call sites for updated signatures. --- .../accounts_register_test.go | 4 +-- .../integrationtests/data_validation_test.go | 2 +- pkg/wbclient/client.go | 25 ++++++++++++++++--- pkg/wbclient/queries.go | 12 ++++----- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/internal/integrationtests/accounts_register_test.go b/internal/integrationtests/accounts_register_test.go index 6ff30a4cc..ee626cc63 100644 --- a/internal/integrationtests/accounts_register_test.go +++ b/internal/integrationtests/accounts_register_test.go @@ -37,7 +37,7 @@ func (suite *AccountRegisterTestSuite) TestParticipantFiltering() { // 3. Verify operations are NOT stored for unregistered account limit := int32(10) - ops, err := client.GetAccountOperations(ctx, unregisteredKP.Address(), &limit, nil, nil, nil) + ops, err := client.GetAccountOperations(ctx, unregisteredKP.Address(), nil, nil, &limit, nil, nil, nil) suite.Require().NoError(err) suite.Require().Empty(ops.Edges, "Expected no operations for unregistered account") @@ -55,7 +55,7 @@ func (suite *AccountRegisterTestSuite) TestParticipantFiltering() { suite.testEnv.WaitForLedgers(ctx, 2) // 6. Verify operations ARE stored now for the registered account - ops, err = client.GetAccountOperations(ctx, unregisteredKP.Address(), &limit, nil, nil, nil) + ops, err = client.GetAccountOperations(ctx, unregisteredKP.Address(), nil, nil, &limit, nil, nil, nil) suite.Require().NoError(err) suite.Require().NotEmpty(ops.Edges, "Expected operations for registered account") diff --git a/internal/integrationtests/data_validation_test.go b/internal/integrationtests/data_validation_test.go index 2ebc2c9fb..4e3a13aaa 100644 --- a/internal/integrationtests/data_validation_test.go +++ b/internal/integrationtests/data_validation_test.go @@ -71,7 +71,7 @@ func (suite *DataValidationTestSuite) fetchStateChangesInParallel( query := q // capture variable group.Submit(func() { sc, err := suite.testEnv.WBClient.GetAccountStateChanges( - ctx, query.account, query.txHash, nil, query.category, query.reason, first, nil, nil, nil) + ctx, query.account, query.txHash, nil, query.category, query.reason, nil, nil, first, nil, nil, nil) if err != nil { errMu.Lock() errs = append(errs, fmt.Errorf("%s: %w", query.name, err)) diff --git a/pkg/wbclient/client.go b/pkg/wbclient/client.go index 2c2647124..96a662cc9 100644 --- a/pkg/wbclient/client.go +++ b/pkg/wbclient/client.go @@ -487,7 +487,7 @@ func (c *Client) GetStateChanges(ctx context.Context, first, last *int32, after, return data.StateChanges, nil } -func (c *Client) GetAccountTransactions(ctx context.Context, address string, first, last *int32, after, before *string, opts ...*QueryOptions) (*types.TransactionConnection, error) { +func (c *Client) GetAccountTransactions(ctx context.Context, address string, since, until *time.Time, first, last *int32, after, before *string, opts ...*QueryOptions) (*types.TransactionConnection, error) { var fields []string if len(opts) > 0 && opts[0] != nil { fields = opts[0].TransactionFields @@ -502,6 +502,12 @@ func (c *Client) GetAccountTransactions(ctx context.Context, address string, fir map[string]interface{}{"address": address}, paginationVars, ) + if since != nil { + variables["since"] = *since + } + if until != nil { + variables["until"] = *until + } data, err := executeGraphQL[AccountTransactionsData](c, ctx, buildAccountTransactionsQuery(fields), variables) if err != nil { @@ -511,7 +517,7 @@ func (c *Client) GetAccountTransactions(ctx context.Context, address string, fir return data.AccountByAddress.Transactions, nil } -func (c *Client) GetAccountOperations(ctx context.Context, address string, first, last *int32, after, before *string, opts ...*QueryOptions) (*types.OperationConnection, error) { +func (c *Client) GetAccountOperations(ctx context.Context, address string, since, until *time.Time, first, last *int32, after, before *string, opts ...*QueryOptions) (*types.OperationConnection, error) { var fields []string if len(opts) > 0 && opts[0] != nil { fields = opts[0].OperationFields @@ -526,6 +532,12 @@ func (c *Client) GetAccountOperations(ctx context.Context, address string, first map[string]interface{}{"address": address}, paginationVars, ) + if since != nil { + variables["since"] = *since + } + if until != nil { + variables["until"] = *until + } data, err := executeGraphQL[AccountOperationsData](c, ctx, buildAccountOperationsQuery(fields), variables) if err != nil { @@ -535,7 +547,7 @@ func (c *Client) GetAccountOperations(ctx context.Context, address string, first return data.AccountByAddress.Operations, nil } -func (c *Client) GetAccountStateChanges(ctx context.Context, address string, transactionHash *string, operationID *int64, category *string, reason *string, first, last *int32, after, before *string) (*types.StateChangeConnection, error) { +func (c *Client) GetAccountStateChanges(ctx context.Context, address string, transactionHash *string, operationID *int64, category *string, reason *string, since, until *time.Time, first, last *int32, after, before *string) (*types.StateChangeConnection, error) { paginationVars, err := buildPaginationVars(first, last, after, before) if err != nil { return nil, fmt.Errorf("building pagination variables: %w", err) @@ -563,6 +575,13 @@ func (c *Client) GetAccountStateChanges(ctx context.Context, address string, tra variables["filter"] = filter } + if since != nil { + variables["since"] = *since + } + if until != nil { + variables["until"] = *until + } + variables = mergeVariables(variables, paginationVars) data, err := executeGraphQL[AccountStateChangesData](c, ctx, buildAccountStateChangesQuery(), variables) diff --git a/pkg/wbclient/queries.go b/pkg/wbclient/queries.go index 4f8769ed4..4d5e5f373 100644 --- a/pkg/wbclient/queries.go +++ b/pkg/wbclient/queries.go @@ -250,9 +250,9 @@ func buildStateChangesQuery() string { func buildAccountTransactionsQuery(fields []string) string { fieldList := buildFieldList(fields, defaultTransactionFields) return fmt.Sprintf(` - query AccountTransactions($address: String!, $first: Int, $after: String, $last: Int, $before: String) { + query AccountTransactions($address: String!, $since: Time, $until: Time, $first: Int, $after: String, $last: Int, $before: String) { accountByAddress(address: $address) { - transactions(first: $first, after: $after, last: $last, before: $before) { + transactions(since: $since, until: $until, first: $first, after: $after, last: $last, before: $before) { edges { node { %s @@ -275,9 +275,9 @@ func buildAccountTransactionsQuery(fields []string) string { func buildAccountOperationsQuery(fields []string) string { fieldList := buildFieldList(fields, defaultOperationFields) return fmt.Sprintf(` - query AccountOperations($address: String!, $first: Int, $after: String, $last: Int, $before: String) { + query AccountOperations($address: String!, $since: Time, $until: Time, $first: Int, $after: String, $last: Int, $before: String) { accountByAddress(address: $address) { - operations(first: $first, after: $after, last: $last, before: $before) { + operations(since: $since, until: $until, first: $first, after: $after, last: $last, before: $before) { edges { node { %s @@ -300,9 +300,9 @@ func buildAccountOperationsQuery(fields []string) string { // Supports optional filtering by transaction hash and/or operation ID func buildAccountStateChangesQuery() string { return fmt.Sprintf(` - query AccountStateChanges($address: String!, $filter: AccountStateChangeFilterInput, $first: Int, $after: String, $last: Int, $before: String) { + query AccountStateChanges($address: String!, $filter: AccountStateChangeFilterInput, $since: Time, $until: Time, $first: Int, $after: String, $last: Int, $before: String) { accountByAddress(address: $address) { - stateChanges(filter: $filter, first: $first, after: $after, last: $last, before: $before) { + stateChanges(filter: $filter, since: $since, until: $until, first: $first, after: $after, last: $last, before: $before) { edges { node { %s From 7c299827aaff90fe289b0fe227cd76efb86485a3 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Sun, 15 Feb 2026 10:20:52 -0500 Subject: [PATCH 61/77] Add tests for time range filtering feature Add appendTimeRangeConditions unit tests covering nil, since-only, until-only, both, and table-qualified column cases. Add resolver tests for Transactions, Operations, and StateChanges with time range params: since in past/future, until before since validation error, and combined with pagination/filters. Update existing resolver tests for new since/until method signatures. Fix BatchGetByToID test calls that had extra nil argument. --- internal/data/query_utils_test.go | 81 +++++- internal/data/statechanges_test.go | 6 +- .../resolvers/account_resolvers_test.go | 253 +++++++++++++++--- 3 files changed, 293 insertions(+), 47 deletions(-) diff --git a/internal/data/query_utils_test.go b/internal/data/query_utils_test.go index cf75f5faf..4134ac6cd 100644 --- a/internal/data/query_utils_test.go +++ b/internal/data/query_utils_test.go @@ -1,8 +1,8 @@ -// Tests for buildDecomposedCursorCondition which decomposes ROW() tuple comparisons -// into OR clauses for TimescaleDB vectorized filtering on compressed hypertables. +// Tests for query utility functions: time range conditions and decomposed cursor conditions. package data import ( + "strings" "testing" "time" @@ -128,3 +128,80 @@ func Test_buildDecomposedCursorCondition(t *testing.T) { }) } } + +func Test_appendTimeRangeConditions(t *testing.T) { + since := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + until := time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC) + + tests := []struct { + name string + timeRange *TimeRange + column string + startArgs []interface{} + startArgIndex int + wantSQL string + wantArgs []interface{} + wantNextIndex int + }{ + { + name: "nil time range appends nothing", + timeRange: nil, + column: "ledger_created_at", + startArgs: []interface{}{"existing"}, + startArgIndex: 2, + wantSQL: "", + wantArgs: []interface{}{"existing"}, + wantNextIndex: 2, + }, + { + name: "since only", + timeRange: &TimeRange{Since: &since}, + column: "ledger_created_at", + startArgs: []interface{}{"existing"}, + startArgIndex: 2, + wantSQL: " AND ledger_created_at >= $2", + wantArgs: []interface{}{"existing", since}, + wantNextIndex: 3, + }, + { + name: "until only", + timeRange: &TimeRange{Until: &until}, + column: "ledger_created_at", + startArgs: []interface{}{"existing"}, + startArgIndex: 2, + wantSQL: " AND ledger_created_at <= $2", + wantArgs: []interface{}{"existing", until}, + wantNextIndex: 3, + }, + { + name: "both since and until", + timeRange: &TimeRange{Since: &since, Until: &until}, + column: "ledger_created_at", + startArgs: []interface{}{"existing"}, + startArgIndex: 2, + wantSQL: " AND ledger_created_at >= $2 AND ledger_created_at <= $3", + wantArgs: []interface{}{"existing", since, until}, + wantNextIndex: 4, + }, + { + name: "table-qualified column name", + timeRange: &TimeRange{Since: &since}, + column: "ta.ledger_created_at", + startArgs: nil, + startArgIndex: 1, + wantSQL: " AND ta.ledger_created_at >= $1", + wantArgs: []interface{}{since}, + wantNextIndex: 2, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var qb strings.Builder + gotArgs, gotNextIndex := appendTimeRangeConditions(&qb, tt.column, tt.timeRange, tt.startArgs, tt.startArgIndex) + assert.Equal(t, tt.wantSQL, qb.String()) + assert.Equal(t, tt.wantArgs, gotArgs) + assert.Equal(t, tt.wantNextIndex, gotNextIndex) + }) + } +} diff --git a/internal/data/statechanges_test.go b/internal/data/statechanges_test.go index 4777f07f6..644bfab0a 100644 --- a/internal/data/statechanges_test.go +++ b/internal/data/statechanges_test.go @@ -862,7 +862,7 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { require.NoError(t, err) t.Run("get all state changes for single to_id", func(t *testing.T) { - stateChanges, err := m.BatchGetByToID(ctx, 1, "", nil, nil, ASC, nil) + stateChanges, err := m.BatchGetByToID(ctx, 1, "", nil, nil, ASC) require.NoError(t, err) assert.Len(t, stateChanges, 3) @@ -879,7 +879,7 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { t.Run("get state changes with pagination - first", func(t *testing.T) { limit := int32(2) - stateChanges, err := m.BatchGetByToID(ctx, 1, "", &limit, nil, ASC, nil) + stateChanges, err := m.BatchGetByToID(ctx, 1, "", &limit, nil, ASC) require.NoError(t, err) assert.Len(t, stateChanges, 2) @@ -911,7 +911,7 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { }) t.Run("no state changes for non-existent to_id", func(t *testing.T) { - stateChanges, err := m.BatchGetByToID(ctx, 999, "", nil, nil, ASC, nil) + stateChanges, err := m.BatchGetByToID(ctx, 999, "", nil, nil, ASC) require.NoError(t, err) assert.Empty(t, stateChanges) }) diff --git a/internal/serve/graphql/resolvers/account_resolvers_test.go b/internal/serve/graphql/resolvers/account_resolvers_test.go index 609786c51..33e39ae8c 100644 --- a/internal/serve/graphql/resolvers/account_resolvers_test.go +++ b/internal/serve/graphql/resolvers/account_resolvers_test.go @@ -15,6 +15,8 @@ import ( "github.com/stellar/wallet-backend/internal/indexer/types" "github.com/stellar/wallet-backend/internal/metrics" graphql1 "github.com/stellar/wallet-backend/internal/serve/graphql/generated" + + "time" ) // testOpXDRAcc returns the expected base64-encoded XDR for test operation N @@ -42,7 +44,7 @@ func TestAccountResolver_Transactions(t *testing.T) { t.Run("get all transactions", func(t *testing.T) { ctx := getTestCtx("transactions", []string{"hash"}) - transactions, err := resolver.Transactions(ctx, parentAccount, nil, nil, nil, nil) + transactions, err := resolver.Transactions(ctx, parentAccount, nil, nil, nil, nil, nil, nil) require.NoError(t, err) require.Len(t, transactions.Edges, 4) @@ -56,7 +58,7 @@ func TestAccountResolver_Transactions(t *testing.T) { t.Run("get transactions with first/after limit and cursor", func(t *testing.T) { ctx := getTestCtx("transactions", []string{"hash"}) first := int32(2) - txs, err := resolver.Transactions(ctx, parentAccount, &first, nil, nil, nil) + txs, err := resolver.Transactions(ctx, parentAccount, nil, nil, &first, nil, nil, nil) require.NoError(t, err) assert.Len(t, txs.Edges, 2) assert.Equal(t, testTxHash1, txs.Edges[0].Node.Hash.String()) @@ -67,7 +69,7 @@ func TestAccountResolver_Transactions(t *testing.T) { // Get the next cursor nextCursor := txs.PageInfo.EndCursor assert.NotNil(t, nextCursor) - txs, err = resolver.Transactions(ctx, parentAccount, &first, nextCursor, nil, nil) + txs, err = resolver.Transactions(ctx, parentAccount, nil, nil, &first, nextCursor, nil, nil) require.NoError(t, err) assert.Len(t, txs.Edges, 2) assert.Equal(t, testTxHash3, txs.Edges[0].Node.Hash.String()) @@ -80,7 +82,7 @@ func TestAccountResolver_Transactions(t *testing.T) { t.Run("get transactions with last/before limit and cursor", func(t *testing.T) { ctx := getTestCtx("transactions", []string{"hash"}) last := int32(2) - txs, err := resolver.Transactions(ctx, parentAccount, nil, nil, &last, nil) + txs, err := resolver.Transactions(ctx, parentAccount, nil, nil, nil, nil, &last, nil) require.NoError(t, err) assert.Len(t, txs.Edges, 2) assert.Equal(t, testTxHash3, txs.Edges[0].Node.Hash.String()) @@ -92,7 +94,7 @@ func TestAccountResolver_Transactions(t *testing.T) { last = int32(1) nextCursor := txs.PageInfo.EndCursor assert.NotNil(t, nextCursor) - txs, err = resolver.Transactions(ctx, parentAccount, nil, nil, &last, nextCursor) + txs, err = resolver.Transactions(ctx, parentAccount, nil, nil, nil, nil, &last, nextCursor) require.NoError(t, err) assert.Len(t, txs.Edges, 1) assert.Equal(t, testTxHash2, txs.Edges[0].Node.Hash.String()) @@ -102,7 +104,7 @@ func TestAccountResolver_Transactions(t *testing.T) { nextCursor = txs.PageInfo.EndCursor assert.NotNil(t, nextCursor) last = int32(10) - txs, err = resolver.Transactions(ctx, parentAccount, nil, nil, &last, nextCursor) + txs, err = resolver.Transactions(ctx, parentAccount, nil, nil, nil, nil, &last, nextCursor) require.NoError(t, err) assert.Len(t, txs.Edges, 1) assert.Equal(t, testTxHash1, txs.Edges[0].Node.Hash.String()) @@ -114,7 +116,7 @@ func TestAccountResolver_Transactions(t *testing.T) { t.Run("account with no transactions", func(t *testing.T) { nonExistentAccount := &types.Account{StellarAddress: types.AddressBytea(sharedNonExistentAccountAddress)} ctx := getTestCtx("transactions", []string{"hash"}) - transactions, err := resolver.Transactions(ctx, nonExistentAccount, nil, nil, nil, nil) + transactions, err := resolver.Transactions(ctx, nonExistentAccount, nil, nil, nil, nil, nil, nil) require.NoError(t, err) assert.Empty(t, transactions.Edges) @@ -127,24 +129,24 @@ func TestAccountResolver_Transactions(t *testing.T) { last := int32(1) after := encodeCursor(int64(4)) before := encodeCursor(int64(1)) - _, err := resolver.Transactions(ctx, parentAccount, &first, &after, nil, nil) + _, err := resolver.Transactions(ctx, parentAccount, nil, nil, &first, &after, nil, nil) require.Error(t, err) assert.Contains(t, err.Error(), "validating pagination params: first must be greater than 0") first = int32(1) - _, err = resolver.Transactions(ctx, parentAccount, &first, nil, &last, nil) + _, err = resolver.Transactions(ctx, parentAccount, nil, nil, &first, nil, &last, nil) require.Error(t, err) assert.Contains(t, err.Error(), "validating pagination params: first and last cannot be used together") - _, err = resolver.Transactions(ctx, parentAccount, nil, &after, nil, &before) + _, err = resolver.Transactions(ctx, parentAccount, nil, nil, nil, &after, nil, &before) require.Error(t, err) assert.Contains(t, err.Error(), "validating pagination params: after and before cannot be used together") - _, err = resolver.Transactions(ctx, parentAccount, &first, nil, nil, &before) + _, err = resolver.Transactions(ctx, parentAccount, nil, nil, &first, nil, nil, &before) require.Error(t, err) assert.Contains(t, err.Error(), "validating pagination params: first and before cannot be used together") - _, err = resolver.Transactions(ctx, parentAccount, nil, &after, &last, nil) + _, err = resolver.Transactions(ctx, parentAccount, nil, nil, nil, &after, &last, nil) require.Error(t, err) assert.Contains(t, err.Error(), "validating pagination params: last and after cannot be used together") }) @@ -169,7 +171,7 @@ func TestAccountResolver_Operations(t *testing.T) { t.Run("get all operations", func(t *testing.T) { ctx := getTestCtx("operations", []string{"operation_xdr"}) - operations, err := resolver.Operations(ctx, parentAccount, nil, nil, nil, nil) + operations, err := resolver.Operations(ctx, parentAccount, nil, nil, nil, nil, nil, nil) require.NoError(t, err) require.Len(t, operations.Edges, 8) @@ -182,7 +184,7 @@ func TestAccountResolver_Operations(t *testing.T) { t.Run("get operations with first/after limit and cursor", func(t *testing.T) { ctx := getTestCtx("operations", []string{"operation_xdr"}) first := int32(2) - ops, err := resolver.Operations(ctx, parentAccount, &first, nil, nil, nil) + ops, err := resolver.Operations(ctx, parentAccount, nil, nil, &first, nil, nil, nil) require.NoError(t, err) assert.Len(t, ops.Edges, 2) assert.Equal(t, testOpXDRAcc(1), ops.Edges[0].Node.OperationXDR.String()) @@ -193,7 +195,7 @@ func TestAccountResolver_Operations(t *testing.T) { // Get the next cursor nextCursor := ops.PageInfo.EndCursor assert.NotNil(t, nextCursor) - ops, err = resolver.Operations(ctx, parentAccount, &first, nextCursor, nil, nil) + ops, err = resolver.Operations(ctx, parentAccount, nil, nil, &first, nextCursor, nil, nil) require.NoError(t, err) assert.Len(t, ops.Edges, 2) assert.Equal(t, testOpXDRAcc(3), ops.Edges[0].Node.OperationXDR.String()) @@ -204,7 +206,7 @@ func TestAccountResolver_Operations(t *testing.T) { first = int32(10) nextCursor = ops.PageInfo.EndCursor assert.NotNil(t, nextCursor) - ops, err = resolver.Operations(ctx, parentAccount, &first, nextCursor, nil, nil) + ops, err = resolver.Operations(ctx, parentAccount, nil, nil, &first, nextCursor, nil, nil) require.NoError(t, err) assert.Len(t, ops.Edges, 4) assert.Equal(t, testOpXDRAcc(5), ops.Edges[0].Node.OperationXDR.String()) @@ -218,7 +220,7 @@ func TestAccountResolver_Operations(t *testing.T) { t.Run("get operations with last/before limit and cursor", func(t *testing.T) { ctx := getTestCtx("operations", []string{"operation_xdr"}) last := int32(2) - ops, err := resolver.Operations(ctx, parentAccount, nil, nil, &last, nil) + ops, err := resolver.Operations(ctx, parentAccount, nil, nil, nil, nil, &last, nil) require.NoError(t, err) assert.Len(t, ops.Edges, 2) assert.Equal(t, testOpXDRAcc(7), ops.Edges[0].Node.OperationXDR.String()) @@ -229,7 +231,7 @@ func TestAccountResolver_Operations(t *testing.T) { // Get the next cursor nextCursor := ops.PageInfo.EndCursor assert.NotNil(t, nextCursor) - ops, err = resolver.Operations(ctx, parentAccount, nil, nil, &last, nextCursor) + ops, err = resolver.Operations(ctx, parentAccount, nil, nil, nil, nil, &last, nextCursor) require.NoError(t, err) assert.Len(t, ops.Edges, 2) assert.Equal(t, testOpXDRAcc(5), ops.Edges[0].Node.OperationXDR.String()) @@ -240,7 +242,7 @@ func TestAccountResolver_Operations(t *testing.T) { nextCursor = ops.PageInfo.EndCursor assert.NotNil(t, nextCursor) last = int32(10) - ops, err = resolver.Operations(ctx, parentAccount, nil, nil, &last, nextCursor) + ops, err = resolver.Operations(ctx, parentAccount, nil, nil, nil, nil, &last, nextCursor) require.NoError(t, err) assert.Len(t, ops.Edges, 4) assert.Equal(t, testOpXDRAcc(1), ops.Edges[0].Node.OperationXDR.String()) @@ -254,7 +256,7 @@ func TestAccountResolver_Operations(t *testing.T) { t.Run("account with no operations", func(t *testing.T) { nonExistentAccount := &types.Account{StellarAddress: types.AddressBytea(sharedNonExistentAccountAddress)} ctx := getTestCtx("operations", []string{"id"}) - operations, err := resolver.Operations(ctx, nonExistentAccount, nil, nil, nil, nil) + operations, err := resolver.Operations(ctx, nonExistentAccount, nil, nil, nil, nil, nil, nil) require.NoError(t, err) assert.Empty(t, operations.Edges) @@ -280,7 +282,7 @@ func TestAccountResolver_StateChanges(t *testing.T) { t.Run("get all state changes", func(t *testing.T) { ctx := getTestCtx("state_changes", []string{""}) - stateChanges, err := resolver.StateChanges(ctx, parentAccount, nil, nil, nil, nil, nil) + stateChanges, err := resolver.StateChanges(ctx, parentAccount, nil, nil, nil, nil, nil, nil, nil) require.NoError(t, err) require.Len(t, stateChanges.Edges, 20) @@ -326,7 +328,7 @@ func TestAccountResolver_StateChanges(t *testing.T) { tx2Op2ID := toid.New(1000, 2, 2).ToInt64() first := int32(3) - stateChanges, err := resolver.StateChanges(ctx, parentAccount, nil, &first, nil, nil, nil) + stateChanges, err := resolver.StateChanges(ctx, parentAccount, nil, nil, nil, &first, nil, nil, nil) require.NoError(t, err) assert.Len(t, stateChanges.Edges, 3) @@ -352,7 +354,7 @@ func TestAccountResolver_StateChanges(t *testing.T) { // Get the next cursor nextCursor := stateChanges.PageInfo.EndCursor assert.NotNil(t, nextCursor) - stateChanges, err = resolver.StateChanges(ctx, parentAccount, nil, &first, nextCursor, nil, nil) + stateChanges, err = resolver.StateChanges(ctx, parentAccount, nil, nil, nil, &first, nextCursor, nil, nil) require.NoError(t, err) assert.Len(t, stateChanges.Edges, 3) @@ -379,7 +381,7 @@ func TestAccountResolver_StateChanges(t *testing.T) { first = int32(100) nextCursor = stateChanges.PageInfo.EndCursor assert.NotNil(t, nextCursor) - stateChanges, err = resolver.StateChanges(ctx, parentAccount, nil, &first, nextCursor, nil, nil) + stateChanges, err = resolver.StateChanges(ctx, parentAccount, nil, nil, nil, &first, nextCursor, nil, nil) require.NoError(t, err) assert.Len(t, stateChanges.Edges, 14) @@ -420,7 +422,7 @@ func TestAccountResolver_StateChanges(t *testing.T) { tx4Op2ID := toid.New(1000, 4, 2).ToInt64() last := int32(3) - stateChanges, err := resolver.StateChanges(ctx, parentAccount, nil, nil, nil, &last, nil) + stateChanges, err := resolver.StateChanges(ctx, parentAccount, nil, nil, nil, nil, nil, &last, nil) require.NoError(t, err) assert.Len(t, stateChanges.Edges, 3) @@ -446,7 +448,7 @@ func TestAccountResolver_StateChanges(t *testing.T) { // Get the next cursor (going backward) nextCursor := stateChanges.PageInfo.EndCursor assert.NotNil(t, nextCursor) - stateChanges, err = resolver.StateChanges(ctx, parentAccount, nil, nil, nil, &last, nextCursor) + stateChanges, err = resolver.StateChanges(ctx, parentAccount, nil, nil, nil, nil, nil, &last, nextCursor) require.NoError(t, err) assert.Len(t, stateChanges.Edges, 3) @@ -472,7 +474,7 @@ func TestAccountResolver_StateChanges(t *testing.T) { nextCursor = stateChanges.PageInfo.EndCursor assert.NotNil(t, nextCursor) last = int32(100) - stateChanges, err = resolver.StateChanges(ctx, parentAccount, nil, nil, nil, &last, nextCursor) + stateChanges, err = resolver.StateChanges(ctx, parentAccount, nil, nil, nil, nil, nil, &last, nextCursor) require.NoError(t, err) assert.Len(t, stateChanges.Edges, 14) @@ -499,7 +501,7 @@ func TestAccountResolver_StateChanges(t *testing.T) { t.Run("account with no state changes", func(t *testing.T) { nonExistentAccount := &types.Account{StellarAddress: types.AddressBytea(sharedNonExistentAccountAddress)} ctx := getTestCtx("state_changes", []string{"to_id", "state_change_order"}) - stateChanges, err := resolver.StateChanges(ctx, nonExistentAccount, nil, nil, nil, nil, nil) + stateChanges, err := resolver.StateChanges(ctx, nonExistentAccount, nil, nil, nil, nil, nil, nil, nil) require.NoError(t, err) assert.Empty(t, stateChanges.Edges) @@ -529,7 +531,7 @@ func TestAccountResolver_StateChanges_WithFilters(t *testing.T) { filter := &graphql1.AccountStateChangeFilterInput{ TransactionHash: &txHash, } - stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, nil, nil) + stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, nil, nil, nil, nil) require.NoError(t, err) // tx1 has 3 operations (0, 1, 2), each operation has 2 state changes except op 0 (1 state change) @@ -578,7 +580,7 @@ func TestAccountResolver_StateChanges_WithFilters(t *testing.T) { filter := &graphql1.AccountStateChangeFilterInput{ OperationID: &opID, } - stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, nil, nil) + stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, nil, nil, nil, nil) require.NoError(t, err) // Operation 1 has 2 state changes @@ -605,7 +607,7 @@ func TestAccountResolver_StateChanges_WithFilters(t *testing.T) { TransactionHash: &txHash, OperationID: &opID, } - stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, nil, nil) + stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, nil, nil, nil, nil) require.NoError(t, err) // Only state changes that match both filters @@ -630,7 +632,7 @@ func TestAccountResolver_StateChanges_WithFilters(t *testing.T) { filter := &graphql1.AccountStateChangeFilterInput{ TransactionHash: &txHash, } - stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, nil, nil) + stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, nil, nil, nil, nil) require.NoError(t, err) require.Empty(t, stateChanges.Edges) @@ -651,7 +653,7 @@ func TestAccountResolver_StateChanges_WithFilters(t *testing.T) { // Get first 2 state changes first := int32(2) - stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, &first, nil, nil, nil) + stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, &first, nil, nil, nil) require.NoError(t, err) require.Len(t, stateChanges.Edges, 2) @@ -672,7 +674,7 @@ func TestAccountResolver_StateChanges_WithFilters(t *testing.T) { // Get next page nextCursor := stateChanges.PageInfo.EndCursor assert.NotNil(t, nextCursor) - stateChanges, err = resolver.StateChanges(ctx, parentAccount, filter, &first, nextCursor, nil, nil) + stateChanges, err = resolver.StateChanges(ctx, parentAccount, filter, nil, nil, &first, nextCursor, nil, nil) require.NoError(t, err) require.Len(t, stateChanges.Edges, 2) @@ -693,7 +695,7 @@ func TestAccountResolver_StateChanges_WithFilters(t *testing.T) { // Get final page nextCursor = stateChanges.PageInfo.EndCursor assert.NotNil(t, nextCursor) - stateChanges, err = resolver.StateChanges(ctx, parentAccount, filter, &first, nextCursor, nil, nil) + stateChanges, err = resolver.StateChanges(ctx, parentAccount, filter, nil, nil, &first, nextCursor, nil, nil) require.NoError(t, err) require.Len(t, stateChanges.Edges, 1) @@ -731,7 +733,7 @@ func TestAccountResolver_StateChanges_WithCategoryReasonFilters(t *testing.T) { filter := &graphql1.AccountStateChangeFilterInput{ Category: &category, } - stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, nil, nil) + stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, nil, nil, nil, nil) require.NoError(t, err) // Verify all returned state changes are BALANCE category for _, sc := range stateChanges.Edges { @@ -745,7 +747,7 @@ func TestAccountResolver_StateChanges_WithCategoryReasonFilters(t *testing.T) { filter := &graphql1.AccountStateChangeFilterInput{ Reason: &reason, } - stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, nil, nil) + stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, nil, nil, nil, nil) require.NoError(t, err) // Verify all returned state changes are CREDIT reason @@ -762,7 +764,7 @@ func TestAccountResolver_StateChanges_WithCategoryReasonFilters(t *testing.T) { Category: &category, Reason: &reason, } - stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, nil, nil) + stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, nil, nil, nil, nil) require.NoError(t, err) // Verify all returned state changes are SIGNER category and ADD reason @@ -785,7 +787,7 @@ func TestAccountResolver_StateChanges_WithCategoryReasonFilters(t *testing.T) { Category: &category, Reason: &reason, } - stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, nil, nil) + stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, nil, nil, nil, nil) require.NoError(t, err) // Verify all returned state changes have correct IDs, category and reason @@ -807,7 +809,7 @@ func TestAccountResolver_StateChanges_WithCategoryReasonFilters(t *testing.T) { // Get first 2 state changes first := int32(2) - stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, &first, nil, nil, nil) + stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, &first, nil, nil, nil) require.NoError(t, err) require.Len(t, stateChanges.Edges, 2) assert.True(t, stateChanges.PageInfo.HasNextPage) @@ -819,7 +821,7 @@ func TestAccountResolver_StateChanges_WithCategoryReasonFilters(t *testing.T) { // Get next page nextCursor := stateChanges.PageInfo.EndCursor assert.NotNil(t, nextCursor) - stateChanges, err = resolver.StateChanges(ctx, parentAccount, filter, &first, nextCursor, nil, nil) + stateChanges, err = resolver.StateChanges(ctx, parentAccount, filter, nil, nil, &first, nextCursor, nil, nil) require.NoError(t, err) assert.LessOrEqual(t, len(stateChanges.Edges), 2) for _, sc := range stateChanges.Edges { @@ -833,7 +835,7 @@ func TestAccountResolver_StateChanges_WithCategoryReasonFilters(t *testing.T) { filter := &graphql1.AccountStateChangeFilterInput{ Category: &category, } - stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, nil, nil) + stateChanges, err := resolver.StateChanges(ctx, parentAccount, filter, nil, nil, nil, nil, nil, nil) require.NoError(t, err) require.Empty(t, stateChanges.Edges) @@ -841,3 +843,170 @@ func TestAccountResolver_StateChanges_WithCategoryReasonFilters(t *testing.T) { assert.False(t, stateChanges.PageInfo.HasPreviousPage) }) } + +func TestAccountResolver_Transactions_WithTimeRange(t *testing.T) { + parentAccount := &types.Account{StellarAddress: types.AddressBytea(sharedTestAccountAddress)} + + mockMetricsService := &metrics.MockMetricsService{} + mockMetricsService.On("IncDBQuery", "BatchGetByAccountAddress", "transactions").Return() + mockMetricsService.On("ObserveDBQueryDuration", "BatchGetByAccountAddress", "transactions", mock.Anything).Return() + defer mockMetricsService.AssertExpectations(t) + + resolver := &accountResolver{&Resolver{ + models: &data.Models{ + Transactions: &data.TransactionModel{ + DB: testDBConnectionPool, + MetricsService: mockMetricsService, + }, + }, + }} + + t.Run("since in the past returns all transactions", func(t *testing.T) { + ctx := getTestCtx("transactions", []string{"hash"}) + pastTime := time.Now().Add(-24 * time.Hour) + txs, err := resolver.Transactions(ctx, parentAccount, &pastTime, nil, nil, nil, nil, nil) + require.NoError(t, err) + assert.Len(t, txs.Edges, 4) + }) + + t.Run("since in the future returns no transactions", func(t *testing.T) { + ctx := getTestCtx("transactions", []string{"hash"}) + futureTime := time.Now().Add(24 * time.Hour) + txs, err := resolver.Transactions(ctx, parentAccount, &futureTime, nil, nil, nil, nil, nil) + require.NoError(t, err) + assert.Empty(t, txs.Edges) + }) + + t.Run("until in the past returns no transactions", func(t *testing.T) { + ctx := getTestCtx("transactions", []string{"hash"}) + pastTime := time.Now().Add(-24 * time.Hour) + txs, err := resolver.Transactions(ctx, parentAccount, nil, &pastTime, nil, nil, nil, nil) + require.NoError(t, err) + assert.Empty(t, txs.Edges) + }) + + t.Run("until in the future returns all transactions", func(t *testing.T) { + ctx := getTestCtx("transactions", []string{"hash"}) + futureTime := time.Now().Add(24 * time.Hour) + txs, err := resolver.Transactions(ctx, parentAccount, nil, &futureTime, nil, nil, nil, nil) + require.NoError(t, err) + assert.Len(t, txs.Edges, 4) + }) + + t.Run("until before since returns error", func(t *testing.T) { + ctx := getTestCtx("transactions", []string{"hash"}) + since := time.Now() + until := since.Add(-1 * time.Hour) + _, err := resolver.Transactions(ctx, parentAccount, &since, &until, nil, nil, nil, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "until must not be before since") + }) + + t.Run("time range combined with pagination", func(t *testing.T) { + ctx := getTestCtx("transactions", []string{"hash"}) + pastTime := time.Now().Add(-24 * time.Hour) + first := int32(2) + txs, err := resolver.Transactions(ctx, parentAccount, &pastTime, nil, &first, nil, nil, nil) + require.NoError(t, err) + assert.Len(t, txs.Edges, 2) + assert.True(t, txs.PageInfo.HasNextPage) + }) +} + +func TestAccountResolver_Operations_WithTimeRange(t *testing.T) { + parentAccount := &types.Account{StellarAddress: types.AddressBytea(sharedTestAccountAddress)} + + mockMetricsService := &metrics.MockMetricsService{} + mockMetricsService.On("IncDBQuery", "BatchGetByAccountAddress", "operations").Return() + mockMetricsService.On("ObserveDBQueryDuration", "BatchGetByAccountAddress", "operations", mock.Anything).Return() + defer mockMetricsService.AssertExpectations(t) + + resolver := &accountResolver{&Resolver{ + models: &data.Models{ + Operations: &data.OperationModel{ + DB: testDBConnectionPool, + MetricsService: mockMetricsService, + }, + }, + }} + + t.Run("since in the past returns all operations", func(t *testing.T) { + ctx := getTestCtx("operations", []string{"operation_xdr"}) + pastTime := time.Now().Add(-24 * time.Hour) + ops, err := resolver.Operations(ctx, parentAccount, &pastTime, nil, nil, nil, nil, nil) + require.NoError(t, err) + assert.Len(t, ops.Edges, 8) + }) + + t.Run("since in the future returns no operations", func(t *testing.T) { + ctx := getTestCtx("operations", []string{"operation_xdr"}) + futureTime := time.Now().Add(24 * time.Hour) + ops, err := resolver.Operations(ctx, parentAccount, &futureTime, nil, nil, nil, nil, nil) + require.NoError(t, err) + assert.Empty(t, ops.Edges) + }) + + t.Run("until before since returns error", func(t *testing.T) { + ctx := getTestCtx("operations", []string{"operation_xdr"}) + since := time.Now() + until := since.Add(-1 * time.Hour) + _, err := resolver.Operations(ctx, parentAccount, &since, &until, nil, nil, nil, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "until must not be before since") + }) +} + +func TestAccountResolver_StateChanges_WithTimeRange(t *testing.T) { + parentAccount := &types.Account{StellarAddress: types.AddressBytea(sharedTestAccountAddress)} + + mockMetricsService := &metrics.MockMetricsService{} + mockMetricsService.On("IncDBQuery", "BatchGetByAccountAddress", "state_changes").Return() + mockMetricsService.On("ObserveDBQueryDuration", "BatchGetByAccountAddress", "state_changes", mock.Anything).Return() + defer mockMetricsService.AssertExpectations(t) + + resolver := &accountResolver{&Resolver{ + models: &data.Models{ + StateChanges: &data.StateChangeModel{ + DB: testDBConnectionPool, + MetricsService: mockMetricsService, + }, + }, + }} + + t.Run("since in the past returns all state changes", func(t *testing.T) { + ctx := getTestCtx("state_changes", []string{""}) + pastTime := time.Now().Add(-24 * time.Hour) + sc, err := resolver.StateChanges(ctx, parentAccount, nil, &pastTime, nil, nil, nil, nil, nil) + require.NoError(t, err) + assert.Len(t, sc.Edges, 20) + }) + + t.Run("since in the future returns no state changes", func(t *testing.T) { + ctx := getTestCtx("state_changes", []string{""}) + futureTime := time.Now().Add(24 * time.Hour) + sc, err := resolver.StateChanges(ctx, parentAccount, nil, &futureTime, nil, nil, nil, nil, nil) + require.NoError(t, err) + assert.Empty(t, sc.Edges) + }) + + t.Run("until before since returns error", func(t *testing.T) { + ctx := getTestCtx("state_changes", []string{""}) + since := time.Now() + until := since.Add(-1 * time.Hour) + _, err := resolver.StateChanges(ctx, parentAccount, nil, &since, &until, nil, nil, nil, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "until must not be before since") + }) + + t.Run("time range combined with filter", func(t *testing.T) { + ctx := getTestCtx("state_changes", []string{""}) + pastTime := time.Now().Add(-24 * time.Hour) + txHash := testTxHash1 + filter := &graphql1.AccountStateChangeFilterInput{ + TransactionHash: &txHash, + } + sc, err := resolver.StateChanges(ctx, parentAccount, filter, &pastTime, nil, nil, nil, nil, nil) + require.NoError(t, err) + assert.Len(t, sc.Edges, 5) + }) +} From dd65f24c0ba9c2140891f2aa7704b0c5dbce7d3b Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Sun, 15 Feb 2026 10:24:14 -0500 Subject: [PATCH 62/77] fix make check --- internal/data/query_utils.go | 4 ++-- internal/serve/graphql/generated/generated.go | 5 +++-- internal/serve/graphql/resolvers/account_resolvers_test.go | 3 +-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/data/query_utils.go b/internal/data/query_utils.go index d61dd4ec8..e44bc4108 100644 --- a/internal/data/query_utils.go +++ b/internal/data/query_utils.go @@ -29,12 +29,12 @@ func appendTimeRangeConditions(qb *strings.Builder, column string, timeRange *Ti return args, argIndex } if timeRange.Since != nil { - qb.WriteString(fmt.Sprintf(" AND %s >= $%d", column, argIndex)) + fmt.Fprintf(qb, " AND %s >= $%d", column, argIndex) args = append(args, *timeRange.Since) argIndex++ } if timeRange.Until != nil { - qb.WriteString(fmt.Sprintf(" AND %s <= $%d", column, argIndex)) + fmt.Fprintf(qb, " AND %s <= $%d", column, argIndex) args = append(args, *timeRange.Until) argIndex++ } diff --git a/internal/serve/graphql/generated/generated.go b/internal/serve/graphql/generated/generated.go index 4b88427a7..0b7ec4768 100644 --- a/internal/serve/graphql/generated/generated.go +++ b/internal/serve/graphql/generated/generated.go @@ -14,10 +14,11 @@ import ( "github.com/99designs/gqlgen/graphql" "github.com/99designs/gqlgen/graphql/introspection" - "github.com/stellar/wallet-backend/internal/indexer/types" - "github.com/stellar/wallet-backend/internal/serve/graphql/scalars" gqlparser "github.com/vektah/gqlparser/v2" "github.com/vektah/gqlparser/v2/ast" + + "github.com/stellar/wallet-backend/internal/indexer/types" + "github.com/stellar/wallet-backend/internal/serve/graphql/scalars" ) // region ************************** generated!.gotpl ************************** diff --git a/internal/serve/graphql/resolvers/account_resolvers_test.go b/internal/serve/graphql/resolvers/account_resolvers_test.go index 33e39ae8c..52e15bf11 100644 --- a/internal/serve/graphql/resolvers/account_resolvers_test.go +++ b/internal/serve/graphql/resolvers/account_resolvers_test.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "fmt" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -15,8 +16,6 @@ import ( "github.com/stellar/wallet-backend/internal/indexer/types" "github.com/stellar/wallet-backend/internal/metrics" graphql1 "github.com/stellar/wallet-backend/internal/serve/graphql/generated" - - "time" ) // testOpXDRAcc returns the expected base64-encoded XDR for test operation N From a8a8057d23b7839de53ad3b7d70180fd6d2467fa Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Sun, 15 Feb 2026 10:27:17 -0500 Subject: [PATCH 63/77] Update query_utils.go --- internal/data/query_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/data/query_utils.go b/internal/data/query_utils.go index e44bc4108..388dbc629 100644 --- a/internal/data/query_utils.go +++ b/internal/data/query_utils.go @@ -97,7 +97,7 @@ type CursorColumn struct { Value interface{} } -// BuildDecomposedCursorCondition decomposes a ROW() tuple comparison into an equivalent +// buildDecomposedCursorCondition decomposes a ROW() tuple comparison into an equivalent // OR clause that TimescaleDB's ColumnarScan can push into vectorized filters. // // For example, (a, b, c) < ($1, $2, $3) becomes: From 7f83fe810964babd05101470e5729d033ee6e8a4 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Sun, 15 Feb 2026 18:27:11 -0500 Subject: [PATCH 64/77] Add config option for tweaking schedule_interval for compression --- cmd/ingest.go | 8 +++ internal/ingest/ingest.go | 5 +- internal/ingest/timescaledb.go | 38 ++++++++++-- internal/ingest/timescaledb_test.go | 96 +++++++++++++++++++++++++---- 4 files changed, 129 insertions(+), 18 deletions(-) diff --git a/cmd/ingest.go b/cmd/ingest.go index db039b157..f738bfd95 100644 --- a/cmd/ingest.go +++ b/cmd/ingest.go @@ -156,6 +156,14 @@ func (c *ingestCmd) Command() *cobra.Command { FlagDefault: "", Required: false, }, + { + Name: "compression-schedule-interval", + Usage: "How frequently the TimescaleDB compression policy job checks for chunks to compress. Does not change which chunks are eligible (that's controlled by compress_after). Empty skips configuration. Uses PostgreSQL INTERVAL syntax.", + OptType: types.String, + ConfigKey: &cfg.CompressionScheduleInterval, + FlagDefault: "", + Required: false, + }, } cmd := &cobra.Command{ diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index b205fb5ef..7d84a09e7 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -93,6 +93,9 @@ type Configs struct { // RetentionPeriod configures automatic data retention. Chunks older than this are dropped. // Empty string disables retention. Uses PostgreSQL INTERVAL syntax (e.g., "30 days", "6 months"). RetentionPeriod string + // CompressionScheduleInterval controls how frequently the compression policy job runs. + // Uses PostgreSQL INTERVAL syntax (e.g., "4 hours", "12 hours"). Empty string skips configuration. + CompressionScheduleInterval string } func Ingest(cfg Configs) error { @@ -141,7 +144,7 @@ func setupDeps(cfg Configs) (services.IngestService, error) { return nil, fmt.Errorf("getting sqlx db: %w", err) } - if err := configureHypertableSettings(ctx, dbConnectionPool, cfg.ChunkInterval, cfg.RetentionPeriod, cfg.OldestLedgerCursorName); err != nil { + if err := configureHypertableSettings(ctx, dbConnectionPool, cfg.ChunkInterval, cfg.RetentionPeriod, cfg.OldestLedgerCursorName, cfg.CompressionScheduleInterval); err != nil { return nil, fmt.Errorf("configuring hypertable settings: %w", err) } diff --git a/internal/ingest/timescaledb.go b/internal/ingest/timescaledb.go index 65c371b5c..7db45bfe6 100644 --- a/internal/ingest/timescaledb.go +++ b/internal/ingest/timescaledb.go @@ -20,12 +20,14 @@ var hypertables = []string{ "state_changes", } -// configureHypertableSettings applies chunk interval and retention policy settings -// to all hypertables. Chunk interval only affects future chunks. Retention policy -// is idempotent: any existing policy is removed before re-adding. When retention -// is enabled, a reconciliation job keeps oldest_ingest_ledger in sync with the -// actual minimum ledger remaining after chunk drops. -func configureHypertableSettings(ctx context.Context, pool db.ConnectionPool, chunkInterval, retentionPeriod, oldestCursorName string) error { +// configureHypertableSettings applies chunk interval, retention policy, and +// compression schedule settings to all hypertables. Chunk interval only affects +// future chunks. Retention policy is idempotent: any existing policy is removed +// before re-adding. When retention is enabled, a reconciliation job keeps +// oldest_ingest_ledger in sync with the actual minimum ledger remaining after +// chunk drops. Compression schedule interval updates how frequently existing +// compression policy jobs run (does not create new policies). +func configureHypertableSettings(ctx context.Context, pool db.ConnectionPool, chunkInterval, retentionPeriod, oldestCursorName, compressionScheduleInterval string) error { for _, table := range hypertables { if _, err := pool.ExecContext(ctx, "SELECT set_chunk_time_interval($1::regclass, $2::interval)", @@ -92,5 +94,29 @@ func configureHypertableSettings(ctx context.Context, pool db.ConnectionPool, ch log.Ctx(ctx).Infof("Scheduled reconcile_oldest_cursor job every %s for cursor %q", chunkInterval, oldestCursorName) } + if compressionScheduleInterval != "" { + for _, table := range hypertables { + var jobID int + err := pool.GetContext(ctx, &jobID, + `SELECT job_id FROM timescaledb_information.jobs + WHERE proc_name = 'policy_compression' + AND hypertable_name = $1`, + table, + ) + if err != nil { + log.Ctx(ctx).Warnf("No compression policy found for %s, skipping schedule update", table) + continue + } + + if _, err := pool.ExecContext(ctx, + "SELECT alter_job($1, schedule_interval => $2::interval)", + jobID, compressionScheduleInterval, + ); err != nil { + return fmt.Errorf("updating compression schedule interval on %s (job %d): %w", table, jobID, err) + } + log.Ctx(ctx).Infof("Set compression schedule interval %q on %s (job %d)", compressionScheduleInterval, table, jobID) + } + } + return nil } diff --git a/internal/ingest/timescaledb_test.go b/internal/ingest/timescaledb_test.go index 4dc165d52..f30ae86db 100644 --- a/internal/ingest/timescaledb_test.go +++ b/internal/ingest/timescaledb_test.go @@ -23,7 +23,7 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "7 days", "", "oldest_ledger_cursor") + err = configureHypertableSettings(ctx, dbConnectionPool, "7 days", "", "oldest_ledger_cursor", "") require.NoError(t, err) // Verify chunk interval was updated for all hypertables @@ -50,7 +50,7 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "") require.NoError(t, err) // Verify retention policy was created for all hypertables @@ -77,7 +77,7 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "", "oldest_ledger_cursor") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "", "oldest_ledger_cursor", "") require.NoError(t, err) // Verify no retention policies were created @@ -101,10 +101,10 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() // Apply retention policy twice with different values to simulate restarts - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "") require.NoError(t, err) - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "90 days", "oldest_ledger_cursor") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "90 days", "oldest_ledger_cursor", "") require.NoError(t, err) // Verify exactly 1 retention policy per table (not duplicated) @@ -131,7 +131,7 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "") require.NoError(t, err) // Verify reconciliation job was created @@ -155,10 +155,10 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() // Apply twice to simulate restarts - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "") require.NoError(t, err) - err = configureHypertableSettings(ctx, dbConnectionPool, "7 days", "90 days", "oldest_ledger_cursor") + err = configureHypertableSettings(ctx, dbConnectionPool, "7 days", "90 days", "oldest_ledger_cursor", "") require.NoError(t, err) // Verify exactly 1 reconciliation job (not duplicated) @@ -181,7 +181,7 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "", "oldest_ledger_cursor") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "", "oldest_ledger_cursor", "") require.NoError(t, err) // Verify no reconciliation job was created @@ -204,7 +204,7 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "not-an-interval", "", "oldest_ledger_cursor") + err = configureHypertableSettings(ctx, dbConnectionPool, "not-an-interval", "", "oldest_ledger_cursor", "") assert.Error(t, err) assert.Contains(t, err.Error(), "setting chunk interval") }) @@ -218,8 +218,82 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "not-an-interval", "oldest_ledger_cursor") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "not-an-interval", "oldest_ledger_cursor", "") assert.Error(t, err) assert.Contains(t, err.Error(), "adding retention policy") }) + + t.Run("compression_schedule_interval", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + + // Compression policies already exist from columnstore hypertable creation. + // Configure with a 4-hour compression schedule interval. + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "", "oldest_ledger_cursor", "4 hours") + require.NoError(t, err) + + // Verify schedule_interval was updated for all compression policy jobs + for _, table := range hypertables { + var intervalSecs float64 + err := dbConnectionPool.GetContext(ctx, &intervalSecs, + `SELECT EXTRACT(EPOCH FROM j.schedule_interval) + FROM timescaledb_information.jobs j + WHERE j.proc_name = 'policy_compression' + AND j.hypertable_name = $1`, + table, + ) + require.NoError(t, err, "querying compression schedule for %s", table) + // 4 hours in seconds = 4 * 60 * 60 + assert.Equal(t, float64(4*60*60), intervalSecs, "compression schedule interval for %s", table) + } + }) + + t.Run("no_compression_schedule_when_empty", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + + // Compression policies already exist from columnstore hypertable creation. + // Record the default schedule interval before calling configureHypertableSettings. + defaultIntervals := make(map[string]float64) + for _, table := range hypertables { + var intervalSecs float64 + err := dbConnectionPool.GetContext(ctx, &intervalSecs, + `SELECT EXTRACT(EPOCH FROM j.schedule_interval) + FROM timescaledb_information.jobs j + WHERE j.proc_name = 'policy_compression' + AND j.hypertable_name = $1`, + table, + ) + require.NoError(t, err, "querying default compression schedule for %s", table) + defaultIntervals[table] = intervalSecs + } + + // Configure with empty compression schedule interval (should skip) + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "", "oldest_ledger_cursor", "") + require.NoError(t, err) + + // Verify schedule_interval was NOT changed + for _, table := range hypertables { + var intervalSecs float64 + err := dbConnectionPool.GetContext(ctx, &intervalSecs, + `SELECT EXTRACT(EPOCH FROM j.schedule_interval) + FROM timescaledb_information.jobs j + WHERE j.proc_name = 'policy_compression' + AND j.hypertable_name = $1`, + table, + ) + require.NoError(t, err, "querying compression schedule for %s", table) + assert.Equal(t, defaultIntervals[table], intervalSecs, "compression schedule interval should remain unchanged for %s", table) + } + }) } From b1af6802aeffe85a894ee30e160aeae811a17078 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Sun, 15 Feb 2026 18:51:54 -0500 Subject: [PATCH 65/77] Add CLI variable for compress_after setting --- cmd/ingest.go | 8 +++ internal/ingest/ingest.go | 5 +- internal/ingest/timescaledb.go | 30 ++++++++- internal/ingest/timescaledb_test.go | 99 +++++++++++++++++++++++++---- 4 files changed, 127 insertions(+), 15 deletions(-) diff --git a/cmd/ingest.go b/cmd/ingest.go index f738bfd95..4fa79a1d9 100644 --- a/cmd/ingest.go +++ b/cmd/ingest.go @@ -164,6 +164,14 @@ func (c *ingestCmd) Command() *cobra.Command { FlagDefault: "", Required: false, }, + { + Name: "compression-compress-after", + Usage: "How long after a chunk is closed before it becomes eligible for compression. Lower values reduce the number of uncompressed chunks. Empty skips configuration. Uses PostgreSQL INTERVAL syntax.", + OptType: types.String, + ConfigKey: &cfg.CompressAfter, + FlagDefault: "", + Required: false, + }, } cmd := &cobra.Command{ diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 7d84a09e7..44697f533 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -96,6 +96,9 @@ type Configs struct { // CompressionScheduleInterval controls how frequently the compression policy job runs. // Uses PostgreSQL INTERVAL syntax (e.g., "4 hours", "12 hours"). Empty string skips configuration. CompressionScheduleInterval string + // CompressAfter controls how long after a chunk is closed before it becomes eligible for compression. + // Uses PostgreSQL INTERVAL syntax (e.g., "1 hour", "12 hours"). Empty string skips configuration. + CompressAfter string } func Ingest(cfg Configs) error { @@ -144,7 +147,7 @@ func setupDeps(cfg Configs) (services.IngestService, error) { return nil, fmt.Errorf("getting sqlx db: %w", err) } - if err := configureHypertableSettings(ctx, dbConnectionPool, cfg.ChunkInterval, cfg.RetentionPeriod, cfg.OldestLedgerCursorName, cfg.CompressionScheduleInterval); err != nil { + if err := configureHypertableSettings(ctx, dbConnectionPool, cfg.ChunkInterval, cfg.RetentionPeriod, cfg.OldestLedgerCursorName, cfg.CompressionScheduleInterval, cfg.CompressAfter); err != nil { return nil, fmt.Errorf("configuring hypertable settings: %w", err) } diff --git a/internal/ingest/timescaledb.go b/internal/ingest/timescaledb.go index 7db45bfe6..938ce679e 100644 --- a/internal/ingest/timescaledb.go +++ b/internal/ingest/timescaledb.go @@ -27,7 +27,9 @@ var hypertables = []string{ // oldest_ingest_ledger in sync with the actual minimum ledger remaining after // chunk drops. Compression schedule interval updates how frequently existing // compression policy jobs run (does not create new policies). -func configureHypertableSettings(ctx context.Context, pool db.ConnectionPool, chunkInterval, retentionPeriod, oldestCursorName, compressionScheduleInterval string) error { +// Compress after updates how long after a chunk closes before it becomes +// eligible for compression. +func configureHypertableSettings(ctx context.Context, pool db.ConnectionPool, chunkInterval, retentionPeriod, oldestCursorName, compressionScheduleInterval, compressAfter string) error { for _, table := range hypertables { if _, err := pool.ExecContext(ctx, "SELECT set_chunk_time_interval($1::regclass, $2::interval)", @@ -118,5 +120,31 @@ func configureHypertableSettings(ctx context.Context, pool db.ConnectionPool, ch } } + if compressAfter != "" { + for _, table := range hypertables { + var jobID int + err := pool.GetContext(ctx, &jobID, + `SELECT job_id FROM timescaledb_information.jobs + WHERE proc_name = 'policy_compression' + AND hypertable_name = $1`, + table, + ) + if err != nil { + log.Ctx(ctx).Warnf("No compression policy found for %s, skipping compress_after update", table) + continue + } + + if _, err := pool.ExecContext(ctx, + `SELECT alter_job($1, config => jsonb_set( + (SELECT config FROM timescaledb_information.jobs WHERE job_id = $1), + '{compress_after}', to_jsonb($2::text)))`, + jobID, compressAfter, + ); err != nil { + return fmt.Errorf("updating compress_after on %s (job %d): %w", table, jobID, err) + } + log.Ctx(ctx).Infof("Set compress_after %q on %s (job %d)", compressAfter, table, jobID) + } + } + return nil } diff --git a/internal/ingest/timescaledb_test.go b/internal/ingest/timescaledb_test.go index f30ae86db..fe2e351de 100644 --- a/internal/ingest/timescaledb_test.go +++ b/internal/ingest/timescaledb_test.go @@ -23,7 +23,7 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "7 days", "", "oldest_ledger_cursor", "") + err = configureHypertableSettings(ctx, dbConnectionPool, "7 days", "", "oldest_ledger_cursor", "", "") require.NoError(t, err) // Verify chunk interval was updated for all hypertables @@ -50,7 +50,7 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "", "") require.NoError(t, err) // Verify retention policy was created for all hypertables @@ -77,7 +77,7 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "", "oldest_ledger_cursor", "") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "", "oldest_ledger_cursor", "", "") require.NoError(t, err) // Verify no retention policies were created @@ -101,10 +101,10 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() // Apply retention policy twice with different values to simulate restarts - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "", "") require.NoError(t, err) - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "90 days", "oldest_ledger_cursor", "") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "90 days", "oldest_ledger_cursor", "", "") require.NoError(t, err) // Verify exactly 1 retention policy per table (not duplicated) @@ -131,7 +131,7 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "", "") require.NoError(t, err) // Verify reconciliation job was created @@ -155,10 +155,10 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() // Apply twice to simulate restarts - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "", "") require.NoError(t, err) - err = configureHypertableSettings(ctx, dbConnectionPool, "7 days", "90 days", "oldest_ledger_cursor", "") + err = configureHypertableSettings(ctx, dbConnectionPool, "7 days", "90 days", "oldest_ledger_cursor", "", "") require.NoError(t, err) // Verify exactly 1 reconciliation job (not duplicated) @@ -181,7 +181,7 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "", "oldest_ledger_cursor", "") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "", "oldest_ledger_cursor", "", "") require.NoError(t, err) // Verify no reconciliation job was created @@ -204,7 +204,7 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "not-an-interval", "", "oldest_ledger_cursor", "") + err = configureHypertableSettings(ctx, dbConnectionPool, "not-an-interval", "", "oldest_ledger_cursor", "", "") assert.Error(t, err) assert.Contains(t, err.Error(), "setting chunk interval") }) @@ -218,7 +218,7 @@ func TestConfigureHypertableSettings(t *testing.T) { ctx := context.Background() - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "not-an-interval", "oldest_ledger_cursor", "") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "not-an-interval", "oldest_ledger_cursor", "", "") assert.Error(t, err) assert.Contains(t, err.Error(), "adding retention policy") }) @@ -234,7 +234,7 @@ func TestConfigureHypertableSettings(t *testing.T) { // Compression policies already exist from columnstore hypertable creation. // Configure with a 4-hour compression schedule interval. - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "", "oldest_ledger_cursor", "4 hours") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "", "oldest_ledger_cursor", "4 hours", "") require.NoError(t, err) // Verify schedule_interval was updated for all compression policy jobs @@ -253,6 +253,79 @@ func TestConfigureHypertableSettings(t *testing.T) { } }) + t.Run("compress_after", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + + // Compression policies already exist from columnstore hypertable creation. + // Configure with a 12-hour compress_after value. + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "", "oldest_ledger_cursor", "", "12 hours") + require.NoError(t, err) + + // Verify compress_after was updated in the config JSONB for all compression policy jobs + for _, table := range hypertables { + var compressAfter string + err := dbConnectionPool.GetContext(ctx, &compressAfter, + `SELECT j.config->>'compress_after' + FROM timescaledb_information.jobs j + WHERE j.proc_name = 'policy_compression' + AND j.hypertable_name = $1`, + table, + ) + require.NoError(t, err, "querying compress_after for %s", table) + assert.Equal(t, "12 hours", compressAfter, "compress_after for %s", table) + } + }) + + t.Run("no_compress_after_when_empty", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + ctx := context.Background() + + // Compression policies already exist from columnstore hypertable creation. + // Record the default compress_after value before calling configureHypertableSettings. + defaultValues := make(map[string]string) + for _, table := range hypertables { + var compressAfter string + err := dbConnectionPool.GetContext(ctx, &compressAfter, + `SELECT j.config->>'compress_after' + FROM timescaledb_information.jobs j + WHERE j.proc_name = 'policy_compression' + AND j.hypertable_name = $1`, + table, + ) + require.NoError(t, err, "querying default compress_after for %s", table) + defaultValues[table] = compressAfter + } + + // Configure with empty compress_after (should skip) + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "", "oldest_ledger_cursor", "", "") + require.NoError(t, err) + + // Verify compress_after was NOT changed + for _, table := range hypertables { + var compressAfter string + err := dbConnectionPool.GetContext(ctx, &compressAfter, + `SELECT j.config->>'compress_after' + FROM timescaledb_information.jobs j + WHERE j.proc_name = 'policy_compression' + AND j.hypertable_name = $1`, + table, + ) + require.NoError(t, err, "querying compress_after for %s", table) + assert.Equal(t, defaultValues[table], compressAfter, "compress_after should remain unchanged for %s", table) + } + }) + t.Run("no_compression_schedule_when_empty", func(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -279,7 +352,7 @@ func TestConfigureHypertableSettings(t *testing.T) { } // Configure with empty compression schedule interval (should skip) - err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "", "oldest_ledger_cursor", "") + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "", "oldest_ledger_cursor", "", "") require.NoError(t, err) // Verify schedule_interval was NOT changed From 2a4d46957edf65ae06658b54fb3db56bf7418e81 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 16 Feb 2026 11:04:34 -0500 Subject: [PATCH 66/77] remove accountsByStateChange data loader --- internal/data/accounts.go | 31 -------- internal/data/accounts_test.go | 71 ------------------- internal/indexer/types/types.go | 5 -- .../graphql/dataloaders/account_loaders.go | 37 +--------- internal/serve/graphql/dataloaders/loaders.go | 5 -- internal/serve/graphql/resolvers/resolver.go | 25 +++---- .../resolvers/statechange.resolvers.go | 18 ++--- .../resolvers/statechange_resolvers_test.go | 53 +++++--------- 8 files changed, 37 insertions(+), 208 deletions(-) diff --git a/internal/data/accounts.go b/internal/data/accounts.go index d2450c242..e2c655de9 100644 --- a/internal/data/accounts.go +++ b/internal/data/accounts.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "strings" "time" "github.com/jackc/pgx/v5" @@ -226,33 +225,3 @@ func (m *AccountModel) BatchGetByOperationIDs(ctx context.Context, operationIDs m.MetricsService.IncDBQuery("BatchGetByOperationIDs", "accounts") return accounts, nil } - -// BatchGetByStateChangeIDs gets the accounts that are associated with the given state change IDs. -func (m *AccountModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs []int64, scOpIDs []int64, scOrders []int64, columns string) ([]*types.AccountWithStateChangeID, error) { - // Build tuples for the IN clause. Since (to_id, operation_id, state_change_order) is the primary key of state_changes, - // it will be faster to search on this tuple. - tuples := make([]string, len(scOrders)) - for i := range scOrders { - tuples[i] = fmt.Sprintf("(%d, %d, %d)", scToIDs[i], scOpIDs[i], scOrders[i]) - } - - query := fmt.Sprintf(` - SELECT account_id AS stellar_address, CONCAT(to_id, '-', operation_id, '-', state_change_order) AS state_change_id - FROM state_changes - WHERE (to_id, operation_id, state_change_order) IN (%s) - ORDER BY ledger_created_at DESC - `, strings.Join(tuples, ", ")) - - var accountsWithStateChanges []*types.AccountWithStateChangeID - start := time.Now() - err := m.DB.SelectContext(ctx, &accountsWithStateChanges, query) - duration := time.Since(start).Seconds() - m.MetricsService.ObserveDBQueryDuration("BatchGetByStateChangeIDs", "accounts", duration) - m.MetricsService.ObserveDBBatchSize("BatchGetByStateChangeIDs", "accounts", len(scOrders)) - if err != nil { - m.MetricsService.IncDBQueryError("BatchGetByStateChangeIDs", "accounts", utils.GetDBErrorType(err)) - return nil, fmt.Errorf("getting accounts by state change IDs: %w", err) - } - m.MetricsService.IncDBQuery("BatchGetByStateChangeIDs", "accounts") - return accountsWithStateChanges, nil -} diff --git a/internal/data/accounts_test.go b/internal/data/accounts_test.go index 65729f27e..c97815637 100644 --- a/internal/data/accounts_test.go +++ b/internal/data/accounts_test.go @@ -385,74 +385,3 @@ func TestAccountModel_IsAccountFeeBumpEligible(t *testing.T) { require.NoError(t, err) assert.True(t, isFeeBumpEligible) } - -func TestAccountModelBatchGetByStateChangeIDs(t *testing.T) { - dbt := dbtest.Open(t) - defer dbt.Close() - dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) - require.NoError(t, err) - defer dbConnectionPool.Close() - - mockMetricsService := metrics.NewMockMetricsService() - mockMetricsService.On("ObserveDBQueryDuration", "BatchGetByStateChangeIDs", "accounts", mock.Anything).Return() - mockMetricsService.On("IncDBQuery", "BatchGetByStateChangeIDs", "accounts").Return() - mockMetricsService.On("ObserveDBBatchSize", "BatchGetByStateChangeIDs", "accounts", mock.Anything).Return().Maybe() - defer mockMetricsService.AssertExpectations(t) - - m := &AccountModel{ - DB: dbConnectionPool, - MetricsService: mockMetricsService, - } - - ctx := context.Background() - address1 := keypair.MustRandom().Address() - address2 := keypair.MustRandom().Address() - toID1 := int64(4096) - toID2 := int64(8192) - stateChangeOrder1 := int64(1) - stateChangeOrder2 := int64(1) - - // Insert test accounts (stellar_address is BYTEA) - _, err = m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)", - types.AddressBytea(address1), types.AddressBytea(address2)) - require.NoError(t, err) - - // Insert test transactions first (hash is BYTEA, using valid 64-char hex strings) - testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001") - testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002") - _, err = m.DB.ExecContext(ctx, "INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at) VALUES ($1, 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, NOW()), ($2, 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, NOW())", testHash1, testHash2) - require.NoError(t, err) - - // Insert test operations (IDs must be in TOID range for each transaction) - xdr1 := types.XDRBytea([]byte("xdr1")) - xdr2 := types.XDRBytea([]byte("xdr2")) - _, err = m.DB.ExecContext(ctx, "INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) VALUES (4097, 'PAYMENT', $1, 'op_success', true, 1, NOW()), (8193, 'PAYMENT', $2, 'op_success', true, 2, NOW())", xdr1, xdr2) - require.NoError(t, err) - - // Insert test state changes that reference the accounts (state_changes.account_id is TEXT) - _, err = m.DB.ExecContext(ctx, ` - INSERT INTO state_changes ( - to_id, state_change_order, state_change_category, ledger_created_at, - ledger_number, account_id, operation_id - ) VALUES - ($1, $2, 'BALANCE', NOW(), 1, $3, 4097), - ($4, $5, 'BALANCE', NOW(), 2, $6, 8193) - `, toID1, stateChangeOrder1, types.AddressBytea(address1), toID2, stateChangeOrder2, types.AddressBytea(address2)) - require.NoError(t, err) - - // Test BatchGetByStateChangeIDs function - scToIDs := []int64{toID1, toID2} - scOpIDs := []int64{4097, 8193} - scOrders := []int64{stateChangeOrder1, stateChangeOrder2} - accounts, err := m.BatchGetByStateChangeIDs(ctx, scToIDs, scOpIDs, scOrders, "") - require.NoError(t, err) - assert.Len(t, accounts, 2) - - // Verify accounts are returned with correct state_change_id (format: to_id-operation_id-state_change_order) - addressSet := make(map[string]string) - for _, acc := range accounts { - addressSet[string(acc.StellarAddress)] = acc.StateChangeID - } - assert.Equal(t, "4096-4097-1", addressSet[address1]) - assert.Equal(t, "8192-8193-1", addressSet[address2]) -} diff --git a/internal/indexer/types/types.go b/internal/indexer/types/types.go index 20717e916..f3bdc59b5 100644 --- a/internal/indexer/types/types.go +++ b/internal/indexer/types/types.go @@ -436,11 +436,6 @@ type OperationWithStateChangeID struct { StateChangeID string `db:"state_change_id"` } -type AccountWithStateChangeID struct { - Account - StateChangeID string `db:"state_change_id"` -} - type StateChangeCategory string const ( diff --git a/internal/serve/graphql/dataloaders/account_loaders.go b/internal/serve/graphql/dataloaders/account_loaders.go index ed2c1a295..5116c12d6 100644 --- a/internal/serve/graphql/dataloaders/account_loaders.go +++ b/internal/serve/graphql/dataloaders/account_loaders.go @@ -2,7 +2,6 @@ package dataloaders import ( "context" - "fmt" "github.com/vikstrous/dataloadgen" @@ -11,10 +10,9 @@ import ( ) type AccountColumnsKey struct { - ToID int64 - OperationID int64 - StateChangeID string - Columns string + ToID int64 + OperationID int64 + Columns string } // accountsByToIDLoader creates a dataloader for fetching accounts by transaction ToID @@ -66,32 +64,3 @@ func accountsByOperationIDLoader(models *data.Models) *dataloadgen.Loader[Accoun }, ) } - -// accountByStateChangeIDLoader creates a dataloader for fetching accounts by state change ID -// This prevents N+1 queries when multiple state changes request their accounts -// The loader batches multiple state change IDs into a single database query -func accountByStateChangeIDLoader(models *data.Models) *dataloadgen.Loader[AccountColumnsKey, *types.Account] { - return newOneToOneLoader( - func(ctx context.Context, keys []AccountColumnsKey) ([]*types.AccountWithStateChangeID, error) { - columns := keys[0].Columns - scIDs := make([]string, len(keys)) - for i, key := range keys { - scIDs[i] = key.StateChangeID - } - scToIDs, scOpIDs, scOrders, err := parseStateChangeIDs(scIDs) - if err != nil { - return nil, fmt.Errorf("parsing state change IDs: %w", err) - } - return models.Account.BatchGetByStateChangeIDs(ctx, scToIDs, scOpIDs, scOrders, columns) - }, - func(item *types.AccountWithStateChangeID) string { - return item.StateChangeID - }, - func(key AccountColumnsKey) string { - return key.StateChangeID - }, - func(item *types.AccountWithStateChangeID) types.Account { - return item.Account - }, - ) -} diff --git a/internal/serve/graphql/dataloaders/loaders.go b/internal/serve/graphql/dataloaders/loaders.go index f144c2b71..e7b6c6898 100644 --- a/internal/serve/graphql/dataloaders/loaders.go +++ b/internal/serve/graphql/dataloaders/loaders.go @@ -48,10 +48,6 @@ type Dataloaders struct { // TransactionByStateChangeIDLoader batches requests for transactions by state change ID // Used by StateChange.transaction field resolver to prevent N+1 queries TransactionByStateChangeIDLoader *dataloadgen.Loader[TransactionColumnsKey, *types.Transaction] - - // AccountByStateChangeIDLoader batches requests for accounts by state change ID - // Used by StateChange.account field resolver to prevent N+1 queries - AccountByStateChangeIDLoader *dataloadgen.Loader[AccountColumnsKey, *types.Account] } // NewDataloaders creates a new instance of all dataloaders @@ -68,7 +64,6 @@ func NewDataloaders(models *data.Models) *Dataloaders { StateChangesByOperationIDLoader: stateChangesByOperationIDLoader(models), AccountsByToIDLoader: accountsByToIDLoader(models), AccountsByOperationIDLoader: accountsByOperationIDLoader(models), - AccountByStateChangeIDLoader: accountByStateChangeIDLoader(models), } } diff --git a/internal/serve/graphql/resolvers/resolver.go b/internal/serve/graphql/resolvers/resolver.go index b853bc102..26e74cf66 100644 --- a/internal/serve/graphql/resolvers/resolver.go +++ b/internal/serve/graphql/resolvers/resolver.go @@ -144,22 +144,15 @@ func (r *Resolver) resolveRequiredJSONBField(field interface{}) (string, error) // Shared resolver functions for BaseStateChange interface // These functions provide common logic that all state change types can use -// resolveStateChangeAccount resolves the account field for any state change type -// Since state changes have a direct account_id reference, we can fetch the account directly -func (r *Resolver) resolveStateChangeAccount(ctx context.Context, toID int64, operationID int64, stateChangeOrder int64) (*types.Account, error) { - loaders := ctx.Value(middleware.LoadersKey).(*dataloaders.Dataloaders) - dbColumns := GetDBColumnsForFields(ctx, types.Account{}) - - stateChangeID := fmt.Sprintf("%d-%d-%d", toID, operationID, stateChangeOrder) - loaderKey := dataloaders.AccountColumnsKey{ - StateChangeID: stateChangeID, - Columns: strings.Join(dbColumns, ", "), - } - account, err := loaders.AccountByStateChangeIDLoader.Load(ctx, loaderKey) - if err != nil { - return nil, fmt.Errorf("loading account for state change %s: %w", stateChangeID, err) - } - return account, nil +// resolveStateChangeAccount resolves the account field for any state change type. +// Uses the already-populated AccountID from the state change row to avoid a re-query. +func (r *Resolver) resolveStateChangeAccount(accountID types.AddressBytea) (*types.Account, error) { + if accountID == "" { + return nil, fmt.Errorf("state change has no account_id") + } + return &types.Account{ + StellarAddress: accountID, + }, nil } // resolveStateChangeOperation resolves the operation field for any state change type diff --git a/internal/serve/graphql/resolvers/statechange.resolvers.go b/internal/serve/graphql/resolvers/statechange.resolvers.go index 9583c2b93..4ff13dcce 100644 --- a/internal/serve/graphql/resolvers/statechange.resolvers.go +++ b/internal/serve/graphql/resolvers/statechange.resolvers.go @@ -24,7 +24,7 @@ func (r *accountChangeResolver) Reason(ctx context.Context, obj *types.AccountSt // Account is the resolver for the account field. func (r *accountChangeResolver) Account(ctx context.Context, obj *types.AccountStateChangeModel) (*types.Account, error) { - return r.resolveStateChangeAccount(ctx, obj.ToID, obj.OperationID, obj.StateChangeOrder) + return r.resolveStateChangeAccount(obj.AccountID) } // Operation is the resolver for the operation field. @@ -54,7 +54,7 @@ func (r *balanceAuthorizationChangeResolver) Reason(ctx context.Context, obj *ty // Account is the resolver for the account field. func (r *balanceAuthorizationChangeResolver) Account(ctx context.Context, obj *types.BalanceAuthorizationStateChangeModel) (*types.Account, error) { - return r.resolveStateChangeAccount(ctx, obj.ToID, obj.OperationID, obj.StateChangeOrder) + return r.resolveStateChangeAccount(obj.AccountID) } // Operation is the resolver for the operation field. @@ -98,7 +98,7 @@ func (r *flagsChangeResolver) Reason(ctx context.Context, obj *types.FlagsStateC // Account is the resolver for the account field. func (r *flagsChangeResolver) Account(ctx context.Context, obj *types.FlagsStateChangeModel) (*types.Account, error) { - return r.resolveStateChangeAccount(ctx, obj.ToID, obj.OperationID, obj.StateChangeOrder) + return r.resolveStateChangeAccount(obj.AccountID) } // Operation is the resolver for the operation field. @@ -132,7 +132,7 @@ func (r *metadataChangeResolver) Reason(ctx context.Context, obj *types.Metadata // Account is the resolver for the account field. func (r *metadataChangeResolver) Account(ctx context.Context, obj *types.MetadataStateChangeModel) (*types.Account, error) { - return r.resolveStateChangeAccount(ctx, obj.ToID, obj.OperationID, obj.StateChangeOrder) + return r.resolveStateChangeAccount(obj.AccountID) } // Operation is the resolver for the operation field. @@ -162,7 +162,7 @@ func (r *reservesChangeResolver) Reason(ctx context.Context, obj *types.Reserves // Account is the resolver for the account field. func (r *reservesChangeResolver) Account(ctx context.Context, obj *types.ReservesStateChangeModel) (*types.Account, error) { - return r.resolveStateChangeAccount(ctx, obj.ToID, obj.OperationID, obj.StateChangeOrder) + return r.resolveStateChangeAccount(obj.AccountID) } // Operation is the resolver for the operation field. @@ -217,7 +217,7 @@ func (r *signerChangeResolver) Reason(ctx context.Context, obj *types.SignerStat // Account is the resolver for the account field. func (r *signerChangeResolver) Account(ctx context.Context, obj *types.SignerStateChangeModel) (*types.Account, error) { - return r.resolveStateChangeAccount(ctx, obj.ToID, obj.OperationID, obj.StateChangeOrder) + return r.resolveStateChangeAccount(obj.AccountID) } // Operation is the resolver for the operation field. @@ -258,7 +258,7 @@ func (r *signerThresholdsChangeResolver) Reason(ctx context.Context, obj *types. // Account is the resolver for the account field. func (r *signerThresholdsChangeResolver) Account(ctx context.Context, obj *types.SignerThresholdsStateChangeModel) (*types.Account, error) { - return r.resolveStateChangeAccount(ctx, obj.ToID, obj.OperationID, obj.StateChangeOrder) + return r.resolveStateChangeAccount(obj.AccountID) } // Operation is the resolver for the operation field. @@ -291,7 +291,7 @@ func (r *standardBalanceChangeResolver) Reason(ctx context.Context, obj *types.S // Account is the resolver for the account field. func (r *standardBalanceChangeResolver) Account(ctx context.Context, obj *types.StandardBalanceStateChangeModel) (*types.Account, error) { - return r.resolveStateChangeAccount(ctx, obj.ToID, obj.OperationID, obj.StateChangeOrder) + return r.resolveStateChangeAccount(obj.AccountID) } // Operation is the resolver for the operation field. @@ -326,7 +326,7 @@ func (r *trustlineChangeResolver) Reason(ctx context.Context, obj *types.Trustli // Account is the resolver for the account field. func (r *trustlineChangeResolver) Account(ctx context.Context, obj *types.TrustlineStateChangeModel) (*types.Account, error) { - return r.resolveStateChangeAccount(ctx, obj.ToID, obj.OperationID, obj.StateChangeOrder) + return r.resolveStateChangeAccount(obj.AccountID) } // Operation is the resolver for the operation field. diff --git a/internal/serve/graphql/resolvers/statechange_resolvers_test.go b/internal/serve/graphql/resolvers/statechange_resolvers_test.go index 8076e5ace..ba2ae1f8b 100644 --- a/internal/serve/graphql/resolvers/statechange_resolvers_test.go +++ b/internal/serve/graphql/resolvers/statechange_resolvers_test.go @@ -224,34 +224,16 @@ func TestStateChangeResolver_TypedFields(t *testing.T) { } func TestStateChangeResolver_Account(t *testing.T) { - mockMetricsService := &metrics.MockMetricsService{} - mockMetricsService.On("IncDBQuery", "BatchGetByStateChangeIDs", "accounts").Return() - mockMetricsService.On("ObserveDBQueryDuration", "BatchGetByStateChangeIDs", "accounts", mock.Anything).Return() - mockMetricsService.On("ObserveDBBatchSize", "BatchGetByStateChangeIDs", "accounts", mock.Anything).Return() - defer mockMetricsService.AssertExpectations(t) - - resolver := &standardBalanceChangeResolver{&Resolver{ - models: &data.Models{ - Account: &data.AccountModel{ - DB: testDBConnectionPool, - MetricsService: mockMetricsService, - }, - }, - }} - opID := toid.New(1000, 1, 1).ToInt64() - txToID := opID &^ 0xFFF // Derive transaction to_id from operation_id using TOID bitmask - parentSC := types.StandardBalanceStateChangeModel{ - StateChange: types.StateChange{ - ToID: txToID, - OperationID: opID, - StateChangeOrder: 1, - StateChangeCategory: types.StateChangeCategoryBalance, - }, - } + resolver := &standardBalanceChangeResolver{&Resolver{}} t.Run("success", func(t *testing.T) { - loaders := dataloaders.NewDataloaders(resolver.models) - ctx := context.WithValue(getTestCtx("accounts", []string{""}), middleware.LoadersKey, loaders) + parentSC := types.StandardBalanceStateChangeModel{ + StateChange: types.StateChange{ + AccountID: types.AddressBytea(sharedTestAccountAddress), + StateChangeCategory: types.StateChangeCategoryBalance, + }, + } + ctx := context.Background() account, err := resolver.Account(ctx, &parentSC) require.NoError(t, err) @@ -259,29 +241,26 @@ func TestStateChangeResolver_Account(t *testing.T) { }) t.Run("nil state change panics", func(t *testing.T) { - loaders := dataloaders.NewDataloaders(resolver.models) - ctx := context.WithValue(getTestCtx("accounts", []string{""}), middleware.LoadersKey, loaders) + ctx := context.Background() assert.Panics(t, func() { _, _ = resolver.Account(ctx, nil) //nolint:errcheck }) }) - t.Run("state change with non-existent account", func(t *testing.T) { - nonExistentSC := types.StandardBalanceStateChangeModel{ + t.Run("state change with empty account_id returns error", func(t *testing.T) { + emptySC := types.StandardBalanceStateChangeModel{ StateChange: types.StateChange{ - ToID: 9999, - OperationID: 0, - StateChangeOrder: 1, + AccountID: "", StateChangeCategory: types.StateChangeCategoryBalance, }, } - loaders := dataloaders.NewDataloaders(resolver.models) - ctx := context.WithValue(getTestCtx("accounts", []string{""}), middleware.LoadersKey, loaders) + ctx := context.Background() - account, err := resolver.Account(ctx, &nonExistentSC) - require.NoError(t, err) // Dataloader returns nil, not error for missing data + account, err := resolver.Account(ctx, &emptySC) + require.Error(t, err) assert.Nil(t, account) + assert.Contains(t, err.Error(), "state change has no account_id") }) } From 18891c990616b25e722e713aa2a6ee5eefb6d592 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 16 Feb 2026 11:14:11 -0500 Subject: [PATCH 67/77] remove account {address} when getting an account's state changes --- pkg/wbclient/queries.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/wbclient/queries.go b/pkg/wbclient/queries.go index 4d5e5f373..7295a1c6e 100644 --- a/pkg/wbclient/queries.go +++ b/pkg/wbclient/queries.go @@ -45,9 +45,7 @@ const ( ingestedAt ledgerCreatedAt ledgerNumber - account { - address - } + ... on StandardBalanceChange { standardBalanceTokenId: tokenId amount From b30ce333fa83626f6e602f012e79728ca994994f Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 16 Feb 2026 11:21:23 -0500 Subject: [PATCH 68/77] Update statechanges.go --- internal/data/statechanges.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/data/statechanges.go b/internal/data/statechanges.go index b58c57c52..16fbaefb3 100644 --- a/internal/data/statechanges.go +++ b/internal/data/statechanges.go @@ -24,7 +24,7 @@ type StateChangeModel struct { // BatchGetByAccountAddress gets the state changes that are associated with the given account address. // Optional filters: txHash, operationID, category, and reason can be used to further filter results. func (m *StateChangeModel) BatchGetByAccountAddress(ctx context.Context, accountAddress string, txHash *string, operationID *int64, category *string, reason *string, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder, timeRange *TimeRange) ([]*types.StateChangeWithCursor, error) { - columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_order") + columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_order", "account_id") var queryBuilder strings.Builder args := []interface{}{types.AddressBytea(accountAddress)} argIndex := 2 @@ -116,7 +116,7 @@ func (m *StateChangeModel) BatchGetByAccountAddress(ctx context.Context, account } func (m *StateChangeModel) GetAll(ctx context.Context, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { - columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_order") + columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_order", "account_id") var queryBuilder strings.Builder var args []interface{} argIndex := 1 @@ -289,7 +289,7 @@ func (m *StateChangeModel) BatchCopy( // BatchGetByToID gets state changes for a single transaction with pagination support. func (m *StateChangeModel) BatchGetByToID(ctx context.Context, toID int64, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { - columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_order") + columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_order", "account_id") var queryBuilder strings.Builder queryBuilder.WriteString(fmt.Sprintf(` SELECT %s, to_id as "cursor.cursor_to_id", operation_id as "cursor.cursor_operation_id", state_change_order as "cursor.cursor_state_change_order" @@ -348,7 +348,7 @@ func (m *StateChangeModel) BatchGetByToID(ctx context.Context, toID int64, colum // BatchGetByToIDs gets the state changes that are associated with the given to_ids. func (m *StateChangeModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, columns string, limit *int32, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { - columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_order") + columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_order", "account_id") var queryBuilder strings.Builder // This CTE query implements per-transaction pagination to ensure balanced results. // Instead of applying a global LIMIT that could return all state changes from just a few @@ -400,7 +400,7 @@ func (m *StateChangeModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, c // BatchGetByOperationID gets state changes for a single operation with pagination support. func (m *StateChangeModel) BatchGetByOperationID(ctx context.Context, operationID int64, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { - columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_order") + columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_order", "account_id") var queryBuilder strings.Builder queryBuilder.WriteString(fmt.Sprintf(` SELECT %s, to_id as "cursor.cursor_to_id", operation_id as "cursor.cursor_operation_id", state_change_order as "cursor.cursor_state_change_order" @@ -459,7 +459,7 @@ func (m *StateChangeModel) BatchGetByOperationID(ctx context.Context, operationI // BatchGetByOperationIDs gets the state changes that are associated with the given operation IDs. func (m *StateChangeModel) BatchGetByOperationIDs(ctx context.Context, operationIDs []int64, columns string, limit *int32, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { - columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_order") + columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_order", "account_id") var queryBuilder strings.Builder // This CTE query implements per-operation pagination to ensure balanced results. // Instead of applying a global LIMIT that could return all state changes from just a few From 6ebcca2a831c61d0e94d6384c3491cdfbaa79831 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 16 Feb 2026 16:18:39 -0500 Subject: [PATCH 69/77] Always set endCursor to edgers[len-1] --- .../resolvers/account_resolvers_test.go | 18 ++++++++--------- .../resolvers/queries_resolvers_test.go | 20 +++++++++---------- .../resolvers/transaction_resolvers_test.go | 4 ++-- internal/serve/graphql/resolvers/utils.go | 6 +----- 4 files changed, 22 insertions(+), 26 deletions(-) diff --git a/internal/serve/graphql/resolvers/account_resolvers_test.go b/internal/serve/graphql/resolvers/account_resolvers_test.go index 52e15bf11..90b9b4bf0 100644 --- a/internal/serve/graphql/resolvers/account_resolvers_test.go +++ b/internal/serve/graphql/resolvers/account_resolvers_test.go @@ -89,9 +89,9 @@ func TestAccountResolver_Transactions(t *testing.T) { assert.False(t, txs.PageInfo.HasNextPage) assert.True(t, txs.PageInfo.HasPreviousPage) - // Get the next cursor + // Get the next cursor (going backward, use StartCursor per Relay spec) last = int32(1) - nextCursor := txs.PageInfo.EndCursor + nextCursor := txs.PageInfo.StartCursor assert.NotNil(t, nextCursor) txs, err = resolver.Transactions(ctx, parentAccount, nil, nil, nil, nil, &last, nextCursor) require.NoError(t, err) @@ -100,7 +100,7 @@ func TestAccountResolver_Transactions(t *testing.T) { assert.True(t, txs.PageInfo.HasNextPage) assert.True(t, txs.PageInfo.HasPreviousPage) - nextCursor = txs.PageInfo.EndCursor + nextCursor = txs.PageInfo.StartCursor assert.NotNil(t, nextCursor) last = int32(10) txs, err = resolver.Transactions(ctx, parentAccount, nil, nil, nil, nil, &last, nextCursor) @@ -227,8 +227,8 @@ func TestAccountResolver_Operations(t *testing.T) { assert.True(t, ops.PageInfo.HasPreviousPage) assert.False(t, ops.PageInfo.HasNextPage) - // Get the next cursor - nextCursor := ops.PageInfo.EndCursor + // Get the next cursor (going backward, use StartCursor per Relay spec) + nextCursor := ops.PageInfo.StartCursor assert.NotNil(t, nextCursor) ops, err = resolver.Operations(ctx, parentAccount, nil, nil, nil, nil, &last, nextCursor) require.NoError(t, err) @@ -238,7 +238,7 @@ func TestAccountResolver_Operations(t *testing.T) { assert.True(t, ops.PageInfo.HasNextPage) assert.True(t, ops.PageInfo.HasPreviousPage) - nextCursor = ops.PageInfo.EndCursor + nextCursor = ops.PageInfo.StartCursor assert.NotNil(t, nextCursor) last = int32(10) ops, err = resolver.Operations(ctx, parentAccount, nil, nil, nil, nil, &last, nextCursor) @@ -444,8 +444,8 @@ func TestAccountResolver_StateChanges(t *testing.T) { assert.True(t, stateChanges.PageInfo.HasPreviousPage) assert.False(t, stateChanges.PageInfo.HasNextPage) - // Get the next cursor (going backward) - nextCursor := stateChanges.PageInfo.EndCursor + // Get the next cursor (going backward, use StartCursor per Relay spec) + nextCursor := stateChanges.PageInfo.StartCursor assert.NotNil(t, nextCursor) stateChanges, err = resolver.StateChanges(ctx, parentAccount, nil, nil, nil, nil, nil, &last, nextCursor) require.NoError(t, err) @@ -470,7 +470,7 @@ func TestAccountResolver_StateChanges(t *testing.T) { assert.True(t, stateChanges.PageInfo.HasNextPage) assert.True(t, stateChanges.PageInfo.HasPreviousPage) - nextCursor = stateChanges.PageInfo.EndCursor + nextCursor = stateChanges.PageInfo.StartCursor assert.NotNil(t, nextCursor) last = int32(100) stateChanges, err = resolver.StateChanges(ctx, parentAccount, nil, nil, nil, nil, nil, &last, nextCursor) diff --git a/internal/serve/graphql/resolvers/queries_resolvers_test.go b/internal/serve/graphql/resolvers/queries_resolvers_test.go index 2be5511cb..c09ed20eb 100644 --- a/internal/serve/graphql/resolvers/queries_resolvers_test.go +++ b/internal/serve/graphql/resolvers/queries_resolvers_test.go @@ -144,9 +144,9 @@ func TestQueryResolver_Transactions(t *testing.T) { assert.False(t, txs.PageInfo.HasNextPage) assert.True(t, txs.PageInfo.HasPreviousPage) - // Get the next cursor + // Get the next cursor (going backward, use StartCursor per Relay spec) last = int32(1) - nextCursor := txs.PageInfo.EndCursor + nextCursor := txs.PageInfo.StartCursor assert.NotNil(t, nextCursor) txs, err = resolver.Transactions(ctx, nil, nil, &last, nextCursor) require.NoError(t, err) @@ -155,7 +155,7 @@ func TestQueryResolver_Transactions(t *testing.T) { assert.True(t, txs.PageInfo.HasNextPage) assert.True(t, txs.PageInfo.HasPreviousPage) - nextCursor = txs.PageInfo.EndCursor + nextCursor = txs.PageInfo.StartCursor assert.NotNil(t, nextCursor) last = int32(10) txs, err = resolver.Transactions(ctx, nil, nil, &last, nextCursor) @@ -341,9 +341,9 @@ func TestQueryResolver_Operations(t *testing.T) { assert.False(t, ops.PageInfo.HasNextPage) assert.True(t, ops.PageInfo.HasPreviousPage) - // Get the previous page + // Get the previous page (use StartCursor per Relay spec) last = int32(1) - prevCursor := ops.PageInfo.EndCursor + prevCursor := ops.PageInfo.StartCursor assert.NotNil(t, prevCursor) ops, err = resolver.Operations(ctx, nil, nil, &last, prevCursor) require.NoError(t, err) @@ -352,7 +352,7 @@ func TestQueryResolver_Operations(t *testing.T) { assert.True(t, ops.PageInfo.HasNextPage) assert.True(t, ops.PageInfo.HasPreviousPage) - prevCursor = ops.PageInfo.EndCursor + prevCursor = ops.PageInfo.StartCursor assert.NotNil(t, prevCursor) last = int32(10) ops, err = resolver.Operations(ctx, nil, nil, &last, prevCursor) @@ -622,8 +622,8 @@ func TestQueryResolver_StateChanges(t *testing.T) { assert.False(t, stateChanges.PageInfo.HasNextPage) assert.True(t, stateChanges.PageInfo.HasPreviousPage) - // Get the previous page - prevCursor := stateChanges.PageInfo.EndCursor + // Get the previous page (use StartCursor per Relay spec) + prevCursor := stateChanges.PageInfo.StartCursor assert.NotNil(t, prevCursor) stateChanges, err = resolver.StateChanges(ctx, nil, nil, &last, prevCursor) require.NoError(t, err) @@ -643,8 +643,8 @@ func TestQueryResolver_StateChanges(t *testing.T) { assert.True(t, stateChanges.PageInfo.HasNextPage) assert.True(t, stateChanges.PageInfo.HasPreviousPage) - // Get more previous items - prevCursor = stateChanges.PageInfo.EndCursor + // Get more previous items (use StartCursor per Relay spec) + prevCursor = stateChanges.PageInfo.StartCursor assert.NotNil(t, prevCursor) last = int32(20) stateChanges, err = resolver.StateChanges(ctx, nil, nil, &last, prevCursor) diff --git a/internal/serve/graphql/resolvers/transaction_resolvers_test.go b/internal/serve/graphql/resolvers/transaction_resolvers_test.go index 275cbefcb..fce1189aa 100644 --- a/internal/serve/graphql/resolvers/transaction_resolvers_test.go +++ b/internal/serve/graphql/resolvers/transaction_resolvers_test.go @@ -106,8 +106,8 @@ func TestTransactionResolver_Operations(t *testing.T) { assert.False(t, ops.PageInfo.HasNextPage) assert.True(t, ops.PageInfo.HasPreviousPage) - // Get the previous page using cursor - prevCursor := ops.PageInfo.EndCursor + // Get the previous page using cursor (use StartCursor per Relay spec) + prevCursor := ops.PageInfo.StartCursor assert.NotNil(t, prevCursor) ops, err = resolver.Operations(ctx, parentTx, nil, nil, &last, prevCursor) require.NoError(t, err) diff --git a/internal/serve/graphql/resolvers/utils.go b/internal/serve/graphql/resolvers/utils.go index 0cdfada32..2402a3ea7 100644 --- a/internal/serve/graphql/resolvers/utils.go +++ b/internal/serve/graphql/resolvers/utils.go @@ -91,11 +91,7 @@ func NewConnectionWithRelayPagination[T any, C int64 | string](nodes []T, params var startCursor, endCursor *string if len(edges) > 0 { startCursor = &edges[0].Cursor - if params.ForwardPagination { - endCursor = &edges[len(edges)-1].Cursor - } else { - endCursor = &edges[0].Cursor - } + endCursor = &edges[len(edges)-1].Cursor } pageInfo := &generated.PageInfo{ From eb21d1147e7206f056a9fc55e4e2f6718c2cfdd8 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 16 Feb 2026 18:06:03 -0500 Subject: [PATCH 70/77] Set client_sorted = True since we will rebuild_columnstore at the end --- internal/services/ingest_backfill.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/services/ingest_backfill.go b/internal/services/ingest_backfill.go index 4ec697f9b..30c8bc8dc 100644 --- a/internal/services/ingest_backfill.go +++ b/internal/services/ingest_backfill.go @@ -425,6 +425,9 @@ func (m *ingestService) flushBatchBufferWithRetry(ctx context.Context, buffer *i if _, err := dbTx.Exec(ctx, "SET LOCAL timescaledb.enable_direct_compress_copy = on"); err != nil { return fmt.Errorf("enabling direct compress: %w", err) } + if _, err := dbTx.Exec(ctx, "SET LOCAL timescaledb.enable_direct_compress_copy_client_sorted = on"); err != nil { + return fmt.Errorf("enabling direct compress client sorted: %w", err) + } } filteredData, err := m.filterParticipantData(ctx, dbTx, buffer) if err != nil { From 8d2c8ffddc169892ae4cad19305ef04a066b41f5 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 16 Feb 2026 18:11:09 -0500 Subject: [PATCH 71/77] Update ingest_backfill.go --- internal/services/ingest_backfill.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/services/ingest_backfill.go b/internal/services/ingest_backfill.go index 30c8bc8dc..7d16c7d26 100644 --- a/internal/services/ingest_backfill.go +++ b/internal/services/ingest_backfill.go @@ -654,7 +654,7 @@ func (m *ingestService) compressTableChunks(ctx context.Context, table string, s rows, err := m.models.DB.PgxPool().Query(ctx, `SELECT chunk_schema || '.' || chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = $1 AND is_compressed - AND range_start < $2::timestamptz AND range_end > $3::timestamptz + AND range_start <= $2::timestamptz AND range_end >= $3::timestamptz AND range_end < NOW()`, table, endTime, startTime) if err != nil { From 1f7fe71a4eecf05b7c2b9ccba991342e077a6756 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 16 Feb 2026 18:16:07 -0500 Subject: [PATCH 72/77] Update ingest.go --- internal/ingest/ingest.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 44697f533..07d3ecef0 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -147,8 +147,10 @@ func setupDeps(cfg Configs) (services.IngestService, error) { return nil, fmt.Errorf("getting sqlx db: %w", err) } - if err := configureHypertableSettings(ctx, dbConnectionPool, cfg.ChunkInterval, cfg.RetentionPeriod, cfg.OldestLedgerCursorName, cfg.CompressionScheduleInterval, cfg.CompressAfter); err != nil { - return nil, fmt.Errorf("configuring hypertable settings: %w", err) + if cfg.IngestionMode == services.IngestionModeLive { + if err := configureHypertableSettings(ctx, dbConnectionPool, cfg.ChunkInterval, cfg.RetentionPeriod, cfg.OldestLedgerCursorName, cfg.CompressionScheduleInterval, cfg.CompressAfter); err != nil { + return nil, fmt.Errorf("configuring hypertable settings: %w", err) + } } metricsService := metrics.NewMetricsService(sqlxDB) From 01bc06ac8cf17368eb0f426d01d49aa80419d65e Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 17 Feb 2026 15:08:40 -0500 Subject: [PATCH 73/77] Update ingest_backfill.go --- internal/services/ingest_backfill.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/services/ingest_backfill.go b/internal/services/ingest_backfill.go index 7d16c7d26..c409eb4c7 100644 --- a/internal/services/ingest_backfill.go +++ b/internal/services/ingest_backfill.go @@ -425,6 +425,9 @@ func (m *ingestService) flushBatchBufferWithRetry(ctx context.Context, buffer *i if _, err := dbTx.Exec(ctx, "SET LOCAL timescaledb.enable_direct_compress_copy = on"); err != nil { return fmt.Errorf("enabling direct compress: %w", err) } + if _, err := dbTx.Exec(ctx, "SET LOCAL timescaledb.enable_direct_compress_copy_sort_batches = off"); err != nil { + return fmt.Errorf("disabling direct compress sort batches") + } if _, err := dbTx.Exec(ctx, "SET LOCAL timescaledb.enable_direct_compress_copy_client_sorted = on"); err != nil { return fmt.Errorf("enabling direct compress client sorted: %w", err) } From e78ede04edf1a7b665fa2c7976c6629cef4f94d2 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 19 Feb 2026 21:38:39 -0500 Subject: [PATCH 74/77] Add parallel recompression logic --- internal/services/ingest_backfill.go | 270 ++++++++++++++++------ internal/services/ingest_backfill_test.go | 200 ++++++++++++++++ internal/services/ingest_test.go | 10 +- 3 files changed, 408 insertions(+), 72 deletions(-) create mode 100644 internal/services/ingest_backfill_test.go diff --git a/internal/services/ingest_backfill.go b/internal/services/ingest_backfill.go index c409eb4c7..831d660d9 100644 --- a/internal/services/ingest_backfill.go +++ b/internal/services/ingest_backfill.go @@ -4,10 +4,12 @@ import ( "context" "fmt" "maps" + "sync" "time" "github.com/google/uuid" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" "github.com/stellar/go-stellar-sdk/support/log" @@ -172,28 +174,29 @@ func (m *ingestService) startBackfilling(ctx context.Context, startLedger, endLe } backfillBatches := m.splitGapsIntoBatches(gaps) + + // Create progressive recompressor for historical mode. + // Recompresses chunks as contiguous batches complete rather than waiting until the end. + var recompressor *progressiveRecompressor + if mode.isHistorical() { + tables := []string{ + "transactions", "transactions_accounts", "operations", + "operations_accounts", "state_changes", + } + recompressor = newProgressiveRecompressor(ctx, m.models.DB.PgxPool(), tables, len(backfillBatches)) + } + startTime := time.Now() - results := m.processBackfillBatchesParallel(ctx, mode, backfillBatches) + results := m.processBackfillBatchesParallel(ctx, mode, backfillBatches, recompressor) duration := time.Since(startTime) numFailedBatches := analyzeBatchResults(ctx, results) - // Compress backfilled chunks for historical mode (no batches failed) - if mode.isHistorical() && numFailedBatches == 0 { - var minTime, maxTime time.Time - for _, result := range results { - if result.Error == nil { - if minTime.IsZero() || result.StartTime.Before(minTime) { - minTime = result.StartTime - } - if result.EndTime.After(maxTime) { - maxTime = result.EndTime - } - } - } - if !minTime.IsZero() { - m.recompressBackfilledChunks(ctx, minTime, maxTime) - } + // Wait for progressive compression to finish (historical mode only). + // Compression proceeds even if some batches failed — already-compressed + // chunks contain valid data and compress_chunk is idempotent. + if recompressor != nil { + recompressor.Wait() } // Update latest ledger cursor and process catchup data for catchup mode @@ -331,15 +334,18 @@ func (m *ingestService) splitGapsIntoBatches(gaps []data.LedgerRange) []Backfill } // processBackfillBatchesParallel processes backfill batches in parallel using a worker pool. -// For historical mode, direct compress handles compression during COPY; a single recompression -// pass runs after all batches complete (in startBackfilling). -func (m *ingestService) processBackfillBatchesParallel(ctx context.Context, mode BackfillMode, batches []BackfillBatch) []BackfillResult { +// For historical mode, data is inserted uncompressed; the optional progressive compressor +// compresses chunks via compress_chunk() as contiguous batches complete. +func (m *ingestService) processBackfillBatchesParallel(ctx context.Context, mode BackfillMode, batches []BackfillBatch, recompressor *progressiveRecompressor) []BackfillResult { results := make([]BackfillResult, len(batches)) group := m.backfillPool.NewGroupContext(ctx) for i, batch := range batches { group.Submit(func() { results[i] = m.processSingleBatch(ctx, mode, batch, i, len(batches)) + if recompressor != nil && results[i].Error == nil { + recompressor.MarkDone(i, results[i].StartTime, results[i].EndTime) + } }) } @@ -410,8 +416,7 @@ func (m *ingestService) setupBatchBackend(ctx context.Context, batch BackfillBat // flushBatchBufferWithRetry persists buffered data to the database within a transaction. // If updateCursorTo is non-nil, it also updates the oldest cursor atomically. -// If directCompress is true, enables TimescaleDB direct compress for COPY operations. -func (m *ingestService) flushBatchBufferWithRetry(ctx context.Context, buffer *indexer.IndexerBuffer, updateCursorTo *uint32, batchChanges *BatchChanges, directCompress bool) error { +func (m *ingestService) flushBatchBufferWithRetry(ctx context.Context, buffer *indexer.IndexerBuffer, updateCursorTo *uint32, batchChanges *BatchChanges) error { var lastErr error for attempt := 0; attempt < maxIngestProcessedDataRetries; attempt++ { select { @@ -421,17 +426,6 @@ func (m *ingestService) flushBatchBufferWithRetry(ctx context.Context, buffer *i } err := db.RunInPgxTransaction(ctx, m.models.DB, func(dbTx pgx.Tx) error { - if directCompress { - if _, err := dbTx.Exec(ctx, "SET LOCAL timescaledb.enable_direct_compress_copy = on"); err != nil { - return fmt.Errorf("enabling direct compress: %w", err) - } - if _, err := dbTx.Exec(ctx, "SET LOCAL timescaledb.enable_direct_compress_copy_sort_batches = off"); err != nil { - return fmt.Errorf("disabling direct compress sort batches") - } - if _, err := dbTx.Exec(ctx, "SET LOCAL timescaledb.enable_direct_compress_copy_client_sorted = on"); err != nil { - return fmt.Errorf("enabling direct compress client sorted: %w", err) - } - } filteredData, err := m.filterParticipantData(ctx, dbTx, buffer) if err != nil { return fmt.Errorf("filtering participant data: %w", err) @@ -540,7 +534,7 @@ func (m *ingestService) processLedgersInBatch( // Flush buffer periodically to control memory usage (intermediate flushes, no cursor update) if ledgersInBuffer >= m.backfillDBInsertBatchSize { - if err := m.flushBatchBufferWithRetry(ctx, batchBuffer, nil, batchChanges, mode.isHistorical()); err != nil { + if err := m.flushBatchBufferWithRetry(ctx, batchBuffer, nil, batchChanges); err != nil { return ledgersProcessed, batchChanges, startTime, endTime, err } batchBuffer.Clear() @@ -554,7 +548,7 @@ func (m *ingestService) processLedgersInBatch( if mode.isHistorical() { cursorUpdate = &batch.StartLedger } - if err := m.flushBatchBufferWithRetry(ctx, batchBuffer, cursorUpdate, batchChanges, mode.isHistorical()); err != nil { + if err := m.flushBatchBufferWithRetry(ctx, batchBuffer, cursorUpdate, batchChanges); err != nil { return ledgersProcessed, batchChanges, startTime, endTime, err } } else if mode.isHistorical() { @@ -629,39 +623,116 @@ func (m *ingestService) processBatchChanges( return nil } -// recompressBackfilledChunks recompresses already-compressed chunks overlapping the backfill range. -// Direct compress produces compressed chunks during COPY; recompression optimizes compression ratios. -// Tables are compressed sequentially to avoid CPU spikes; chunks within each table also stay sequential to avoid OOM. -// Skips chunks where range_end >= NOW() to avoid compressing active live ingestion chunks. -func (m *ingestService) recompressBackfilledChunks(ctx context.Context, startTime, endTime time.Time) { - tables := []string{"transactions", "transactions_accounts", "operations", "operations_accounts", "state_changes"} +// progressiveRecompressor compresses uncompressed TimescaleDB chunks as they become safe during backfill. +// Tracks batch completion via a watermark to determine when chunks are fully written. +type progressiveRecompressor struct { + pool *pgxpool.Pool + tables []string + ctx context.Context + + mu sync.Mutex + completed []bool + endTimes []time.Time + watermarkIdx int // index of highest contiguous completed batch (-1 = none) + globalStart time.Time // lower bound for chunk queries (batch 0's StartTime) + globalEnd time.Time // upper bound for verification (max EndTime across completed batches) + + triggerCh chan time.Time // safeEnd for recompression window + done chan struct{} +} + +// newProgressiveRecompressor creates a compressor that progressively compresses uncompressed chunks +// as contiguous batches complete. Starts a background goroutine for compression work. +func newProgressiveRecompressor(ctx context.Context, pool *pgxpool.Pool, tables []string, totalBatches int) *progressiveRecompressor { + r := &progressiveRecompressor{ + pool: pool, + tables: tables, + ctx: ctx, + completed: make([]bool, totalBatches), + endTimes: make([]time.Time, totalBatches), + watermarkIdx: -1, + triggerCh: make(chan time.Time, totalBatches), + done: make(chan struct{}), + } + go r.runCompression() + return r +} + +// MarkDone records a batch as complete and advances the watermark if possible. +// If the watermark advances, triggers recompression of chunks in the safe window. +func (r *progressiveRecompressor) MarkDone(batchIdx int, startTime, endTime time.Time) { + r.mu.Lock() + defer r.mu.Unlock() + + r.completed[batchIdx] = true + r.endTimes[batchIdx] = endTime - tableCounts := make([]int, len(tables)) + // Record global start from batch 0 (earliest time boundary for queries) + if batchIdx == 0 { + r.globalStart = startTime + } - for i, table := range tables { - tableCounts[i] = m.compressTableChunks(ctx, table, startTime, endTime) + // Track the maximum EndTime across all completed batches for verification scope + if endTime.After(r.globalEnd) { + r.globalEnd = endTime } + // Advance watermark past contiguous completed batches + oldWatermark := r.watermarkIdx + for r.watermarkIdx+1 < len(r.completed) && r.completed[r.watermarkIdx+1] { + r.watermarkIdx++ + } + + // Trigger recompression if watermark advanced + if r.watermarkIdx > oldWatermark { + r.triggerCh <- r.endTimes[r.watermarkIdx] + } +} + +// Wait closes the trigger channel and waits for background compression to finish. +func (r *progressiveRecompressor) Wait() { + close(r.triggerCh) + <-r.done +} + +// runCompression processes compression triggers in the background. +// For each safe window, queries and compresses uncompressed chunks per table. +// After all windows, runs verification to catch any missed chunks (including +// trailing boundary chunks) scoped to [globalStart, globalEnd] to avoid +// touching chunks compressed by TimescaleDB policy from live ingestion. +func (r *progressiveRecompressor) runCompression() { + defer close(r.done) + totalCompressed := 0 - for _, count := range tableCounts { - totalCompressed += count + for safeEnd := range r.triggerCh { + for _, table := range r.tables { + count := r.compressTableChunks(table, safeEnd) + totalCompressed += count + } } - log.Ctx(ctx).Infof("Recompressed %d total chunks for time range [%s - %s]", - totalCompressed, startTime.Format(time.RFC3339), endTime.Format(time.RFC3339)) + + // Final verification: overlap query catches trailing boundary chunk + any missed chunks. + // Scoped to [globalStart, globalEnd] — the actual backfill range — to avoid touching + // chunks compressed by TimescaleDB policy from live ingestion. + totalCompressed += r.verifyAllChunksCompressed() + + log.Ctx(r.ctx).Infof("Progressive compression complete: %d total chunks compressed", totalCompressed) } -// compressTableChunks recompresses already-compressed chunks for a single hypertable. -// Direct compress produces compressed chunks during COPY; this pass optimizes compression ratios. -// Chunks are recompressed sequentially within the table to avoid OOM errors. -func (m *ingestService) compressTableChunks(ctx context.Context, table string, startTime, endTime time.Time) int { - rows, err := m.models.DB.PgxPool().Query(ctx, - `SELECT chunk_schema || '.' || chunk_name FROM timescaledb_information.chunks - WHERE hypertable_name = $1 AND is_compressed - AND range_start <= $2::timestamptz AND range_end >= $3::timestamptz - AND range_end < NOW()`, - table, endTime, startTime) +// compressTableChunks compresses uncompressed chunks for a single table within the safe window. +// Queries chunks where range_end falls within (globalStart, safeEnd] to catch the leading +// boundary chunk that overlaps globalStart. +func (r *progressiveRecompressor) compressTableChunks(table string, safeEnd time.Time) int { + rows, err := r.pool.Query(r.ctx, + `SELECT c.chunk_schema || '.' || c.chunk_name + FROM timescaledb_information.chunks c + WHERE c.hypertable_name = $1 + AND NOT c.is_compressed + AND c.range_end <= $2::timestamptz + AND c.range_end > $3::timestamptz`, + table, safeEnd, r.globalStart) if err != nil { - log.Ctx(ctx).Warnf("Failed to get chunks for %s: %v", table, err) + log.Ctx(r.ctx).Warnf("Failed to get chunks for %s: %v", table, err) return 0 } @@ -676,23 +747,88 @@ func (m *ingestService) compressTableChunks(ctx context.Context, table string, s rows.Close() compressed := 0 - for i, chunk := range chunks { + for _, chunk := range chunks { select { - case <-ctx.Done(): - log.Ctx(ctx).Warnf("Recompression cancelled for %s after %d chunks", table, compressed) + case <-r.ctx.Done(): + log.Ctx(r.ctx).Warnf("Compression cancelled for %s after %d chunks", table, compressed) return compressed default: } - _, err := m.models.DB.PgxPool().Exec(ctx, - `CALL _timescaledb_functions.rebuild_columnstore($1::regclass)`, chunk) + _, err := r.pool.Exec(r.ctx, `SELECT compress_chunk($1::regclass)`, chunk) if err != nil { - log.Ctx(ctx).Warnf("Failed to recompress chunk %s: %v", chunk, err) + log.Ctx(r.ctx).Warnf("Failed to compress chunk %s: %v", chunk, err) continue } compressed++ - log.Ctx(ctx).Debugf("Recompressed chunk %d/%d for %s: %s", i+1, len(chunks), table, chunk) + log.Ctx(r.ctx).Debugf("Compressed chunk for %s: %s", table, chunk) } - log.Ctx(ctx).Infof("Recompressed %d chunks for table %s", len(chunks), table) + + if compressed > 0 { + log.Ctx(r.ctx).Infof("Compressed %d chunks for table %s (window end: %s)", + compressed, table, safeEnd.Format(time.RFC3339)) + } + return compressed } + +// verifyAllChunksCompressed catches any chunks missed by progressive windows. +// Uses overlap logic (range_end > globalStart AND range_start < globalEnd) to find +// uncompressed chunks in the backfill range, including trailing boundary chunks. +func (r *progressiveRecompressor) verifyAllChunksCompressed() int { + r.mu.Lock() + globalStart := r.globalStart + globalEnd := r.globalEnd + r.mu.Unlock() + + if globalStart.IsZero() || globalEnd.IsZero() { + return 0 + } + + totalMissed := 0 + for _, table := range r.tables { + rows, err := r.pool.Query(r.ctx, + `SELECT c.chunk_schema || '.' || c.chunk_name + FROM timescaledb_information.chunks c + WHERE c.hypertable_name = $1 + AND NOT c.is_compressed + AND c.range_end > $2::timestamptz + AND c.range_start < $3::timestamptz`, + table, globalStart, globalEnd) + if err != nil { + log.Ctx(r.ctx).Warnf("Verification query failed for %s: %v", table, err) + continue + } + + var chunks []string + for rows.Next() { + var chunk string + if err := rows.Scan(&chunk); err != nil { + continue + } + chunks = append(chunks, chunk) + } + rows.Close() + + for _, chunk := range chunks { + select { + case <-r.ctx.Done(): + return totalMissed + default: + } + + log.Ctx(r.ctx).Warnf("Verification found missed chunk %s for table %s", chunk, table) + _, err := r.pool.Exec(r.ctx, `SELECT compress_chunk($1::regclass)`, chunk) + if err != nil { + log.Ctx(r.ctx).Warnf("Failed to compress missed chunk %s: %v", chunk, err) + continue + } + totalMissed++ + } + } + + if totalMissed > 0 { + log.Ctx(r.ctx).Infof("Verification compressed %d missed chunks", totalMissed) + } + return totalMissed +} diff --git a/internal/services/ingest_backfill_test.go b/internal/services/ingest_backfill_test.go new file mode 100644 index 000000000..e62f4d198 --- /dev/null +++ b/internal/services/ingest_backfill_test.go @@ -0,0 +1,200 @@ +// Tests for progressive recompression watermark logic during historical backfill. +package services + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newTestRecompressor creates a progressiveRecompressor for testing watermark logic. +// No background goroutine is started; triggerCh is buffered for direct inspection. +func newTestRecompressor(totalBatches int) *progressiveRecompressor { + return &progressiveRecompressor{ + completed: make([]bool, totalBatches), + endTimes: make([]time.Time, totalBatches), + watermarkIdx: -1, + triggerCh: make(chan time.Time, totalBatches), + } +} + +// drainWindows reads all available safeEnd values from triggerCh without blocking. +func drainWindows(r *progressiveRecompressor) []time.Time { + var windows []time.Time + for { + select { + case w := <-r.triggerCh: + windows = append(windows, w) + default: + return windows + } + } +} + +func Test_progressiveRecompressor_MarkDone_sequential(t *testing.T) { + r := newTestRecompressor(5) + base := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + + // Complete batches 0-4 in order + for i := 0; i < 5; i++ { + startTime := base.Add(time.Duration(i) * time.Hour) + endTime := base.Add(time.Duration(i+1) * time.Hour) + r.MarkDone(i, startTime, endTime) + } + + windows := drainWindows(r) + // Each call advances the watermark — expect 5 windows + require.Len(t, windows, 5) + + // First window: safeEnd = batch 0 endTime + assert.Equal(t, base.Add(1*time.Hour), windows[0]) + + // Second window: safeEnd = batch 1 endTime + assert.Equal(t, base.Add(2*time.Hour), windows[1]) + + // Last window: safeEnd = batch 4 endTime + assert.Equal(t, base.Add(5*time.Hour), windows[4]) + + // Verify globalStart set from batch 0 + assert.Equal(t, base, r.globalStart) +} + +func Test_progressiveRecompressor_MarkDone_outOfOrder(t *testing.T) { + r := newTestRecompressor(5) + base := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + endTime := func(i int) time.Time { return base.Add(time.Duration(i+1) * time.Hour) } + startTime := func(i int) time.Time { return base.Add(time.Duration(i) * time.Hour) } + + // Complete batch 2 first — watermark can't advance (batch 0 not done) + r.MarkDone(2, startTime(2), endTime(2)) + windows := drainWindows(r) + assert.Empty(t, windows) + assert.Equal(t, -1, r.watermarkIdx) + + // Complete batch 4 — still no advancement + r.MarkDone(4, startTime(4), endTime(4)) + windows = drainWindows(r) + assert.Empty(t, windows) + + // Complete batch 0 — watermark advances to 0 only (batch 1 missing) + r.MarkDone(0, startTime(0), endTime(0)) + windows = drainWindows(r) + require.Len(t, windows, 1) + assert.Equal(t, endTime(0), windows[0]) + assert.Equal(t, 0, r.watermarkIdx) + + // Complete batch 1 — watermark jumps from 0 to 2 (batch 2 was already done) + r.MarkDone(1, startTime(1), endTime(1)) + windows = drainWindows(r) + require.Len(t, windows, 1) + assert.Equal(t, endTime(2), windows[0]) // safeEnd = batch 2's endTime + assert.Equal(t, 2, r.watermarkIdx) + + // Complete batch 3 — watermark jumps from 2 to 4 (batch 4 was already done) + r.MarkDone(3, startTime(3), endTime(3)) + windows = drainWindows(r) + require.Len(t, windows, 1) + assert.Equal(t, endTime(4), windows[0]) + assert.Equal(t, 4, r.watermarkIdx) +} + +func Test_progressiveRecompressor_MarkDone_failedBatchBlocksWatermark(t *testing.T) { + r := newTestRecompressor(5) + base := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + endTime := func(i int) time.Time { return base.Add(time.Duration(i+1) * time.Hour) } + startTime := func(i int) time.Time { return base.Add(time.Duration(i) * time.Hour) } + + // Complete batches 0 and 1 + r.MarkDone(0, startTime(0), endTime(0)) + r.MarkDone(1, startTime(1), endTime(1)) + _ = drainWindows(r) // consume windows + + // Batch 2 fails (never call MarkDone for it) + + // Complete batches 3 and 4 + r.MarkDone(3, startTime(3), endTime(3)) + r.MarkDone(4, startTime(4), endTime(4)) + + // No new windows — watermark stuck at 1 + windows := drainWindows(r) + assert.Empty(t, windows) + assert.Equal(t, 1, r.watermarkIdx) +} + +func Test_progressiveRecompressor_MarkDone_singleBatch(t *testing.T) { + r := newTestRecompressor(1) + start := time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2025, 6, 1, 1, 0, 0, 0, time.UTC) + + r.MarkDone(0, start, end) + + windows := drainWindows(r) + require.Len(t, windows, 1) + assert.Equal(t, end, windows[0]) + assert.Equal(t, 0, r.watermarkIdx) + assert.Equal(t, start, r.globalStart) +} + +func Test_progressiveRecompressor_MarkDone_globalStartSetFromBatchZero(t *testing.T) { + r := newTestRecompressor(3) + base := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + + // Complete batch 1 first — globalStart should NOT be set + r.MarkDone(1, base.Add(1*time.Hour), base.Add(2*time.Hour)) + assert.True(t, r.globalStart.IsZero()) + + // Complete batch 0 — globalStart should be set to batch 0's startTime + r.MarkDone(0, base, base.Add(1*time.Hour)) + assert.Equal(t, base, r.globalStart) + + // Complete batch 2 — globalStart unchanged + r.MarkDone(2, base.Add(2*time.Hour), base.Add(3*time.Hour)) + assert.Equal(t, base, r.globalStart) +} + +func Test_progressiveRecompressor_MarkDone_allSimultaneous(t *testing.T) { + r := newTestRecompressor(4) + base := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + endTime := func(i int) time.Time { return base.Add(time.Duration(i+1) * time.Hour) } + startTime := func(i int) time.Time { return base.Add(time.Duration(i) * time.Hour) } + + // Complete all batches in reverse order except batch 0 + r.MarkDone(3, startTime(3), endTime(3)) + r.MarkDone(2, startTime(2), endTime(2)) + r.MarkDone(1, startTime(1), endTime(1)) + windows := drainWindows(r) + assert.Empty(t, windows) // Nothing yet — batch 0 missing + + // Complete batch 0 — watermark jumps from -1 to 3 in one step + r.MarkDone(0, startTime(0), endTime(0)) + windows = drainWindows(r) + require.Len(t, windows, 1) + assert.Equal(t, endTime(3), windows[0]) // safeEnd = last batch + assert.Equal(t, 3, r.watermarkIdx) +} + +func Test_progressiveRecompressor_MarkDone_globalEndTracked(t *testing.T) { + r := newTestRecompressor(4) + base := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + + // globalEnd starts as zero + assert.True(t, r.globalEnd.IsZero()) + + // Complete batch 2 — globalEnd updates to batch 2's endTime + r.MarkDone(2, base.Add(2*time.Hour), base.Add(3*time.Hour)) + assert.Equal(t, base.Add(3*time.Hour), r.globalEnd) + + // Complete batch 0 — globalEnd should NOT decrease (batch 0 endTime < batch 2 endTime) + r.MarkDone(0, base, base.Add(1*time.Hour)) + assert.Equal(t, base.Add(3*time.Hour), r.globalEnd) + + // Complete batch 3 — globalEnd updates to batch 3's endTime (the new max) + r.MarkDone(3, base.Add(3*time.Hour), base.Add(4*time.Hour)) + assert.Equal(t, base.Add(4*time.Hour), r.globalEnd) + + // Complete batch 1 — globalEnd should NOT decrease + r.MarkDone(1, base.Add(1*time.Hour), base.Add(2*time.Hour)) + assert.Equal(t, base.Add(4*time.Hour), r.globalEnd) +} diff --git a/internal/services/ingest_test.go b/internal/services/ingest_test.go index b1571ae47..e4d0f918b 100644 --- a/internal/services/ingest_test.go +++ b/internal/services/ingest_test.go @@ -1317,7 +1317,7 @@ func Test_ingestService_flushBatchBufferWithRetry(t *testing.T) { buffer := tc.setupBuffer() // Call flushBatchBuffer - err = svc.flushBatchBufferWithRetry(ctx, buffer, tc.updateCursorTo, nil, false) + err = svc.flushBatchBufferWithRetry(ctx, buffer, tc.updateCursorTo, nil) require.NoError(t, err) // Verify the cursor value @@ -1688,7 +1688,7 @@ func Test_ingestService_processBackfillBatchesParallel_PartialFailure(t *testing }) require.NoError(t, svcErr) - results := svc.processBackfillBatchesParallel(ctx, BackfillModeHistorical, tc.batches) + results := svc.processBackfillBatchesParallel(ctx, BackfillModeHistorical, tc.batches, nil) // Verify results require.Len(t, results, len(tc.batches)) @@ -1946,7 +1946,7 @@ func Test_ingestService_processBackfillBatches_PartialFailure_OnlySuccessfulBatc require.NoError(t, svcErr) // Process both batches in parallel - results := svc.processBackfillBatchesParallel(ctx, BackfillModeHistorical, batches) + results := svc.processBackfillBatchesParallel(ctx, BackfillModeHistorical, batches, nil) // Verify we got results for both batches require.Len(t, results, 2) @@ -2820,7 +2820,7 @@ func Test_ingestService_flushBatchBuffer_batchChanges(t *testing.T) { buffer := tc.setupBuffer() - err = svc.flushBatchBufferWithRetry(ctx, buffer, nil, tc.batchChanges, false) + err = svc.flushBatchBufferWithRetry(ctx, buffer, nil, tc.batchChanges) require.NoError(t, err) // Verify collected token changes match expected values @@ -3182,7 +3182,7 @@ func Test_ingestService_processBackfillBatchesParallel_BothModes(t *testing.T) { {StartLedger: 101, EndLedger: 101}, } - results := svc.processBackfillBatchesParallel(ctx, tc.mode, batches) + results := svc.processBackfillBatchesParallel(ctx, tc.mode, batches, nil) // All batches should succeed require.Len(t, results, 2) From c801128ba5e6e287918c65088f95e23d5ac525ab Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Sat, 21 Feb 2026 16:40:23 -0500 Subject: [PATCH 75/77] remove account id validation --- .../integrationtests/data_validation_test.go | 144 ++++++++---------- pkg/wbclient/types/statechange.go | 7 - 2 files changed, 60 insertions(+), 91 deletions(-) diff --git a/internal/integrationtests/data_validation_test.go b/internal/integrationtests/data_validation_test.go index 4e3a13aaa..cd1089d5a 100644 --- a/internal/integrationtests/data_validation_test.go +++ b/internal/integrationtests/data_validation_test.go @@ -137,30 +137,27 @@ func validateStateChangeBase(suite *DataValidationTestSuite, sc types.StateChang } // validateBalanceChange validates a balance state change -func validateBalanceChange(suite *DataValidationTestSuite, bc *types.StandardBalanceChange, expectedTokenID, expectedAmount, expectedAccount string, expectedReason types.StateChangeReason) { +func validateBalanceChange(suite *DataValidationTestSuite, bc *types.StandardBalanceChange, expectedTokenID, expectedAmount string, expectedReason types.StateChangeReason) { suite.Require().NotNil(bc, "balance change should not be nil") suite.Require().Equal(types.StateChangeCategoryBalance, bc.GetType(), "should be BALANCE type") suite.Require().Equal(expectedReason, bc.GetReason(), "reason mismatch") suite.Require().Equal(expectedTokenID, bc.TokenID, "token ID mismatch") suite.Require().Equal(expectedAmount, bc.Amount, "amount mismatch") - suite.Require().Equal(expectedAccount, bc.GetAccountID(), "account ID mismatch") } // validateAccountChange validates an account state change -func validateAccountChange(suite *DataValidationTestSuite, ac *types.AccountChange, expectedAccount, expectedFunderAddress string, expectedReason types.StateChangeReason) { +func validateAccountChange(suite *DataValidationTestSuite, ac *types.AccountChange, expectedFunderAddress string, expectedReason types.StateChangeReason) { suite.Require().NotNil(ac, "account change should not be nil") suite.Require().Equal(types.StateChangeCategoryAccount, ac.GetType(), "should be ACCOUNT type") suite.Require().Equal(expectedReason, ac.GetReason(), "reason mismatch") - suite.Require().Equal(expectedAccount, ac.GetAccountID(), "account ID mismatch") suite.Require().Equal(expectedFunderAddress, *ac.FunderAddress, "funder address mismatch") } // validateSignerChange validates a signer state change -func validateSignerChange(suite *DataValidationTestSuite, sc *types.SignerChange, expectedAccount string, expectedSignerAddress string, expectedSignerWeights int32, expectedReason types.StateChangeReason) { +func validateSignerChange(suite *DataValidationTestSuite, sc *types.SignerChange, expectedSignerAddress string, expectedSignerWeights int32, expectedReason types.StateChangeReason) { suite.Require().NotNil(sc, "signer change should not be nil") suite.Require().Equal(types.StateChangeCategorySigner, sc.GetType(), "should be SIGNER type") suite.Require().Equal(expectedReason, sc.GetReason(), "reason mismatch") - suite.Require().Equal(expectedAccount, sc.GetAccountID(), "account ID mismatch") suite.Require().NotNil(sc.SignerAddress, "signer address should not be nil") suite.Require().Equal(expectedSignerAddress, *sc.SignerAddress, "signer address mismatch") @@ -175,12 +172,11 @@ func validateSignerChange(suite *DataValidationTestSuite, sc *types.SignerChange } // validateMetadataChange validates a metadata state change -func validateMetadataChange(suite *DataValidationTestSuite, mc *types.MetadataChange, expectedAccount string, expectedReason types.StateChangeReason, expectedKey, expectedInnerKey, expectedValue string) { +func validateMetadataChange(suite *DataValidationTestSuite, mc *types.MetadataChange, expectedReason types.StateChangeReason, expectedKey, expectedInnerKey, expectedValue string) { suite.Require().NotNil(mc, "metadata change should not be nil") suite.Require().Equal(types.StateChangeCategoryMetadata, mc.GetType(), "should be METADATA type") suite.Require().Equal(expectedReason, mc.GetReason(), "reason mismatch") suite.Require().NotEmpty(mc.KeyValue, "key value should not be empty") - suite.Require().Equal(expectedAccount, mc.GetAccountID(), "account ID mismatch") // Decode the key value var result map[string]map[string]string @@ -194,23 +190,21 @@ func validateMetadataChange(suite *DataValidationTestSuite, mc *types.MetadataCh } // validateReservesChange validates a reserves state change -func validateReservesSponsorshipChangeForSponsoredAccount(suite *DataValidationTestSuite, rc *types.ReservesChange, expectedAccount string, +func validateReservesSponsorshipChangeForSponsoredAccount(suite *DataValidationTestSuite, rc *types.ReservesChange, expectedReason types.StateChangeReason, expectedSponsorAddress string, ) { suite.Require().NotNil(rc, "reserves sponsorship change should not be nil") suite.Require().Equal(types.StateChangeCategoryReserves, rc.GetType(), "should be RESERVES type") suite.Require().Equal(expectedReason, rc.GetReason(), "reason mismatch") - suite.Require().Equal(expectedAccount, rc.GetAccountID(), "account ID mismatch") suite.Require().Equal(expectedSponsorAddress, *rc.SponsorAddress, "sponsor address mismatch") } -func validateReservesSponsorshipChangeForSponsoringAccount(suite *DataValidationTestSuite, rc *types.ReservesChange, expectedAccount string, +func validateReservesSponsorshipChangeForSponsoringAccount(suite *DataValidationTestSuite, rc *types.ReservesChange, expectedReason types.StateChangeReason, expectedSponsoredAddress string, ) { suite.Require().NotNil(rc, "reserves sponsorship change should not be nil") suite.Require().Equal(types.StateChangeCategoryReserves, rc.GetType(), "should be RESERVES type") suite.Require().Equal(expectedReason, rc.GetReason(), "reason mismatch") - suite.Require().Equal(expectedAccount, rc.GetAccountID(), "account ID mismatch") if expectedSponsoredAddress != "" { suite.Require().Equal(expectedSponsoredAddress, *rc.SponsoredAddress, "sponsored address mismatch") } @@ -238,11 +232,10 @@ func sumAmounts(suite *DataValidationTestSuite, sc *types.StateChangeConnection, } // validateTrustlineChangeDetailed validates a trustline state change with detailed checks -func validateTrustlineChange(suite *DataValidationTestSuite, tc *types.TrustlineChange, expectedAccount string, expectedTokenID string, expectedLiquidityPoolID string, expectedReason types.StateChangeReason) { +func validateTrustlineChange(suite *DataValidationTestSuite, tc *types.TrustlineChange, expectedTokenID string, expectedLiquidityPoolID string, expectedReason types.StateChangeReason) { suite.Require().NotNil(tc, "trustline change should not be nil") suite.Require().Equal(types.StateChangeCategoryTrustline, tc.GetType(), "should be TRUSTLINE type") suite.Require().Equal(expectedReason, tc.GetReason(), "reason mismatch") - suite.Require().Equal(expectedAccount, tc.GetAccountID(), "account ID mismatch") if expectedTokenID != "" { suite.Require().NotNil(tc.TokenID, "token ID should not be nil") suite.Require().Equal(expectedTokenID, *tc.TokenID, "token ID mismatch") @@ -258,13 +251,12 @@ func validateTrustlineChange(suite *DataValidationTestSuite, tc *types.Trustline } // validateBalanceAuthorizationChange validates a balance authorization state change -func validateBalanceAuthorizationChange(suite *DataValidationTestSuite, bac *types.BalanceAuthorizationChange, expectedAccount string, +func validateBalanceAuthorizationChange(suite *DataValidationTestSuite, bac *types.BalanceAuthorizationChange, expectedReason types.StateChangeReason, expectedFlags []string, expectedTokenID string, ) { suite.Require().NotNil(bac, "balance authorization change should not be nil") suite.Require().Equal(types.StateChangeCategoryBalanceAuthorization, bac.GetType(), "should be BALANCE_AUTHORIZATION type") suite.Require().Equal(expectedReason, bac.GetReason(), "reason mismatch") - suite.Require().Equal(expectedAccount, bac.GetAccountID(), "account ID mismatch") suite.Require().Equal(len(expectedFlags), len(bac.Flags), "flags count mismatch") for _, expectedFlag := range expectedFlags { suite.Require().Contains(bac.Flags, expectedFlag, "expected flag not found: %s", expectedFlag) @@ -275,11 +267,10 @@ func validateBalanceAuthorizationChange(suite *DataValidationTestSuite, bac *typ } // validateFlagsChange validates a flags state change -func validateFlagsChange(suite *DataValidationTestSuite, fc *types.FlagsChange, expectedAccount string, expectedReason types.StateChangeReason, expectedFlags []string) { +func validateFlagsChange(suite *DataValidationTestSuite, fc *types.FlagsChange, expectedReason types.StateChangeReason, expectedFlags []string) { suite.Require().NotNil(fc, "flags change should not be nil") suite.Require().Equal(types.StateChangeCategoryFlags, fc.GetType(), "should be FLAGS type") suite.Require().Equal(expectedReason, fc.GetReason(), "reason mismatch") - suite.Require().Equal(expectedAccount, fc.GetAccountID(), "account ID mismatch") suite.Require().Equal(len(expectedFlags), len(fc.Flags), "flags count mismatch") for _, expectedFlag := range expectedFlags { suite.Require().Contains(fc.Flags, expectedFlag, "expected flag not found: %s", expectedFlag) @@ -355,12 +346,12 @@ func (suite *DataValidationTestSuite) validatePaymentStateChanges(ctx context.Co // 1 DEBIT change for primary account suite.Require().Len(primaryStateChanges.Edges, 1, "should have exactly 1 state change for primary account") sc := primaryStateChanges.Edges[0].Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, sc, xlmContractAddress, "100000000", primaryAccount, types.StateChangeReasonDebit) + validateBalanceChange(suite, sc, xlmContractAddress, "100000000", types.StateChangeReasonDebit) // 1 CREDIT change for secondary account suite.Require().Len(secondaryStateChanges.Edges, 1, "should have exactly 1 state change for secondary account") sc = secondaryStateChanges.Edges[0].Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, sc, xlmContractAddress, "100000000", secondaryAccount, types.StateChangeReasonCredit) + validateBalanceChange(suite, sc, xlmContractAddress, "100000000", types.StateChangeReasonCredit) } func (suite *DataValidationTestSuite) TestSponsoredAccountCreationDataValidation() { @@ -461,37 +452,37 @@ func (suite *DataValidationTestSuite) validateSponsoredAccountCreationStateChang // 1 BALANCE/DEBIT change for primary account (sending starting balance) suite.Require().Len(primaryBalanceChanges.Edges, 1, "should have exactly 1 BALANCE/DEBIT balance change for primary account") balanceChange := primaryBalanceChanges.Edges[0].Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, balanceChange, xlmContractAddress, "50000000", primaryAccount, types.StateChangeReasonDebit) + validateBalanceChange(suite, balanceChange, xlmContractAddress, "50000000", types.StateChangeReasonDebit) // 1 BALANCE/CREDIT change for sponsored account (receiving starting balance) suite.Require().Len(sponsoredBalanceChanges.Edges, 1, "should have exactly 1 BALANCE/CREDIT balance change for sponsored account") balanceChange = sponsoredBalanceChanges.Edges[0].Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, balanceChange, xlmContractAddress, "50000000", sponsoredNewAccount, types.StateChangeReasonCredit) + validateBalanceChange(suite, balanceChange, xlmContractAddress, "50000000", types.StateChangeReasonCredit) // 1 ACCOUNT/CREATE account change for sponsored account suite.Require().Len(sponsoredAccountChanges.Edges, 1, "should have exactly 1 ACCOUNT/CREATE account change") accountChange := sponsoredAccountChanges.Edges[0].Node.(*types.AccountChange) - validateAccountChange(suite, accountChange, sponsoredNewAccount, primaryAccount, types.StateChangeReasonCreate) + validateAccountChange(suite, accountChange, primaryAccount, types.StateChangeReasonCreate) // 1 METADATA/DATA_ENTRY metadata change for primary account suite.Require().Len(primaryMetadataChanges.Edges, 1, "should have exactly 1 METADATA/DATA_ENTRY metadata change for primary account") metadataChange := primaryMetadataChanges.Edges[0].Node.(*types.MetadataChange) - validateMetadataChange(suite, metadataChange, primaryAccount, types.StateChangeReasonDataEntry, "foo", "new", "bar") + validateMetadataChange(suite, metadataChange, types.StateChangeReasonDataEntry, "foo", "new", "bar") // 1 RESERVES/SPONSOR change for sponsored account - sponsorship begin suite.Require().Len(sponsoredReservesChanges.Edges, 1, "should have exactly 1 RESERVES/SPONSOR reserves change for sponsored account") reserveChange := sponsoredReservesChanges.Edges[0].Node.(*types.ReservesChange) - validateReservesSponsorshipChangeForSponsoredAccount(suite, reserveChange, sponsoredNewAccount, types.StateChangeReasonSponsor, primaryAccount) + validateReservesSponsorshipChangeForSponsoredAccount(suite, reserveChange, types.StateChangeReasonSponsor, primaryAccount) // 1 RESERVES/SPONSOR change for sponsoring account - sponsorship begin suite.Require().Len(primaryReservesChanges.Edges, 1, "should have exactly 1 RESERVES/SPONSOR reserves change for sponsoring account") reserveChange = primaryReservesChanges.Edges[0].Node.(*types.ReservesChange) - validateReservesSponsorshipChangeForSponsoringAccount(suite, reserveChange, primaryAccount, types.StateChangeReasonSponsor, sponsoredNewAccount) + validateReservesSponsorshipChangeForSponsoringAccount(suite, reserveChange, types.StateChangeReasonSponsor, sponsoredNewAccount) // 1 SIGNER/ADD change for sponsored account with default signer weight = 1 suite.Require().Len(sponsoredSignerChanges.Edges, 1, "should have exactly 1 SIGNER/CREATE signer change for sponsored account") signerChange := sponsoredSignerChanges.Edges[0].Node.(*types.SignerChange) - validateSignerChange(suite, signerChange, sponsoredNewAccount, sponsoredNewAccount, 1, types.StateChangeReasonAdd) + validateSignerChange(suite, signerChange, sponsoredNewAccount, 1, types.StateChangeReasonAdd) } func (suite *DataValidationTestSuite) TestCustomAssetsOpsDataValidation() { @@ -629,10 +620,10 @@ func (suite *DataValidationTestSuite) validateCustomAssetsStateChanges(ctx conte tc := edge.Node.(*types.TrustlineChange) if tc.GetReason() == types.StateChangeReasonAdd { - validateTrustlineChange(suite, tc, secondaryAccount, test2ContractAddress, "", types.StateChangeReasonAdd) + validateTrustlineChange(suite, tc, test2ContractAddress, "", types.StateChangeReasonAdd) foundAdd = true } else if tc.GetReason() == types.StateChangeReasonRemove { - validateTrustlineChange(suite, tc, secondaryAccount, test2ContractAddress, "", types.StateChangeReasonRemove) + validateTrustlineChange(suite, tc, test2ContractAddress, "", types.StateChangeReasonRemove) foundRemove = true } } @@ -642,14 +633,13 @@ func (suite *DataValidationTestSuite) validateCustomAssetsStateChanges(ctx conte // 3b. BALANCE_AUTHORIZATION Changes: Secondary should have exactly 1 (SET with authorized flag) suite.Require().Len(authChanges.Edges, 1, "should have exactly 1 BALANCE_AUTHORIZATION/SET change") authChange := authChanges.Edges[0].Node.(*types.BalanceAuthorizationChange) - validateBalanceAuthorizationChange(suite, authChange, secondaryAccount, types.StateChangeReasonSet, []string{"authorized"}, test2ContractAddress) + validateBalanceAuthorizationChange(suite, authChange, types.StateChangeReasonSet, []string{"authorized"}, test2ContractAddress) // 4. SPECIFIC BALANCE CHANGE VALIDATIONS // 4a. Validate MINT changes have correct token ID and account for _, edge := range mintChanges.Edges { bc := edge.Node.(*types.StandardBalanceChange) suite.Require().Equal(test2ContractAddress, bc.TokenID, "MINT token should be TEST2") - suite.Require().Equal(primaryAccount, bc.GetAccountID(), "MINT account should be Primary") suite.Require().NotEmpty(bc.Amount, "MINT amount should not be empty") } @@ -657,7 +647,6 @@ func (suite *DataValidationTestSuite) validateCustomAssetsStateChanges(ctx conte for _, edge := range burnChanges.Edges { bc := edge.Node.(*types.StandardBalanceChange) suite.Require().Equal(test2ContractAddress, bc.TokenID, "BURN token should be TEST2") - suite.Require().Equal(primaryAccount, bc.GetAccountID(), "BURN account should be Primary") suite.Require().NotEmpty(bc.Amount, "BURN amount should not be empty") } @@ -666,7 +655,6 @@ func (suite *DataValidationTestSuite) validateCustomAssetsStateChanges(ctx conte for _, edge := range creditChanges.Edges { bc := edge.Node.(*types.StandardBalanceChange) suite.Require().True(tokenSet.Contains(bc.TokenID), "CREDIT token should be TEST2 or XLM") - suite.Require().Equal(secondaryAccount, bc.GetAccountID(), "CREDIT account should be Secondary") suite.Require().NotEmpty(bc.Amount, "CREDIT amount should not be empty") } @@ -674,7 +662,6 @@ func (suite *DataValidationTestSuite) validateCustomAssetsStateChanges(ctx conte for _, edge := range debitChanges.Edges { bc := edge.Node.(*types.StandardBalanceChange) suite.Require().True(tokenSet.Contains(bc.TokenID), "DEBIT token should be TEST2 or XLM") - suite.Require().Equal(secondaryAccount, bc.GetAccountID(), "DEBIT account should be Secondary") suite.Require().NotEmpty(bc.Amount, "DEBIT amount should not be empty") } } @@ -787,7 +774,7 @@ func (suite *DataValidationTestSuite) validateAuthRequiredIssuerSetupStateChange expectedFlags := []string{"auth_required", "auth_revocable", "auth_clawback_enabled"} flagsSetChange := flagsSetPrimary.Edges[0].Node.(*types.FlagsChange) validateStateChangeBase(suite, flagsSetChange, ledgerNumber) - validateFlagsChange(suite, flagsSetChange, primaryAccount, types.StateChangeReasonSet, expectedFlags) + validateFlagsChange(suite, flagsSetChange, types.StateChangeReasonSet, expectedFlags) } func (suite *DataValidationTestSuite) validateAuthRequiredAssetStateChanges(ctx context.Context, txHash string, ledgerNumber int64) { @@ -870,26 +857,26 @@ func (suite *DataValidationTestSuite) validateAuthRequiredAssetStateChanges(ctx // First SET change: clawback_enabled flag from trustline creation authSetSecondaryClawback := balanceAuthSetSecondary.Edges[0].Node.(*types.BalanceAuthorizationChange) - validateBalanceAuthorizationChange(suite, authSetSecondaryClawback, secondaryAccount, types.StateChangeReasonSet, []string{"clawback_enabled"}, test1ContractAddress) + validateBalanceAuthorizationChange(suite, authSetSecondaryClawback, types.StateChangeReasonSet, []string{"clawback_enabled"}, test1ContractAddress) // Second SET change: authorized flag from SetTrustLineFlags authSetSecondaryAuthorized := balanceAuthSetSecondary.Edges[1].Node.(*types.BalanceAuthorizationChange) - validateBalanceAuthorizationChange(suite, authSetSecondaryAuthorized, secondaryAccount, types.StateChangeReasonSet, []string{"authorized"}, test1ContractAddress) + validateBalanceAuthorizationChange(suite, authSetSecondaryAuthorized, types.StateChangeReasonSet, []string{"authorized"}, test1ContractAddress) // Secondary account: BALANCE_AUTHORIZATION/CLEAR with "authorized" flag suite.Require().Len(balanceAuthClearSecondary.Edges, 1, "should have exactly 1 BALANCE_AUTHORIZATION/CLEAR for secondary") authClearSecondary := balanceAuthClearSecondary.Edges[0].Node.(*types.BalanceAuthorizationChange) - validateBalanceAuthorizationChange(suite, authClearSecondary, secondaryAccount, types.StateChangeReasonClear, []string{"authorized"}, test1ContractAddress) + validateBalanceAuthorizationChange(suite, authClearSecondary, types.StateChangeReasonClear, []string{"authorized"}, test1ContractAddress) // 5. TRUSTLINE STATE CHANGES VALIDATION FOR SECONDARY ACCOUNT suite.Require().Len(trustlineAdd.Edges, 1, "should have exactly 1 TRUSTLINE/ADD") suite.Require().Len(trustlineRemove.Edges, 1, "should have exactly 1 TRUSTLINE/REMOVE") trustlineAddChange := trustlineAdd.Edges[0].Node.(*types.TrustlineChange) - validateTrustlineChange(suite, trustlineAddChange, secondaryAccount, test1ContractAddress, "", types.StateChangeReasonAdd) + validateTrustlineChange(suite, trustlineAddChange, test1ContractAddress, "", types.StateChangeReasonAdd) trustlineRemoveChange := trustlineRemove.Edges[0].Node.(*types.TrustlineChange) - validateTrustlineChange(suite, trustlineRemoveChange, secondaryAccount, test1ContractAddress, "", types.StateChangeReasonRemove) + validateTrustlineChange(suite, trustlineRemoveChange, test1ContractAddress, "", types.StateChangeReasonRemove) // 6. BALANCE STATE CHANGES VALIDATION // Validate counts @@ -900,19 +887,19 @@ func (suite *DataValidationTestSuite) validateAuthRequiredAssetStateChanges(ctx // Validate MINT mintChange := balanceMint.Edges[0].Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, mintChange, test1ContractAddress, "10000000000", primaryAccount, types.StateChangeReasonMint) + validateBalanceChange(suite, mintChange, test1ContractAddress, "10000000000", types.StateChangeReasonMint) // Validate CREDIT creditChange := balanceCredit.Edges[0].Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, creditChange, test1ContractAddress, "10000000000", secondaryAccount, types.StateChangeReasonCredit) + validateBalanceChange(suite, creditChange, test1ContractAddress, "10000000000", types.StateChangeReasonCredit) // Validate BURN burnChange := balanceBurn.Edges[0].Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, burnChange, test1ContractAddress, "10000000000", primaryAccount, types.StateChangeReasonBurn) + validateBalanceChange(suite, burnChange, test1ContractAddress, "10000000000", types.StateChangeReasonBurn) // Validate DEBIT (from clawback) debitChange := balanceDebit.Edges[0].Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, debitChange, test1ContractAddress, "10000000000", secondaryAccount, types.StateChangeReasonDebit) + validateBalanceChange(suite, debitChange, test1ContractAddress, "10000000000", types.StateChangeReasonDebit) // 7. CONSERVATION LAW VALIDATIONS totalMint := sumAmounts(suite, balanceMint, test1ContractAddress) @@ -1011,32 +998,31 @@ func (suite *DataValidationTestSuite) validateAccountMergeStateChanges(ctx conte accountChange := accountMergeChanges.Edges[0].Node.(*types.AccountChange) suite.Require().Equal(types.StateChangeCategoryAccount, accountChange.GetType(), "should be ACCOUNT type") suite.Require().Equal(types.StateChangeReasonMerge, accountChange.GetReason(), "reason should be MERGE") - suite.Require().Equal(primaryAccount, accountChange.GetAccountID(), "account ID should be the destination account (receiving the merge)") // Validate BALANCE/CREDIT change suite.Require().Len(balanceCreditChanges.Edges, 1, "should have exactly 1 BALANCE/CREDIT change") balanceCreditChange := balanceCreditChanges.Edges[0].Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, balanceCreditChange, xlmContractAddress, "50000000", primaryAccount, types.StateChangeReasonCredit) + validateBalanceChange(suite, balanceCreditChange, xlmContractAddress, "50000000", types.StateChangeReasonCredit) // 5. RESERVES/UNSPONSOR STATE CHANGES VALIDATION FOR SPONSORED ACCOUNT suite.Require().Len(sponsoredReservesUnsponsorChanges.Edges, 1, "should have exactly 1 RESERVES/UNSPONSOR for sponsored account") sponsoredReservesChange := sponsoredReservesUnsponsorChanges.Edges[0].Node.(*types.ReservesChange) - validateReservesSponsorshipChangeForSponsoredAccount(suite, sponsoredReservesChange, sponsoredNewAccount, types.StateChangeReasonUnsponsor, primaryAccount) + validateReservesSponsorshipChangeForSponsoredAccount(suite, sponsoredReservesChange, types.StateChangeReasonUnsponsor, primaryAccount) // Validate BALANCE/DEBIT change suite.Require().Len(balanceDebitChanges.Edges, 1, "should have exactly 1 BALANCE/DEBIT change") balanceDebitChange := balanceDebitChanges.Edges[0].Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, balanceDebitChange, xlmContractAddress, "50000000", sponsoredNewAccount, types.StateChangeReasonDebit) + validateBalanceChange(suite, balanceDebitChange, xlmContractAddress, "50000000", types.StateChangeReasonDebit) // Validate RESERVES/UNSPONSOR for sponsored account suite.Require().Len(sponsoredReservesUnsponsorChanges.Edges, 1, "should have exactly 1 RESERVES/UNSPONSOR for sponsored account") sponsoredReservesChange = sponsoredReservesUnsponsorChanges.Edges[0].Node.(*types.ReservesChange) - validateReservesSponsorshipChangeForSponsoredAccount(suite, sponsoredReservesChange, sponsoredNewAccount, types.StateChangeReasonUnsponsor, primaryAccount) + validateReservesSponsorshipChangeForSponsoredAccount(suite, sponsoredReservesChange, types.StateChangeReasonUnsponsor, primaryAccount) // Validate RESERVES/UNSPONSOR for sponsor account suite.Require().Len(sponsorReservesUnsponsorChanges.Edges, 1, "should have exactly 1 RESERVES/UNSPONSOR for sponsor account") sponsorReservesChange := sponsorReservesUnsponsorChanges.Edges[0].Node.(*types.ReservesChange) - validateReservesSponsorshipChangeForSponsoringAccount(suite, sponsorReservesChange, primaryAccount, types.StateChangeReasonUnsponsor, sponsoredNewAccount) + validateReservesSponsorshipChangeForSponsoringAccount(suite, sponsorReservesChange, types.StateChangeReasonUnsponsor, sponsoredNewAccount) } func (suite *DataValidationTestSuite) TestInvokeContractOpsDataValidation() { @@ -1118,12 +1104,12 @@ func (suite *DataValidationTestSuite) validateInvokeContractStateChanges(ctx con // Validate BALANCE/CREDIT change suite.Require().Len(balanceCreditChanges.Edges, 1, "should have exactly 1 BALANCE/CREDIT change") balanceCreditChange := balanceCreditChanges.Edges[0].Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, balanceCreditChange, xlmContractAddress, "100000000", primaryAccount, types.StateChangeReasonCredit) + validateBalanceChange(suite, balanceCreditChange, xlmContractAddress, "100000000", types.StateChangeReasonCredit) // Validate BALANCE/DEBIT change suite.Require().Len(balanceDebitChanges.Edges, 1, "should have exactly 1 BALANCE/DEBIT change") balanceDebitChange := balanceDebitChanges.Edges[0].Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, balanceDebitChange, xlmContractAddress, "100000000", primaryAccount, types.StateChangeReasonDebit) + validateBalanceChange(suite, balanceDebitChange, xlmContractAddress, "100000000", types.StateChangeReasonDebit) } func (suite *DataValidationTestSuite) TestCreateClaimableBalanceOpsDataValidation() { @@ -1191,16 +1177,6 @@ func (suite *DataValidationTestSuite) validateCreateClaimableBalanceStateChanges suite.Require().NoError(err, "failed to marshal state change") fmt.Printf("%s\n", string(jsonBytes)) validateStateChangeBase(suite, edge.Node, ledgerNumber) - - // Validate that no state changes have claimable balance IDs as accounts - accountID := edge.Node.GetAccountID() - suite.Require().NotEmpty(accountID, "account ID should not be empty") - - // Decode the account ID to check its version byte - versionByte, _, err := strkey.DecodeAny(accountID) - suite.Require().NoError(err, "account ID should be a valid strkey: %s", accountID) - suite.Require().NotEqual(strkey.VersionByteClaimableBalance, versionByte, - "state change should not have claimable balance ID as account: %s", accountID) } fmt.Printf("primary account: %s\n", primaryAccount) fmt.Printf("secondary account: %s\n", secondaryAccount) @@ -1230,7 +1206,7 @@ func (suite *DataValidationTestSuite) validateCreateClaimableBalanceStateChanges // 3. TRUSTLINE STATE CHANGES VALIDATION FOR SECONDARY ACCOUNT suite.Require().Len(trustlineAdd.Edges, 1, "should have exactly 1 TRUSTLINE/ADD") trustlineAddChange := trustlineAdd.Edges[0].Node.(*types.TrustlineChange) - validateTrustlineChange(suite, trustlineAddChange, secondaryAccount, test3ContractAddress, "", types.StateChangeReasonAdd) + validateTrustlineChange(suite, trustlineAddChange, test3ContractAddress, "", types.StateChangeReasonAdd) // 4. BALANCE_AUTHORIZATION STATE CHANGES VALIDATION // Secondary account should have 2 BALANCE_AUTHORIZATION/SET changes: @@ -1238,28 +1214,28 @@ func (suite *DataValidationTestSuite) validateCreateClaimableBalanceStateChanges // - One with authorized flag (from SetTrustLineFlags operation) suite.Require().Len(balanceAuthSet.Edges, 2, "should have exactly 2 BALANCE_AUTHORIZATION/SET for secondary") authSetSecondary := balanceAuthSet.Edges[0].Node.(*types.BalanceAuthorizationChange) - validateBalanceAuthorizationChange(suite, authSetSecondary, secondaryAccount, types.StateChangeReasonSet, []string{"clawback_enabled"}, test3ContractAddress) + validateBalanceAuthorizationChange(suite, authSetSecondary, types.StateChangeReasonSet, []string{"clawback_enabled"}, test3ContractAddress) // Second SET change: authorized flag from SetTrustLineFlags authSetSecondaryAuthorized := balanceAuthSet.Edges[1].Node.(*types.BalanceAuthorizationChange) - validateBalanceAuthorizationChange(suite, authSetSecondaryAuthorized, secondaryAccount, types.StateChangeReasonSet, []string{"authorized"}, test3ContractAddress) + validateBalanceAuthorizationChange(suite, authSetSecondaryAuthorized, types.StateChangeReasonSet, []string{"authorized"}, test3ContractAddress) // 5. BALANCE STATE CHANGES VALIDATION - 2 claimable balances are created suite.Require().Len(balanceMint.Edges, 2, "should have exactly 2 BALANCE/MINT") for _, edge := range balanceMint.Edges { mintChange := edge.Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, mintChange, test3ContractAddress, "10000000", primaryAccount, types.StateChangeReasonMint) + validateBalanceChange(suite, mintChange, test3ContractAddress, "10000000", types.StateChangeReasonMint) } // 6. 2 RESERVES/SPONSOR STATE CHANGES VALIDATION FOR SPONSORING ACCOUNT for 2 claimable balances suite.Require().Len(reservesSponsorForSponsor.Edges, 2, "should have exactly 2 RESERVES/SPONSOR for sponsor") change := reservesSponsorForSponsor.Edges[0].Node.(*types.ReservesChange) suite.Require().Equal(suite.testEnv.ClaimBalanceID, *change.ClaimableBalanceID, "claimable balance ID does not match") - validateReservesSponsorshipChangeForSponsoringAccount(suite, change, primaryAccount, types.StateChangeReasonSponsor, "") + validateReservesSponsorshipChangeForSponsoringAccount(suite, change, types.StateChangeReasonSponsor, "") change = reservesSponsorForSponsor.Edges[1].Node.(*types.ReservesChange) suite.Require().Equal(suite.testEnv.ClawbackBalanceID, *change.ClaimableBalanceID, "claimable balance ID for clawback does not match") - validateReservesSponsorshipChangeForSponsoringAccount(suite, change, primaryAccount, types.StateChangeReasonSponsor, "") + validateReservesSponsorshipChangeForSponsoringAccount(suite, change, types.StateChangeReasonSponsor, "") } func (suite *DataValidationTestSuite) TestClaimClaimableBalanceDataValidation() { @@ -1336,13 +1312,13 @@ func (suite *DataValidationTestSuite) validateClaimClaimableBalanceStateChanges( // 3. VALIDATE BALANCE/CREDIT CHANGE suite.Require().Len(balanceCreditChanges.Edges, 1, "should have exactly 1 BALANCE/CREDIT change") balanceCreditChange := balanceCreditChanges.Edges[0].Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, balanceCreditChange, test3ContractAddress, "10000000", secondaryAccount, types.StateChangeReasonCredit) + validateBalanceChange(suite, balanceCreditChange, test3ContractAddress, "10000000", types.StateChangeReasonCredit) // 4. RESERVES/UNSPONSOR STATE CHANGES VALIDATION FOR SPONSORING ACCOUNT suite.Require().Len(reservesUnsponsorForSponsor.Edges, 1, "should have exactly 1 RESERVES/UNSPONSOR for sponsor") reservesUnsponsorForSponsorChange := reservesUnsponsorForSponsor.Edges[0].Node.(*types.ReservesChange) suite.Require().Equal(suite.testEnv.ClaimBalanceID, *reservesUnsponsorForSponsorChange.ClaimableBalanceID, "claimable balance ID does not match") - validateReservesSponsorshipChangeForSponsoringAccount(suite, reservesUnsponsorForSponsorChange, primaryAccount, types.StateChangeReasonUnsponsor, "") + validateReservesSponsorshipChangeForSponsoringAccount(suite, reservesUnsponsorForSponsorChange, types.StateChangeReasonUnsponsor, "") } func (suite *DataValidationTestSuite) TestClawbackClaimableBalanceDataValidation() { @@ -1418,13 +1394,13 @@ func (suite *DataValidationTestSuite) validateClawbackClaimableBalanceStateChang // 3. VALIDATE BALANCE/BURN CHANGE suite.Require().Len(balanceBurnChanges.Edges, 1, "should have exactly 1 BALANCE/BURN change") balanceBurnChange := balanceBurnChanges.Edges[0].Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, balanceBurnChange, test3ContractAddress, "10000000", primaryAccount, types.StateChangeReasonBurn) + validateBalanceChange(suite, balanceBurnChange, test3ContractAddress, "10000000", types.StateChangeReasonBurn) // 4. RESERVES/UNSPONSOR STATE CHANGES VALIDATION FOR SPONSORING ACCOUNT FOR CLAWBACK BALANCE suite.Require().Len(reservesUnsponsorForSponsor.Edges, 1, "should have exactly 1 RESERVES/UNSPONSOR for sponsor") reservesUnsponsorForSponsorChange := reservesUnsponsorForSponsor.Edges[0].Node.(*types.ReservesChange) suite.Require().Equal(suite.testEnv.ClawbackBalanceID, *reservesUnsponsorForSponsorChange.ClaimableBalanceID, "claimable balance ID for clawback does not match") - validateReservesSponsorshipChangeForSponsoringAccount(suite, reservesUnsponsorForSponsorChange, primaryAccount, types.StateChangeReasonUnsponsor, "") + validateReservesSponsorshipChangeForSponsoringAccount(suite, reservesUnsponsorForSponsorChange, types.StateChangeReasonUnsponsor, "") } func (suite *DataValidationTestSuite) TestClearAuthFlagsOpsDataValidation() { @@ -1498,7 +1474,7 @@ func (suite *DataValidationTestSuite) validateClearAuthFlagsStateChanges(ctx con suite.Require().Len(flagsClearPrimary.Edges, 1, "should have exactly 1 FLAGS/CLEAR change for primary") expectedFlags := []string{"auth_required", "auth_revocable", "auth_clawback_enabled"} flagsClearChange := flagsClearPrimary.Edges[0].Node.(*types.FlagsChange) - validateFlagsChange(suite, flagsClearChange, primaryAccount, types.StateChangeReasonClear, expectedFlags) + validateFlagsChange(suite, flagsClearChange, types.StateChangeReasonClear, expectedFlags) } func (suite *DataValidationTestSuite) TestLiquidityPoolOpsDataValidation() { @@ -1609,38 +1585,38 @@ func (suite *DataValidationTestSuite) validateLiquidityPoolStateChanges(ctx cont suite.Require().Len(balanceAuthSet.Edges, 1, "should have exactly 1 BALANCE_AUTHORIZATION/SET for liquidity pool") balanceAuth := balanceAuthSet.Edges[0].Node.(*types.BalanceAuthorizationChange) suite.Require().Equal(suite.testEnv.LiquidityPoolID, *balanceAuth.LiquidityPoolID, "balance auth change liquidity pool ID does not match") - validateBalanceAuthorizationChange(suite, balanceAuth, primaryAccount, types.StateChangeReasonSet, []string{}, "") + validateBalanceAuthorizationChange(suite, balanceAuth, types.StateChangeReasonSet, []string{}, "") // 4. TRUSTLINE VALIDATION // LP trustlines should have null tokenId and pool ID in liquidityPoolId suite.Require().Len(trustlineAdd.Edges, 1, "should have exactly 1 TRUSTLINE/ADD for liquidity pool") trustlineAddChange := trustlineAdd.Edges[0].Node.(*types.TrustlineChange) - validateTrustlineChange(suite, trustlineAddChange, primaryAccount, "", suite.testEnv.LiquidityPoolID, types.StateChangeReasonAdd) + validateTrustlineChange(suite, trustlineAddChange, "", suite.testEnv.LiquidityPoolID, types.StateChangeReasonAdd) suite.Require().Len(trustlineRemove.Edges, 1, "should have exactly 1 TRUSTLINE/REMOVE for liquidity pool") trustlineRemoveChange := trustlineRemove.Edges[0].Node.(*types.TrustlineChange) - validateTrustlineChange(suite, trustlineRemoveChange, primaryAccount, "", suite.testEnv.LiquidityPoolID, types.StateChangeReasonRemove) + validateTrustlineChange(suite, trustlineRemoveChange, "", suite.testEnv.LiquidityPoolID, types.StateChangeReasonRemove) // 5. BALANCE CHANGES VALIDATION // DEBIT: XLM deposited into pool (amount = 1000000000) suite.Require().Len(balanceDebit.Edges, 1, "should have exactly 1 BALANCE/DEBIT") debitChange := balanceDebit.Edges[0].Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, debitChange, xlmContractAddress, "1000000000", primaryAccount, types.StateChangeReasonDebit) + validateBalanceChange(suite, debitChange, xlmContractAddress, "1000000000", types.StateChangeReasonDebit) // CREDIT: XLM withdrawn from pool (amount = 1000000000) suite.Require().Len(balanceCredit.Edges, 1, "should have exactly 1 BALANCE/CREDIT") creditChange := balanceCredit.Edges[0].Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, creditChange, xlmContractAddress, "1000000000", primaryAccount, types.StateChangeReasonCredit) + validateBalanceChange(suite, creditChange, xlmContractAddress, "1000000000", types.StateChangeReasonCredit) // MINT: TEST2 minted to LP (amount = 1000000000) suite.Require().Len(balanceMint.Edges, 1, "should have exactly 1 BALANCE/MINT") mintChange := balanceMint.Edges[0].Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, mintChange, test2ContractAddress, "1000000000", primaryAccount, types.StateChangeReasonMint) + validateBalanceChange(suite, mintChange, test2ContractAddress, "1000000000", types.StateChangeReasonMint) // BURN: TEST2 burned from LP back to issuer (amount = 1000000000) suite.Require().Len(balanceBurn.Edges, 1, "should have exactly 1 BALANCE/BURN") burnChange := balanceBurn.Edges[0].Node.(*types.StandardBalanceChange) - validateBalanceChange(suite, burnChange, test2ContractAddress, "1000000000", primaryAccount, types.StateChangeReasonBurn) + validateBalanceChange(suite, burnChange, test2ContractAddress, "1000000000", types.StateChangeReasonBurn) } func (suite *DataValidationTestSuite) TestRevokeSponsorshipOpsDataValidation() { @@ -1730,15 +1706,15 @@ func (suite *DataValidationTestSuite) validateRevokeSponsorshipStateChanges(ctx // Validate sponsorship revocation mc := metadataDataEntry.Edges[0].Node.(*types.MetadataChange) - validateMetadataChange(suite, mc, secondaryAccount, types.StateChangeReasonDataEntry, "sponsored_data", "new", "test_value") + validateMetadataChange(suite, mc, types.StateChangeReasonDataEntry, "sponsored_data", "new", "test_value") mc = metadataDataEntry.Edges[1].Node.(*types.MetadataChange) - validateMetadataChange(suite, mc, secondaryAccount, types.StateChangeReasonDataEntry, "sponsored_data", "old", "test_value") + validateMetadataChange(suite, mc, types.StateChangeReasonDataEntry, "sponsored_data", "old", "test_value") // 5. RESERVES STATE CHANGES VALIDATION rc := primaryReservesSponsor.Edges[0].Node.(*types.ReservesChange) suite.Require().Equal("sponsored_data", *rc.SponsoredData, "sponsored data value does not match") - validateReservesSponsorshipChangeForSponsoringAccount(suite, rc, primaryAccount, types.StateChangeReasonSponsor, "") + validateReservesSponsorshipChangeForSponsoringAccount(suite, rc, types.StateChangeReasonSponsor, "") rc = primaryReservesUnsponsor.Edges[0].Node.(*types.ReservesChange) suite.Require().Equal("sponsored_data", *rc.SponsoredData, "sponsored data value does not match") - validateReservesSponsorshipChangeForSponsoringAccount(suite, rc, primaryAccount, types.StateChangeReasonUnsponsor, "") + validateReservesSponsorshipChangeForSponsoringAccount(suite, rc, types.StateChangeReasonUnsponsor, "") } diff --git a/pkg/wbclient/types/statechange.go b/pkg/wbclient/types/statechange.go index e07923d88..94e414d71 100644 --- a/pkg/wbclient/types/statechange.go +++ b/pkg/wbclient/types/statechange.go @@ -15,7 +15,6 @@ type StateChangeNode interface { GetIngestedAt() time.Time GetLedgerCreatedAt() time.Time GetLedgerNumber() uint32 - GetAccountID() string } // BaseStateChangeFields contains the common fields shared by all state change types @@ -25,7 +24,6 @@ type BaseStateChangeFields struct { IngestedAt time.Time `json:"ingestedAt"` LedgerCreatedAt time.Time `json:"ledgerCreatedAt"` LedgerNumber uint32 `json:"ledgerNumber"` - Account Account `json:"account"` } // GetType returns the state change category @@ -53,11 +51,6 @@ func (b BaseStateChangeFields) GetLedgerNumber() uint32 { return b.LedgerNumber } -// GetAccountID returns the account address -func (b BaseStateChangeFields) GetAccountID() string { - return b.Account.Address -} - // StandardBalanceChange represents a standard balance state change type StandardBalanceChange struct { BaseStateChangeFields From 8cd6492edfde8f84d131fbaa52373aaa4ef8cac4 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Sat, 21 Feb 2026 21:30:48 -0500 Subject: [PATCH 76/77] Regenerate GraphQL code after merge Cosmetic changes from gqlgen: import reordering and type declaration formatting. Co-Authored-By: Claude Opus 4.6 --- internal/serve/graphql/generated/generated.go | 5 ++--- .../graphql/resolvers/mutations.resolvers.go | 3 +-- .../graphql/resolvers/queries.resolvers.go | 3 +-- .../resolvers/statechange.resolvers.go | 20 +++++++++---------- 4 files changed, 13 insertions(+), 18 deletions(-) diff --git a/internal/serve/graphql/generated/generated.go b/internal/serve/graphql/generated/generated.go index 9fd4244ab..eccdb3737 100644 --- a/internal/serve/graphql/generated/generated.go +++ b/internal/serve/graphql/generated/generated.go @@ -14,11 +14,10 @@ import ( "github.com/99designs/gqlgen/graphql" "github.com/99designs/gqlgen/graphql/introspection" - gqlparser "github.com/vektah/gqlparser/v2" - "github.com/vektah/gqlparser/v2/ast" - "github.com/stellar/wallet-backend/internal/indexer/types" "github.com/stellar/wallet-backend/internal/serve/graphql/scalars" + gqlparser "github.com/vektah/gqlparser/v2" + "github.com/vektah/gqlparser/v2/ast" ) // region ************************** generated!.gotpl ************************** diff --git a/internal/serve/graphql/resolvers/mutations.resolvers.go b/internal/serve/graphql/resolvers/mutations.resolvers.go index 9b2a6390e..4b5d1086c 100644 --- a/internal/serve/graphql/resolvers/mutations.resolvers.go +++ b/internal/serve/graphql/resolvers/mutations.resolvers.go @@ -10,14 +10,13 @@ import ( "fmt" "github.com/stellar/go-stellar-sdk/txnbuild" - "github.com/vektah/gqlparser/v2/gqlerror" - "github.com/stellar/wallet-backend/internal/entities" graphql1 "github.com/stellar/wallet-backend/internal/serve/graphql/generated" "github.com/stellar/wallet-backend/internal/services" "github.com/stellar/wallet-backend/internal/signing" "github.com/stellar/wallet-backend/internal/signing/store" "github.com/stellar/wallet-backend/pkg/sorobanauth" + "github.com/vektah/gqlparser/v2/gqlerror" ) // BuildTransaction is the resolver for the buildTransaction field. diff --git a/internal/serve/graphql/resolvers/queries.resolvers.go b/internal/serve/graphql/resolvers/queries.resolvers.go index de99037a2..d4bfef5ff 100644 --- a/internal/serve/graphql/resolvers/queries.resolvers.go +++ b/internal/serve/graphql/resolvers/queries.resolvers.go @@ -11,13 +11,12 @@ import ( "sync" "github.com/stellar/go-stellar-sdk/support/log" - "github.com/vektah/gqlparser/v2/gqlerror" - "github.com/stellar/wallet-backend/internal/data" "github.com/stellar/wallet-backend/internal/entities" "github.com/stellar/wallet-backend/internal/indexer/types" graphql1 "github.com/stellar/wallet-backend/internal/serve/graphql/generated" "github.com/stellar/wallet-backend/internal/utils" + "github.com/vektah/gqlparser/v2/gqlerror" ) // TransactionByHash is the resolver for the transactionByHash field. diff --git a/internal/serve/graphql/resolvers/statechange.resolvers.go b/internal/serve/graphql/resolvers/statechange.resolvers.go index 15db70ed1..8d66548b5 100644 --- a/internal/serve/graphql/resolvers/statechange.resolvers.go +++ b/internal/serve/graphql/resolvers/statechange.resolvers.go @@ -407,14 +407,12 @@ func (r *Resolver) TrustlineChange() graphql1.TrustlineChangeResolver { return &trustlineChangeResolver{r} } -type ( - accountChangeResolver struct{ *Resolver } - balanceAuthorizationChangeResolver struct{ *Resolver } - flagsChangeResolver struct{ *Resolver } - metadataChangeResolver struct{ *Resolver } - reservesChangeResolver struct{ *Resolver } - signerChangeResolver struct{ *Resolver } - signerThresholdsChangeResolver struct{ *Resolver } - standardBalanceChangeResolver struct{ *Resolver } - trustlineChangeResolver struct{ *Resolver } -) +type accountChangeResolver struct{ *Resolver } +type balanceAuthorizationChangeResolver struct{ *Resolver } +type flagsChangeResolver struct{ *Resolver } +type metadataChangeResolver struct{ *Resolver } +type reservesChangeResolver struct{ *Resolver } +type signerChangeResolver struct{ *Resolver } +type signerThresholdsChangeResolver struct{ *Resolver } +type standardBalanceChangeResolver struct{ *Resolver } +type trustlineChangeResolver struct{ *Resolver } From cfe95819df77fb47d5f8265b14979dcc63c35886 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Sat, 21 Feb 2026 21:33:35 -0500 Subject: [PATCH 77/77] Remove unused AccountModel.BatchGetByStateChangeIDs query-optimize simplified state change account resolution to use AccountID directly from the row, making this method and its type obsolete. Also includes gofumpt formatting fixes. Co-Authored-By: Claude Opus 4.6 --- internal/data/accounts.go | 31 --------- internal/data/accounts_test.go | 66 ------------------- internal/serve/graphql/generated/generated.go | 5 +- .../graphql/resolvers/mutations.resolvers.go | 3 +- .../graphql/resolvers/queries.resolvers.go | 3 +- .../resolvers/statechange.resolvers.go | 20 +++--- 6 files changed, 18 insertions(+), 110 deletions(-) diff --git a/internal/data/accounts.go b/internal/data/accounts.go index 1b0ff6a82..6e722076d 100644 --- a/internal/data/accounts.go +++ b/internal/data/accounts.go @@ -5,7 +5,6 @@ package data import ( "context" "fmt" - "strings" "time" "github.com/lib/pq" @@ -77,33 +76,3 @@ func (m *AccountModel) BatchGetByOperationIDs(ctx context.Context, operationIDs m.MetricsService.IncDBQuery("BatchGetByOperationIDs", "operations_accounts") return accounts, nil } - -// BatchGetByStateChangeIDs gets the accounts that are associated with the given state change IDs. -func (m *AccountModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs []int64, scOpIDs []int64, scOrders []int64, columns string) ([]*types.AccountWithStateChangeID, error) { - // Build tuples for the IN clause. Since (to_id, operation_id, state_change_order) is the primary key of state_changes, - // it will be faster to search on this tuple. - tuples := make([]string, len(scOrders)) - for i := range scOrders { - tuples[i] = fmt.Sprintf("(%d, %d, %d)", scToIDs[i], scOpIDs[i], scOrders[i]) - } - - query := fmt.Sprintf(` - SELECT account_id AS stellar_address, CONCAT(to_id, '-', operation_id, '-', state_change_order) AS state_change_id - FROM state_changes - WHERE (to_id, operation_id, state_change_order) IN (%s) - ORDER BY ledger_created_at DESC - `, strings.Join(tuples, ", ")) - - var accountsWithStateChanges []*types.AccountWithStateChangeID - start := time.Now() - err := m.DB.SelectContext(ctx, &accountsWithStateChanges, query) - duration := time.Since(start).Seconds() - m.MetricsService.ObserveDBQueryDuration("BatchGetByStateChangeIDs", "state_changes", duration) - m.MetricsService.ObserveDBBatchSize("BatchGetByStateChangeIDs", "state_changes", len(scOrders)) - if err != nil { - m.MetricsService.IncDBQueryError("BatchGetByStateChangeIDs", "state_changes", utils.GetDBErrorType(err)) - return nil, fmt.Errorf("getting accounts by state change IDs: %w", err) - } - m.MetricsService.IncDBQuery("BatchGetByStateChangeIDs", "state_changes") - return accountsWithStateChanges, nil -} diff --git a/internal/data/accounts_test.go b/internal/data/accounts_test.go index 623db9d80..118bee18b 100644 --- a/internal/data/accounts_test.go +++ b/internal/data/accounts_test.go @@ -151,69 +151,3 @@ func TestAccountModel_IsAccountFeeBumpEligible(t *testing.T) { require.NoError(t, err) assert.True(t, isFeeBumpEligible) } - -func TestAccountModelBatchGetByStateChangeIDs(t *testing.T) { - dbt := dbtest.Open(t) - defer dbt.Close() - dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) - require.NoError(t, err) - defer dbConnectionPool.Close() - - mockMetricsService := metrics.NewMockMetricsService() - mockMetricsService.On("ObserveDBQueryDuration", "BatchGetByStateChangeIDs", "state_changes", mock.Anything).Return() - mockMetricsService.On("IncDBQuery", "BatchGetByStateChangeIDs", "state_changes").Return() - mockMetricsService.On("ObserveDBBatchSize", "BatchGetByStateChangeIDs", "state_changes", mock.Anything).Return().Maybe() - defer mockMetricsService.AssertExpectations(t) - - m := &AccountModel{ - DB: dbConnectionPool, - MetricsService: mockMetricsService, - } - - ctx := context.Background() - address1 := keypair.MustRandom().Address() - address2 := keypair.MustRandom().Address() - toID1 := int64(4096) - toID2 := int64(8192) - stateChangeOrder1 := int64(1) - stateChangeOrder2 := int64(1) - - // Insert test transactions first (hash is BYTEA, using valid 64-char hex strings) - testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001") - testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002") - _, err = m.DB.ExecContext(ctx, "INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at) VALUES ($1, 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, NOW()), ($2, 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, NOW())", testHash1, testHash2) - require.NoError(t, err) - - // Insert test operations (IDs must be in TOID range for each transaction) - xdr1 := types.XDRBytea([]byte("xdr1")) - xdr2 := types.XDRBytea([]byte("xdr2")) - _, err = m.DB.ExecContext(ctx, "INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) VALUES (4097, 'PAYMENT', $1, 'op_success', true, 1, NOW()), (8193, 'PAYMENT', $2, 'op_success', true, 2, NOW())", xdr1, xdr2) - require.NoError(t, err) - - // Insert test state changes that reference the accounts (state_changes.account_id is TEXT) - _, err = m.DB.ExecContext(ctx, ` - INSERT INTO state_changes ( - to_id, state_change_order, state_change_category, ledger_created_at, - ledger_number, account_id, operation_id - ) VALUES - ($1, $2, 'BALANCE', NOW(), 1, $3, 4097), - ($4, $5, 'BALANCE', NOW(), 2, $6, 8193) - `, toID1, stateChangeOrder1, types.AddressBytea(address1), toID2, stateChangeOrder2, types.AddressBytea(address2)) - require.NoError(t, err) - - // Test BatchGetByStateChangeIDs function - scToIDs := []int64{toID1, toID2} - scOpIDs := []int64{4097, 8193} - scOrders := []int64{stateChangeOrder1, stateChangeOrder2} - accounts, err := m.BatchGetByStateChangeIDs(ctx, scToIDs, scOpIDs, scOrders, "") - require.NoError(t, err) - assert.Len(t, accounts, 2) - - // Verify accounts are returned with correct state_change_id (format: to_id-operation_id-state_change_order) - addressSet := make(map[string]string) - for _, acc := range accounts { - addressSet[string(acc.StellarAddress)] = acc.StateChangeID - } - assert.Equal(t, "4096-4097-1", addressSet[address1]) - assert.Equal(t, "8192-8193-1", addressSet[address2]) -} diff --git a/internal/serve/graphql/generated/generated.go b/internal/serve/graphql/generated/generated.go index eccdb3737..9fd4244ab 100644 --- a/internal/serve/graphql/generated/generated.go +++ b/internal/serve/graphql/generated/generated.go @@ -14,10 +14,11 @@ import ( "github.com/99designs/gqlgen/graphql" "github.com/99designs/gqlgen/graphql/introspection" - "github.com/stellar/wallet-backend/internal/indexer/types" - "github.com/stellar/wallet-backend/internal/serve/graphql/scalars" gqlparser "github.com/vektah/gqlparser/v2" "github.com/vektah/gqlparser/v2/ast" + + "github.com/stellar/wallet-backend/internal/indexer/types" + "github.com/stellar/wallet-backend/internal/serve/graphql/scalars" ) // region ************************** generated!.gotpl ************************** diff --git a/internal/serve/graphql/resolvers/mutations.resolvers.go b/internal/serve/graphql/resolvers/mutations.resolvers.go index 4b5d1086c..9b2a6390e 100644 --- a/internal/serve/graphql/resolvers/mutations.resolvers.go +++ b/internal/serve/graphql/resolvers/mutations.resolvers.go @@ -10,13 +10,14 @@ import ( "fmt" "github.com/stellar/go-stellar-sdk/txnbuild" + "github.com/vektah/gqlparser/v2/gqlerror" + "github.com/stellar/wallet-backend/internal/entities" graphql1 "github.com/stellar/wallet-backend/internal/serve/graphql/generated" "github.com/stellar/wallet-backend/internal/services" "github.com/stellar/wallet-backend/internal/signing" "github.com/stellar/wallet-backend/internal/signing/store" "github.com/stellar/wallet-backend/pkg/sorobanauth" - "github.com/vektah/gqlparser/v2/gqlerror" ) // BuildTransaction is the resolver for the buildTransaction field. diff --git a/internal/serve/graphql/resolvers/queries.resolvers.go b/internal/serve/graphql/resolvers/queries.resolvers.go index d4bfef5ff..de99037a2 100644 --- a/internal/serve/graphql/resolvers/queries.resolvers.go +++ b/internal/serve/graphql/resolvers/queries.resolvers.go @@ -11,12 +11,13 @@ import ( "sync" "github.com/stellar/go-stellar-sdk/support/log" + "github.com/vektah/gqlparser/v2/gqlerror" + "github.com/stellar/wallet-backend/internal/data" "github.com/stellar/wallet-backend/internal/entities" "github.com/stellar/wallet-backend/internal/indexer/types" graphql1 "github.com/stellar/wallet-backend/internal/serve/graphql/generated" "github.com/stellar/wallet-backend/internal/utils" - "github.com/vektah/gqlparser/v2/gqlerror" ) // TransactionByHash is the resolver for the transactionByHash field. diff --git a/internal/serve/graphql/resolvers/statechange.resolvers.go b/internal/serve/graphql/resolvers/statechange.resolvers.go index 8d66548b5..15db70ed1 100644 --- a/internal/serve/graphql/resolvers/statechange.resolvers.go +++ b/internal/serve/graphql/resolvers/statechange.resolvers.go @@ -407,12 +407,14 @@ func (r *Resolver) TrustlineChange() graphql1.TrustlineChangeResolver { return &trustlineChangeResolver{r} } -type accountChangeResolver struct{ *Resolver } -type balanceAuthorizationChangeResolver struct{ *Resolver } -type flagsChangeResolver struct{ *Resolver } -type metadataChangeResolver struct{ *Resolver } -type reservesChangeResolver struct{ *Resolver } -type signerChangeResolver struct{ *Resolver } -type signerThresholdsChangeResolver struct{ *Resolver } -type standardBalanceChangeResolver struct{ *Resolver } -type trustlineChangeResolver struct{ *Resolver } +type ( + accountChangeResolver struct{ *Resolver } + balanceAuthorizationChangeResolver struct{ *Resolver } + flagsChangeResolver struct{ *Resolver } + metadataChangeResolver struct{ *Resolver } + reservesChangeResolver struct{ *Resolver } + signerChangeResolver struct{ *Resolver } + signerThresholdsChangeResolver struct{ *Resolver } + standardBalanceChangeResolver struct{ *Resolver } + trustlineChangeResolver struct{ *Resolver } +)