Skip to content

Remove envelope_xdr and meta_xdr from transactions table #549

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aditya1702 merged 32 commits into from
Mar 30, 2026
Merged

Remove envelope_xdr and meta_xdr from transactions table #549

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
9ed3a3f
metrics: add concrete metric structs with wallet_ namespace prefix
aditya1702 Mar 18, 2026
b6d99b8
metrics: migrate data models to use concrete *DBMetrics struct
aditya1702 Mar 18, 2026
1415c10
metrics: migrate RPC service to use concrete *RPCMetrics struct
aditya1702 Mar 18, 2026
94b6923
metrics: migrate middleware to use concrete metric structs
aditya1702 Mar 18, 2026
179a048
metrics: migrate ingestion, indexer, and processors to concrete structs
aditya1702 Mar 18, 2026
fb5bed8
metrics: migrate all tests to real registries, delete legacy interface
aditya1702 Mar 18, 2026
3d65ee8
refactor db metrics
aditya1702 Mar 19, 2026
c3b624d
make check
aditya1702 Mar 19, 2026
c91fa31
Add metrics tests
aditya1702 Mar 19, 2026
a3ad0a6
Add CollectAndCompare tests
aditya1702 Mar 19, 2026
4c6d429
Merge branch 'feature/remove-metrics-interface' into feature/db-metrics
aditya1702 Mar 19, 2026
14661b9
fix db test
aditya1702 Mar 19, 2026
b8c6b65
Add operation-level GraphQL metrics and middleware
aditya1702 Mar 19, 2026
bc117ef
Create graphql_field_metrics_test.go
aditya1702 Mar 19, 2026
12f661a
make check
aditya1702 Mar 19, 2026
40655b2
Add comments for DB metrics
aditya1702 Mar 19, 2026
6acc2f6
Refactor ingestion metrics; add retries/errors
aditya1702 Mar 19, 2026
9ccda4d
Enhance RPC metrics with histograms and gauges
aditya1702 Mar 19, 2026
9908e41
Update rpc.go
aditya1702 Mar 19, 2026
141c63a
Rename pool label and expand pool/DB metrics
aditya1702 Mar 19, 2026
6599e2f
Add QueryExecMode to DB pool config
aditya1702 Mar 20, 2026
5db8d49
remove envelope_xdr and meta_xdr - 1
aditya1702 Mar 26, 2026
39e1971
fix all tests
aditya1702 Mar 26, 2026
90ef7ef
Add back the envelopeXDR and metaXDR temporarily for tests
aditya1702 Mar 26, 2026
17697bc
Refactor GraphQL metrics and remove RPC heartbeat
aditya1702 Mar 30, 2026
c3a1f20
Merge branch 'feature/fix-all-metrics' into refactor/remove-txn-xdrs
aditya1702 Mar 30, 2026
97bbe87
Break up the huge `MetricsService` interface (#543)
aditya1702 Mar 30, 2026
5d50854
Merge branch 'feature/finalize-metrics' into feature/fix-all-metrics
aditya1702 Mar 30, 2026
2a4deb8
Merge branch 'feature/fix-all-metrics' into refactor/remove-txn-xdrs
aditya1702 Mar 30, 2026
4f95a33
Fix all metrics (#545)
aditya1702 Mar 30, 2026
e7419ae
Merge branch 'feature/finalize-metrics' into refactor/remove-txn-xdrs
aditya1702 Mar 30, 2026
dad409f
Merge branch 'main' into refactor/remove-txn-xdrs
aditya1702 Mar 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 0 additions & 16 deletions cmd/ingest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,22 +122,6 @@ func (c *ingestCmd) Command() *cobra.Command {
FlagDefault: "config/datastore-pubnet.toml",
Required: false,
},
{
Name: "skip-tx-meta",
Usage: "Skip storing transaction metadata (meta_xdr) to reduce storage space and improve insertion performance.",
OptType: types.Bool,
ConfigKey: &cfg.SkipTxMeta,
FlagDefault: true,
Required: false,
},
{
Name: "skip-tx-envelope",
Usage: "Skip storing transaction envelope (envelope_xdr) to reduce storage space and improve insertion performance.",
OptType: types.Bool,
ConfigKey: &cfg.SkipTxEnvelope,
FlagDefault: true,
Required: false,
},
{
Name: "chunk-interval",
Usage: "TimescaleDB chunk time interval for hypertables. Only affects future chunks. Uses PostgreSQL INTERVAL syntax.",
Expand Down
20 changes: 0 additions & 20 deletions cmd/loadtest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,6 @@ func (c *loadtestCmd) runCommand() *cobra.Command {
DatabaseURL string
NetworkPassphrase string
ServerPort int
SkipTxMeta bool
SkipTxEnvelope bool
StartLedger int
SeedDataPath string
}{}
Expand Down Expand Up @@ -106,22 +104,6 @@ func (c *loadtestCmd) runCommand() *cobra.Command {
FlagDefault: 8003,
Required: false,
},
{
Name: "skip-tx-meta",
Usage: "Skip storing transaction metadata (meta_xdr)",
OptType: types.Bool,
ConfigKey: &cfg.SkipTxMeta,
FlagDefault: true,
Required: false,
},
{
Name: "skip-tx-envelope",
Usage: "Skip storing transaction envelope (envelope_xdr)",
OptType: types.Bool,
ConfigKey: &cfg.SkipTxEnvelope,
FlagDefault: true,
Required: false,
},
{
Name: "start-ledger",
Usage: "Ledger sequence to start ingestion from (defaults to 606644043 for pubnet datastore)",
Expand Down Expand Up @@ -192,8 +174,6 @@ Example:
DatabaseURL: cfg.DatabaseURL,
NetworkPassphrase: cfg.NetworkPassphrase,
ServerPort: cfg.ServerPort,
SkipTxMeta: cfg.SkipTxMeta,
SkipTxEnvelope: cfg.SkipTxEnvelope,
StartLedger: uint32(cfg.StartLedger),
SeedDataPath: cfg.SeedDataPath,
})
Expand Down
4 changes: 2 additions & 2 deletions internal/data/accounts_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func TestAccountModelBatchGetByToIDs(t *testing.T) {
// Insert test transactions first (hash is BYTEA, using valid 64-char hex strings)
testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001")
testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002")
_, err = m.DB.Exec(ctx, "INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at) VALUES ($1, $2, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, NOW()), ($3, $4, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, NOW())", testHash1, toID1, testHash2, toID2)
_, err = m.DB.Exec(ctx, "INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at) VALUES ($1, $2, 100, 'TransactionResultCodeTxSuccess', 1, NOW()), ($3, $4, 200, 'TransactionResultCodeTxSuccess', 2, NOW())", testHash1, toID1, testHash2, toID2)
require.NoError(t, err)

// Insert test transactions_accounts links
Expand Down Expand Up @@ -85,7 +85,7 @@ func TestAccountModelBatchGetByOperationIDs(t *testing.T) {
// Insert test transactions first (hash is BYTEA, using valid 64-char hex strings)
testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001")
testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002")
_, err = m.DB.Exec(ctx, "INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at) VALUES ($1, 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, NOW()), ($2, 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, NOW())", testHash1, testHash2)
_, err = m.DB.Exec(ctx, "INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at) VALUES ($1, 4096, 100, 'TransactionResultCodeTxSuccess', 1, NOW()), ($2, 8192, 200, 'TransactionResultCodeTxSuccess', 2, NOW())", testHash1, testHash2)
require.NoError(t, err)

// Insert test operations (IDs don't need to be in TOID range here since we're just testing operations_accounts links)
Expand Down
20 changes: 10 additions & 10 deletions internal/data/ingest_store_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,8 +222,8 @@ func Test_IngestStoreModel_GetLedgerGaps(t *testing.T) {
// Insert consecutive ledgers: 100, 101, 102
for i, ledger := range []uint32{100, 101, 102} {
_, err := dbConnectionPool.Exec(ctx,
`INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at)
VALUES ($1, $2, 'env', 100, 'TransactionResultCodeTxSuccess', 'meta', $3, NOW())`,
`INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at)
VALUES ($1, $2, 100, 'TransactionResultCodeTxSuccess', $3, NOW())`,
fmt.Sprintf("hash%d", i), i+1, ledger)
require.NoError(t, err)
}
Expand All @@ -236,8 +236,8 @@ func Test_IngestStoreModel_GetLedgerGaps(t *testing.T) {
// Insert ledgers 100 and 105, creating gap 101-104
for i, ledger := range []uint32{100, 105} {
_, err := dbConnectionPool.Exec(ctx,
`INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at)
VALUES ($1, $2, 'env', 100, 'TransactionResultCodeTxSuccess', 'meta', $3, NOW())`,
`INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at)
VALUES ($1, $2, 100, 'TransactionResultCodeTxSuccess', $3, NOW())`,
fmt.Sprintf("hash%d", i), i+1, ledger)
require.NoError(t, err)
}
Expand All @@ -252,8 +252,8 @@ func Test_IngestStoreModel_GetLedgerGaps(t *testing.T) {
// Insert ledgers 100, 105, 110, creating gaps 101-104 and 106-109
for i, ledger := range []uint32{100, 105, 110} {
_, err := dbConnectionPool.Exec(ctx,
`INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at)
VALUES ($1, $2, 'env', 100, 'TransactionResultCodeTxSuccess', 'meta', $3, NOW())`,
`INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at)
VALUES ($1, $2, 100, 'TransactionResultCodeTxSuccess', $3, NOW())`,
fmt.Sprintf("hash%d", i), i+1, ledger)
require.NoError(t, err)
}
Expand All @@ -269,8 +269,8 @@ func Test_IngestStoreModel_GetLedgerGaps(t *testing.T) {
// Insert ledgers 100 and 102, creating gap of just 101
for i, ledger := range []uint32{100, 102} {
_, err := dbConnectionPool.Exec(ctx,
`INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at)
VALUES ($1, $2, 'env', 100, 'TransactionResultCodeTxSuccess', 'meta', $3, NOW())`,
`INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at)
VALUES ($1, $2, 100, 'TransactionResultCodeTxSuccess', $3, NOW())`,
fmt.Sprintf("hash%d", i), i+1, ledger)
require.NoError(t, err)
}
Expand Down Expand Up @@ -329,8 +329,8 @@ func Test_IngestStoreModel_GetOldestLedger(t *testing.T) {
// returns the correct oldest regardless of to_id ordering.
for i, ledger := range []uint32{150, 100, 200} {
_, err := dbConnectionPool.Exec(ctx,
`INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at)
VALUES ($1, $2, 'env', 100, 'TransactionResultCodeTxSuccess', 'meta', $3, NOW() - INTERVAL '1 day' * $4)`,
`INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at)
VALUES ($1, $2, 100, 'TransactionResultCodeTxSuccess', $3, NOW() - INTERVAL '1 day' * $4)`,
fmt.Sprintf("hash%d", i), i+1, ledger, 300-int(ledger))
require.NoError(t, err)
}
Expand Down
58 changes: 26 additions & 32 deletions internal/data/operations_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,37 +57,31 @@ func Test_OperationModel_BatchCopy(t *testing.T) {

// Create referenced transactions first with specific ToIDs
// Operations IDs must be in TOID range for each transaction: (to_id, to_id + 4096)
meta1, meta2 := "meta1", "meta2"
envelope1, envelope2 := "envelope1", "envelope2"
tx1 := types.Transaction{
Hash: "d176b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877",
ToID: 4096,
EnvelopeXDR: &envelope1,
FeeCharged: 100,
ResultCode: "TransactionResultCodeTxSuccess",
MetaXDR: &meta1,
LedgerNumber: 1,
LedgerCreatedAt: now,
IsFeeBump: false,
}
tx2 := types.Transaction{
Hash: "e176b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877",
ToID: 8192,
EnvelopeXDR: &envelope2,
FeeCharged: 200,
ResultCode: "TransactionResultCodeTxSuccess",
MetaXDR: &meta2,
LedgerNumber: 2,
LedgerCreatedAt: now,
IsFeeBump: true,
}

// Insert transactions using direct SQL
_, err = dbConnectionPool.Exec(ctx, `
INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9), ($10, $11, $12, $13, $14, $15, $16, $17, $18)
`, tx1.Hash, tx1.ToID, *tx1.EnvelopeXDR, tx1.FeeCharged, tx1.ResultCode, *tx1.MetaXDR, tx1.LedgerNumber, tx1.LedgerCreatedAt, tx1.IsFeeBump,
tx2.Hash, tx2.ToID, *tx2.EnvelopeXDR, tx2.FeeCharged, tx2.ResultCode, *tx2.MetaXDR, tx2.LedgerNumber, tx2.LedgerCreatedAt, tx2.IsFeeBump)
INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at, is_fee_bump)
VALUES ($1, $2, $3, $4, $5, $6, $7), ($8, $9, $10, $11, $12, $13, $14)
`, tx1.Hash, tx1.ToID, tx1.FeeCharged, tx1.ResultCode, tx1.LedgerNumber, tx1.LedgerCreatedAt, tx1.IsFeeBump,
tx2.Hash, tx2.ToID, tx2.FeeCharged, tx2.ResultCode, tx2.LedgerNumber, tx2.LedgerCreatedAt, tx2.IsFeeBump)
require.NoError(t, err)

// Operations IDs must be in TOID range: (to_id, to_id + 4096)
Expand Down Expand Up @@ -220,11 +214,11 @@ func TestOperationModel_GetAll(t *testing.T) {
testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002")
testHash3 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000003")
_, err = dbConnectionPool.Exec(ctx, `
INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump)
INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at, is_fee_bump)
VALUES
($2, 1, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false),
($3, 2, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true),
($4, 3, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false)
($2, 1, 100, 'TransactionResultCodeTxSuccess', 1, $1, false),
($3, 2, 200, 'TransactionResultCodeTxSuccess', 2, $1, true),
($4, 3, 300, 'TransactionResultCodeTxSuccess', 3, $1, false)
`, now, testHash1, testHash2, testHash3)
require.NoError(t, err)

Expand Down Expand Up @@ -275,11 +269,11 @@ func TestOperationModel_BatchGetByToIDs(t *testing.T) {
testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002")
testHash3 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000003")
_, err = dbConnectionPool.Exec(ctx, `
INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump)
INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at, is_fee_bump)
VALUES
($2, 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false),
($3, 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true),
($4, 12288, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false)
($2, 4096, 100, 'TransactionResultCodeTxSuccess', 1, $1, false),
($3, 8192, 200, 'TransactionResultCodeTxSuccess', 2, $1, true),
($4, 12288, 300, 'TransactionResultCodeTxSuccess', 3, $1, false)
`, now, testHash1, testHash2, testHash3)
require.NoError(t, err)

Expand Down Expand Up @@ -469,10 +463,10 @@ func TestOperationModel_BatchGetByToID(t *testing.T) {
testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001")
testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002")
_, err = dbConnectionPool.Exec(ctx, `
INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump)
INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at, is_fee_bump)
VALUES
($2, 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false),
($3, 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true)
($2, 4096, 100, 'TransactionResultCodeTxSuccess', 1, $1, false),
($3, 8192, 200, 'TransactionResultCodeTxSuccess', 2, $1, true)
`, now, testHash1, testHash2)
require.NoError(t, err)

Expand Down Expand Up @@ -525,11 +519,11 @@ func TestOperationModel_BatchGetByAccountAddresses(t *testing.T) {
testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002")
testHash3 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000003")
_, err = dbConnectionPool.Exec(ctx, `
INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump)
INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at, is_fee_bump)
VALUES
($2, 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false),
($3, 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true),
($4, 12288, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false)
($2, 4096, 100, 'TransactionResultCodeTxSuccess', 1, $1, false),
($3, 8192, 200, 'TransactionResultCodeTxSuccess', 2, $1, true),
($4, 12288, 300, 'TransactionResultCodeTxSuccess', 3, $1, false)
`, now, testHash1, testHash2, testHash3)
require.NoError(t, err)

Expand Down Expand Up @@ -578,10 +572,10 @@ func TestOperationModel_GetByID(t *testing.T) {
testHash1 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000001")
testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002")
_, err = dbConnectionPool.Exec(ctx, `
INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump)
INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at, is_fee_bump)
VALUES
($2, 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false),
($3, 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true)
($2, 4096, 100, 'TransactionResultCodeTxSuccess', 1, $1, false),
($3, 8192, 200, 'TransactionResultCodeTxSuccess', 2, $1, true)
`, now, testHash1, testHash2)
require.NoError(t, err)

Expand Down Expand Up @@ -637,11 +631,11 @@ func TestOperationModel_BatchGetByStateChangeIDs(t *testing.T) {
testHash2 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000002")
testHash3 := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000003")
_, err = dbConnectionPool.Exec(ctx, `
INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at, is_fee_bump)
INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at, is_fee_bump)
VALUES
($2, 4096, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, $1, false),
($3, 8192, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, $1, true),
($4, 12288, 'env3', 300, 'TransactionResultCodeTxSuccess', 'meta3', 3, $1, false)
($2, 4096, 100, 'TransactionResultCodeTxSuccess', 1, $1, false),
($3, 8192, 200, 'TransactionResultCodeTxSuccess', 2, $1, true),
($4, 12288, 300, 'TransactionResultCodeTxSuccess', 3, $1, false)
`, now, testHash1, testHash2, testHash3)
require.NoError(t, err)

Expand Down
Loading
Loading