From 6873d1806de330c7732fcb87ce7c873d5950205c Mon Sep 17 00:00:00 2001 From: Longs Pemun Gotar Date: Fri, 28 Nov 2025 21:53:24 +0100 Subject: [PATCH 1/4] core/rawdb: add v7 tx lookup format with transaction index Stores [blockNumber|txIndex] (16 bytes) to avoid hashing all transactions. Maintains backward compatibility with v3-v6 formats. --- core/rawdb/accessors_indexes.go | 113 ++++++++++++++++++++++++-------- 1 file changed, 87 insertions(+), 26 deletions(-) diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 10eb454015b..7373703bd96 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -32,63 +32,75 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -// DecodeTxLookupEntry decodes the supplied tx lookup data. -func DecodeTxLookupEntry(data []byte, db ethdb.Reader) *uint64 { +// DecodeTxLookupEntry decodes the supplied tx lookup data. It returns the block +// number and optionally the transaction index within the block. The transaction +// index is only available in database v7+ format; for older formats it returns nil. +func DecodeTxLookupEntry(data []byte, db ethdb.Reader) (*uint64, *uint64) { + // Database v7 tx lookup stores block number (8 bytes) + tx index (8 bytes) = 16 bytes + if len(data) == 16 { + number := binary.BigEndian.Uint64(data[:8]) + txIndex := binary.BigEndian.Uint64(data[8:16]) + return &number, &txIndex + } // Database v6 tx lookup just stores the block number if len(data) < common.HashLength { number := new(big.Int).SetBytes(data).Uint64() - return &number + return &number, nil } // Database v4-v5 tx lookup format just stores the hash if len(data) == common.HashLength { number, ok := ReadHeaderNumber(db, common.BytesToHash(data)) if !ok { - return nil + return nil, nil } - return &number + return &number, nil } // Finally try database v3 tx lookup format var entry LegacyTxLookupEntry if err := rlp.DecodeBytes(data, &entry); err != nil { log.Error("Invalid transaction lookup entry RLP", "blob", data, "err", err) - return nil + return nil, nil } - return &entry.BlockIndex + return &entry.BlockIndex, nil } // ReadTxLookupEntry retrieves the positional metadata associated with a transaction -// hash to allow retrieving the transaction or receipt by hash. -func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) *uint64 { +// hash to allow retrieving the transaction or receipt by hash. It returns the block +// number and optionally the transaction index within the block (if available in the +// database format). +func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) (*uint64, *uint64) { data, _ := db.Get(txLookupKey(hash)) if len(data) == 0 { - return nil + return nil, nil } return DecodeTxLookupEntry(data, db) } -// writeTxLookupEntry stores a positional metadata for a transaction, -// enabling hash based transaction and receipt lookups. -func writeTxLookupEntry(db ethdb.KeyValueWriter, hash common.Hash, numberBytes []byte) { - if err := db.Put(txLookupKey(hash), numberBytes); err != nil { +// writeTxLookupEntryV7 stores a positional metadata for a transaction in database +// v7 format, which includes both the block number and transaction index. +func writeTxLookupEntryV7(db ethdb.KeyValueWriter, hash common.Hash, blockNumber uint64, txIndex uint64) { + var data [16]byte + binary.BigEndian.PutUint64(data[:8], blockNumber) + binary.BigEndian.PutUint64(data[8:16], txIndex) + if err := db.Put(txLookupKey(hash), data[:]); err != nil { log.Crit("Failed to store transaction lookup entry", "err", err) } } -// WriteTxLookupEntries is identical to WriteTxLookupEntry, but it works on -// a list of hashes +// WriteTxLookupEntries stores positional metadata for all transactions in the given +// hashes list, using the new database v7 format that includes transaction indices. func WriteTxLookupEntries(db ethdb.KeyValueWriter, number uint64, hashes []common.Hash) { - numberBytes := new(big.Int).SetUint64(number).Bytes() - for _, hash := range hashes { - writeTxLookupEntry(db, hash, numberBytes) + for i, hash := range hashes { + writeTxLookupEntryV7(db, hash, number, uint64(i)) } } // WriteTxLookupEntriesByBlock stores a positional metadata for every transaction from // a block, enabling hash based transaction and receipt lookups. func WriteTxLookupEntriesByBlock(db ethdb.KeyValueWriter, block *types.Block) { - numberBytes := block.Number().Bytes() - for _, tx := range block.Transactions() { - writeTxLookupEntry(db, tx.Hash(), numberBytes) + number := block.Number().Uint64() + for i, tx := range block.Transactions() { + writeTxLookupEntryV7(db, tx.Hash(), number, uint64(i)) } } @@ -134,6 +146,39 @@ func DeleteAllTxLookupEntries(db ethdb.KeyValueStore, condition func(common.Hash } } +// extractTransactionAtIndex extracts a single transaction from the RLP-encoded +// block body at the specified index. This is more efficient than findTxInBlockBody +// when the transaction index is known, as it avoids hashing all transactions. +func extractTransactionAtIndex(blockbody rlp.RawValue, targetIndex uint64) (*types.Transaction, error) { + txnListRLP, _, err := rlp.SplitList(blockbody) + if err != nil { + return nil, err + } + iter, err := rlp.NewListIterator(txnListRLP) + if err != nil { + return nil, err + } + for i := uint64(0); i < targetIndex; i++ { + if !iter.Next() { + return nil, fmt.Errorf("transaction index %d out of bounds", targetIndex) + } + if iter.Err() != nil { + return nil, iter.Err() + } + } + if !iter.Next() { + return nil, fmt.Errorf("transaction index %d out of bounds", targetIndex) + } + if iter.Err() != nil { + return nil, iter.Err() + } + var tx types.Transaction + if err := rlp.DecodeBytes(iter.Value(), &tx); err != nil { + return nil, err + } + return &tx, nil +} + // findTxInBlockBody traverses the given RLP-encoded block body, searching for // the transaction specified by its hash. func findTxInBlockBody(blockbody rlp.RawValue, target common.Hash) (*types.Transaction, uint64, error) { @@ -178,7 +223,7 @@ func findTxInBlockBody(blockbody rlp.RawValue, target common.Hash) (*types.Trans // with its added positional metadata. Notably, only the transaction in the canonical // chain is visible. func ReadCanonicalTransaction(db ethdb.Reader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) { - blockNumber := ReadTxLookupEntry(db, hash) + blockNumber, txIndex := ReadTxLookupEntry(db, hash) if blockNumber == nil { return nil, common.Hash{}, 0, 0 } @@ -191,12 +236,20 @@ func ReadCanonicalTransaction(db ethdb.Reader, hash common.Hash) (*types.Transac log.Error("Transaction referenced missing", "number", *blockNumber, "hash", blockHash) return nil, common.Hash{}, 0, 0 } - tx, txIndex, err := findTxInBlockBody(bodyRLP, hash) + if txIndex != nil { + tx, err := extractTransactionAtIndex(bodyRLP, *txIndex) + if err != nil { + log.Error("Transaction not found at index", "number", *blockNumber, "hash", blockHash, "txhash", hash, "index", *txIndex, "err", err) + return nil, common.Hash{}, 0, 0 + } + return tx, blockHash, *blockNumber, *txIndex + } + tx, foundIndex, err := findTxInBlockBody(bodyRLP, hash) if err != nil { log.Error("Transaction not found", "number", *blockNumber, "hash", blockHash, "txhash", hash, "err", err) return nil, common.Hash{}, 0, 0 } - return tx, blockHash, *blockNumber, txIndex + return tx, blockHash, *blockNumber, foundIndex } // ReadCanonicalReceipt retrieves a specific transaction receipt from the database, @@ -204,7 +257,7 @@ func ReadCanonicalTransaction(db ethdb.Reader, hash common.Hash) (*types.Transac // chain is visible. func ReadCanonicalReceipt(db ethdb.Reader, hash common.Hash, config *params.ChainConfig) (*types.Receipt, common.Hash, uint64, uint64) { // Retrieve the context of the receipt based on the transaction hash - blockNumber := ReadTxLookupEntry(db, hash) + blockNumber, txIndex := ReadTxLookupEntry(db, hash) if blockNumber == nil { return nil, common.Hash{}, 0, 0 } @@ -216,6 +269,14 @@ func ReadCanonicalReceipt(db ethdb.Reader, hash common.Hash, config *params.Chai if blockHeader == nil { return nil, common.Hash{}, 0, 0 } + if txIndex != nil { + receipts := ReadReceipts(db, blockHash, *blockNumber, blockHeader.Time, config) + if *txIndex < uint64(len(receipts)) { + return receipts[*txIndex], blockHash, *blockNumber, *txIndex + } + log.Error("Receipt index out of bounds", "number", *blockNumber, "hash", blockHash, "txhash", hash, "index", *txIndex) + return nil, common.Hash{}, 0, 0 + } // Read all the receipts from the block and return the one with the matching hash receipts := ReadReceipts(db, blockHash, *blockNumber, blockHeader.Time, config) for receiptIndex, receipt := range receipts { From aee8c1505faab30258b8f4bcad72243982ec41b1 Mon Sep 17 00:00:00 2001 From: Longs Pemun Gotar Date: Fri, 28 Nov 2025 21:54:21 +0100 Subject: [PATCH 2/4] core: update callers for new ReadTxLookupEntry signature ReadTxLookupEntry now returns (blockNumber, txIndex) instead of just blockNumber. --- core/rawdb/chain_iterator.go | 13 ++++++++++--- core/rawdb/chain_iterator_test.go | 6 +++--- core/txindexer.go | 2 +- core/txindexer_test.go | 2 +- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index e7c89ca8d91..0a92f7770e3 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -378,11 +378,18 @@ func PruneTransactionIndex(db ethdb.Database, pruneBlock uint64) { if count%10000000 == 0 { log.Info("Pruning tx index", "count", count, "removed", removed) } - if len(v) > 8 { - log.Error("Skipping legacy tx index entry", "hash", txhash) + var bn uint64 + // Database v7: block number (8 bytes) + tx index (8 bytes) = 16 bytes + if len(v) == 16 { + bn = binary.BigEndian.Uint64(v[:8]) + } else if len(v) <= 8 { + // Database v6 or earlier + bn = decodeNumber(v) + } else { + // Unknown format + log.Error("Skipping unknown tx index entry format", "hash", txhash, "len", len(v)) return false } - bn := decodeNumber(v) if bn < pruneBlock { removed++ return true diff --git a/core/rawdb/chain_iterator_test.go b/core/rawdb/chain_iterator_test.go index 75bd5a9a942..8913bb00ff3 100644 --- a/core/rawdb/chain_iterator_test.go +++ b/core/rawdb/chain_iterator_test.go @@ -160,7 +160,7 @@ func TestIndexTransactions(t *testing.T) { if i == 0 { continue } - number := ReadTxLookupEntry(chainDB, txs[i-1].Hash()) + number, _ := ReadTxLookupEntry(chainDB, txs[i-1].Hash()) if exist && number == nil { t.Fatalf("Transaction index %d missing", i) } @@ -229,7 +229,7 @@ func TestPruneTransactionIndex(t *testing.T) { // Check all transactions are in index. for _, block := range blocks { for _, tx := range block.Transactions() { - num := ReadTxLookupEntry(chainDB, tx.Hash()) + num, _ := ReadTxLookupEntry(chainDB, tx.Hash()) if num == nil || *num != block.NumberU64() { t.Fatalf("wrong TxLookup entry: %x -> %v", tx.Hash(), num) } @@ -241,7 +241,7 @@ func TestPruneTransactionIndex(t *testing.T) { // Check transactions from old blocks not included. for _, block := range blocks { for _, tx := range block.Transactions() { - num := ReadTxLookupEntry(chainDB, tx.Hash()) + num, _ := ReadTxLookupEntry(chainDB, tx.Hash()) if block.NumberU64() < pruneBlock && num != nil { t.Fatalf("TxLookup entry not removed: %x -> %v", tx.Hash(), num) } diff --git a/core/txindexer.go b/core/txindexer.go index b2a94a6ead1..929972f7007 100644 --- a/core/txindexer.go +++ b/core/txindexer.go @@ -204,7 +204,7 @@ func (indexer *txIndexer) repair(head uint64) { indexer.tail.Store(&indexer.cutoff) rawdb.WriteTxIndexTail(indexer.db, indexer.cutoff) rawdb.DeleteAllTxLookupEntries(indexer.db, func(txhash common.Hash, blob []byte) bool { - n := rawdb.DecodeTxLookupEntry(blob, indexer.db) + n, _ := rawdb.DecodeTxLookupEntry(blob, indexer.db) return n != nil && *n < indexer.cutoff }) log.Warn("Purge transaction indexes below cutoff", "tail", *tail, "cutoff", indexer.cutoff) diff --git a/core/txindexer_test.go b/core/txindexer_test.go index 71c78d506bc..f718ceeda0c 100644 --- a/core/txindexer_test.go +++ b/core/txindexer_test.go @@ -31,7 +31,7 @@ import ( func verifyIndexes(t *testing.T, db ethdb.Database, block *types.Block, exist bool) { for _, tx := range block.Transactions() { - lookup := rawdb.ReadTxLookupEntry(db, tx.Hash()) + lookup, _ := rawdb.ReadTxLookupEntry(db, tx.Hash()) if exist && lookup == nil { t.Fatalf("missing %d %x", block.NumberU64(), tx.Hash().Hex()) } From 50c75a9a4cb36407b3439de864b3755b7ea3a519 Mon Sep 17 00:00:00 2001 From: Longs Pemun Gotar Date: Fri, 28 Nov 2025 21:54:53 +0100 Subject: [PATCH 3/4] core/rawdb: add tests and benchmarks for v7 tx lookup Includes: - V7 encoding/decoding tests - Backward compatibility tests (v3-v7) - Fast path verification tests - Benchmarks showing 20-40x speedup for blocks with 100-200 txs --- core/rawdb/accessors_indexes_test.go | 331 ++++++++++++++++++++++++++- 1 file changed, 330 insertions(+), 1 deletion(-) diff --git a/core/rawdb/accessors_indexes_test.go b/core/rawdb/accessors_indexes_test.go index a812fefeaa3..c7b3546f7d7 100644 --- a/core/rawdb/accessors_indexes_test.go +++ b/core/rawdb/accessors_indexes_test.go @@ -18,6 +18,7 @@ package rawdb import ( "errors" + "fmt" "math/big" "testing" @@ -39,11 +40,20 @@ func TestLookupStorage(t *testing.T) { writeTxLookupEntriesByBlock func(ethdb.KeyValueWriter, *types.Block) }{ { - "DatabaseV6", + "DatabaseV7", func(db ethdb.KeyValueWriter, block *types.Block) { WriteTxLookupEntriesByBlock(db, block) }, }, + { + "DatabaseV6", + func(db ethdb.KeyValueWriter, block *types.Block) { + number := block.Number().Bytes() + for _, tx := range block.Transactions() { + db.Put(txLookupKey(tx.Hash()), number) + } + }, + }, { "DatabaseV4-V5", func(db ethdb.KeyValueWriter, block *types.Block) { @@ -297,3 +307,322 @@ func TestExtractReceiptFields(t *testing.T) { } } } + +// TestExtractTransactionAtIndex tests the extractTransactionAtIndex function +// which is the core optimization for v7 database format. +func TestExtractTransactionAtIndex(t *testing.T) { + tx1 := types.NewTx(&types.LegacyTx{ + Nonce: 1, + GasPrice: big.NewInt(1), + Gas: 1, + To: new(common.Address), + Value: big.NewInt(5), + Data: []byte{0x11, 0x11, 0x11}, + }) + tx2 := types.NewTx(&types.AccessListTx{ + ChainID: big.NewInt(1), + Nonce: 2, + GasPrice: big.NewInt(2), + Gas: 2, + To: new(common.Address), + Value: big.NewInt(10), + Data: []byte{0x22, 0x22, 0x22}, + }) + tx3 := types.NewTx(&types.DynamicFeeTx{ + ChainID: big.NewInt(1), + Nonce: 3, + Gas: 3, + To: new(common.Address), + Value: big.NewInt(15), + Data: []byte{0x33, 0x33, 0x33}, + GasTipCap: big.NewInt(55), + GasFeeCap: big.NewInt(1055), + }) + + txs := []*types.Transaction{tx1, tx2, tx3} + block := types.NewBlock(&types.Header{Number: big.NewInt(100)}, &types.Body{Transactions: txs}, nil, newTestHasher()) + db := NewMemoryDatabase() + WriteBlock(db, block) + + bodyRLP := ReadBodyRLP(db, block.Hash(), block.NumberU64()) + + for i, expectedTx := range txs { + extractedTx, err := extractTransactionAtIndex(bodyRLP, uint64(i)) + if err != nil { + t.Fatalf("Failed to extract transaction at index %d: %v", i, err) + } + if extractedTx.Hash() != expectedTx.Hash() { + t.Fatalf("Transaction mismatch at index %d: got %x, want %x", i, extractedTx.Hash(), expectedTx.Hash()) + } + } + + _, err := extractTransactionAtIndex(bodyRLP, uint64(len(txs))) + if err == nil { + t.Fatal("Expected error for out of bounds index, got nil") + } + + singleTx := types.NewTransaction(100, common.BytesToAddress([]byte{0xaa}), big.NewInt(1000), 21000, big.NewInt(1), nil) + singleBlock := types.NewBlock(&types.Header{Number: big.NewInt(200)}, &types.Body{Transactions: []*types.Transaction{singleTx}}, nil, newTestHasher()) + WriteBlock(db, singleBlock) + singleBodyRLP := ReadBodyRLP(db, singleBlock.Hash(), singleBlock.NumberU64()) + + extractedTx, err := extractTransactionAtIndex(singleBodyRLP, 0) + if err != nil { + t.Fatalf("Failed to extract single transaction: %v", err) + } + if extractedTx.Hash() != singleTx.Hash() { + t.Fatalf("Single transaction mismatch: got %x, want %x", extractedTx.Hash(), singleTx.Hash()) + } +} + +// TestTxLookupV7Encoding tests the v7 database format encoding and decoding. +func TestTxLookupV7Encoding(t *testing.T) { + db := NewMemoryDatabase() + + testCases := []struct { + blockNumber uint64 + txIndex uint64 + txHash common.Hash + }{ + {0, 0, common.BytesToHash([]byte{0x01})}, + {1, 0, common.BytesToHash([]byte{0x02})}, + {100, 5, common.BytesToHash([]byte{0x03})}, + {999999, 199, common.BytesToHash([]byte{0x04})}, + {18446744073709551615, 255, common.BytesToHash([]byte{0x05})}, // max uint64 + } + + for _, tc := range testCases { + writeTxLookupEntryV7(db, tc.txHash, tc.blockNumber, tc.txIndex) + + blockNum, txIdx := ReadTxLookupEntry(db, tc.txHash) + if blockNum == nil { + t.Fatalf("Failed to read block number for hash %x", tc.txHash) + } + if *blockNum != tc.blockNumber { + t.Fatalf("Block number mismatch: got %d, want %d", *blockNum, tc.blockNumber) + } + if txIdx == nil { + t.Fatalf("Failed to read tx index for hash %x", tc.txHash) + } + if *txIdx != tc.txIndex { + t.Fatalf("Tx index mismatch: got %d, want %d", *txIdx, tc.txIndex) + } + } +} + +// TestTxLookupBackwardCompatibility tests that all database versions can be read correctly. +func TestTxLookupBackwardCompatibility(t *testing.T) { + db := NewMemoryDatabase() + + tx := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11}) + txHash := tx.Hash() + blockNumber := uint64(314) + txIndex := uint64(2) + + writeTxLookupEntryV7(db, txHash, blockNumber, txIndex) + num, idx := ReadTxLookupEntry(db, txHash) + if num == nil || *num != blockNumber { + t.Fatalf("V7: block number mismatch, got %v, want %d", num, blockNumber) + } + if idx == nil || *idx != txIndex { + t.Fatalf("V7: tx index mismatch, got %v, want %d", idx, txIndex) + } + + v6Hash := common.BytesToHash([]byte{0x02}) + db.Put(txLookupKey(v6Hash), big.NewInt(int64(blockNumber)).Bytes()) + num, idx = ReadTxLookupEntry(db, v6Hash) + if num == nil || *num != blockNumber { + t.Fatalf("V6: block number mismatch, got %v, want %d", num, blockNumber) + } + if idx != nil { + t.Fatalf("V6: expected nil tx index, got %d", *idx) + } + + v4Hash := common.BytesToHash([]byte{0x03}) + blockHash := common.BytesToHash([]byte{0xaa, 0xbb, 0xcc}) + db.Put(txLookupKey(v4Hash), blockHash.Bytes()) + WriteCanonicalHash(db, blockHash, blockNumber) + WriteHeaderNumber(db, blockHash, blockNumber) + num, idx = ReadTxLookupEntry(db, v4Hash) + if num == nil || *num != blockNumber { + t.Fatalf("V4-V5: block number mismatch, got %v, want %d", num, blockNumber) + } + if idx != nil { + t.Fatalf("V4-V5: expected nil tx index, got %d", *idx) + } + + v3Hash := common.BytesToHash([]byte{0x04}) + entry := LegacyTxLookupEntry{ + BlockHash: blockHash, + BlockIndex: blockNumber, + Index: txIndex, + } + data, _ := rlp.EncodeToBytes(entry) + db.Put(txLookupKey(v3Hash), data) + num, idx = ReadTxLookupEntry(db, v3Hash) + if num == nil || *num != blockNumber { + t.Fatalf("V3: block number mismatch, got %v, want %d", num, blockNumber) + } + if idx != nil { + t.Fatalf("V3: expected nil tx index for legacy format, got %d", *idx) + } +} + +// TestReadCanonicalTransactionV7FastPath tests that v7 entries use the fast path +// which skips hashing all transactions. +func TestReadCanonicalTransactionV7FastPath(t *testing.T) { + db := NewMemoryDatabase() + + var txs []*types.Transaction + for i := 0; i < 50; i++ { + tx := types.NewTransaction(uint64(i), common.BytesToAddress([]byte{byte(i)}), big.NewInt(int64(i)), 21000, big.NewInt(1), nil) + txs = append(txs, tx) + } + + block := types.NewBlock(&types.Header{Number: big.NewInt(500)}, &types.Body{Transactions: txs}, nil, newTestHasher()) + WriteCanonicalHash(db, block.Hash(), block.NumberU64()) + WriteBlock(db, block) + WriteTxLookupEntriesByBlock(db, block) + + for i, tx := range txs { + readTx, hash, number, index := ReadCanonicalTransaction(db, tx.Hash()) + if readTx == nil { + t.Fatalf("Transaction %d not found", i) + } + if readTx.Hash() != tx.Hash() { + t.Fatalf("Transaction hash mismatch at index %d", i) + } + if hash != block.Hash() { + t.Fatalf("Block hash mismatch at index %d", i) + } + if number != block.NumberU64() { + t.Fatalf("Block number mismatch at index %d", i) + } + if index != uint64(i) { + t.Fatalf("Transaction index mismatch: got %d, want %d", index, i) + } + } +} + +func createBenchmarkBlock(numTxs int, blockNum uint64) (*types.Block, []*types.Transaction) { + var txs []*types.Transaction + for i := 0; i < numTxs; i++ { + tx := types.NewTransaction( + uint64(i), + common.BytesToAddress([]byte{byte(i), byte(i >> 8)}), + big.NewInt(int64(i)*1000), + 21000, + big.NewInt(int64(i+1)*1e9), + nil, + ) + txs = append(txs, tx) + } + return types.NewBlock(&types.Header{Number: big.NewInt(int64(blockNum))}, &types.Body{Transactions: txs}, nil, newTestHasher()), txs +} + +// BenchmarkReadCanonicalTransactionV6 benchmarks v6 format without tx index. +func BenchmarkReadCanonicalTransactionV6(b *testing.B) { + sizes := []int{10, 50, 100, 200} + + for _, size := range sizes { + b.Run(fmt.Sprintf("Size%d", size), func(b *testing.B) { + db := NewMemoryDatabase() + block, txs := createBenchmarkBlock(size, 1000) + + WriteCanonicalHash(db, block.Hash(), block.NumberU64()) + WriteBlock(db, block) + + number := block.Number().Bytes() + for _, tx := range txs { + db.Put(txLookupKey(tx.Hash()), number) + } + + targetTx := txs[len(txs)-1] + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + tx, _, _, _ := ReadCanonicalTransaction(db, targetTx.Hash()) + if tx == nil { + b.Fatal("Transaction not found") + } + } + }) + } +} + +// BenchmarkReadCanonicalTransactionV7 benchmarks v7 format with tx index. +func BenchmarkReadCanonicalTransactionV7(b *testing.B) { + sizes := []int{10, 50, 100, 200} + + for _, size := range sizes { + b.Run(fmt.Sprintf("Size%d", size), func(b *testing.B) { + db := NewMemoryDatabase() + block, txs := createBenchmarkBlock(size, 1000) + + WriteCanonicalHash(db, block.Hash(), block.NumberU64()) + WriteBlock(db, block) + WriteTxLookupEntriesByBlock(db, block) + + targetTx := txs[len(txs)-1] + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + tx, _, _, _ := ReadCanonicalTransaction(db, targetTx.Hash()) + if tx == nil { + b.Fatal("Transaction not found") + } + } + }) + } +} + +// BenchmarkExtractTransactionAtIndex benchmarks extracting by index vs searching by hash. +func BenchmarkExtractTransactionAtIndex(b *testing.B) { + sizes := []int{10, 50, 100, 200} + + for _, size := range sizes { + b.Run(fmt.Sprintf("ByIndex_Size%d", size), func(b *testing.B) { + db := NewMemoryDatabase() + block, _ := createBenchmarkBlock(size, 1000) + WriteBlock(db, block) + bodyRLP := ReadBodyRLP(db, block.Hash(), block.NumberU64()) + + targetIndex := uint64(size - 1) + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + tx, err := extractTransactionAtIndex(bodyRLP, targetIndex) + if err != nil { + b.Fatal(err) + } + if tx == nil { + b.Fatal("Transaction is nil") + } + } + }) + + b.Run(fmt.Sprintf("ByHash_Size%d", size), func(b *testing.B) { + db := NewMemoryDatabase() + block, txs := createBenchmarkBlock(size, 1000) + WriteBlock(db, block) + bodyRLP := ReadBodyRLP(db, block.Hash(), block.NumberU64()) + + targetHash := txs[len(txs)-1].Hash() + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + tx, _, err := findTxInBlockBody(bodyRLP, targetHash) + if err != nil { + b.Fatal(err) + } + if tx == nil { + b.Fatal("Transaction is nil") + } + } + }) + } +} From acbfd6709b79e0341f644a611a0b647a37a4d374 Mon Sep 17 00:00:00 2001 From: Longs Pemun Gotar Date: Tue, 16 Dec 2025 13:07:21 +0100 Subject: [PATCH 4/4] core/rawdb: change tx index from uint64 to uint32 --- core/rawdb/accessors_indexes.go | 34 ++++++++++++++-------------- core/rawdb/accessors_indexes_test.go | 14 ++++++------ core/rawdb/chain_iterator.go | 4 ++-- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 7373703bd96..75b263f60eb 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -35,11 +35,11 @@ import ( // DecodeTxLookupEntry decodes the supplied tx lookup data. It returns the block // number and optionally the transaction index within the block. The transaction // index is only available in database v7+ format; for older formats it returns nil. -func DecodeTxLookupEntry(data []byte, db ethdb.Reader) (*uint64, *uint64) { - // Database v7 tx lookup stores block number (8 bytes) + tx index (8 bytes) = 16 bytes - if len(data) == 16 { +func DecodeTxLookupEntry(data []byte, db ethdb.Reader) (*uint64, *uint32) { + // Database v7 tx lookup stores block number (8 bytes) + tx index (4 bytes) = 12 bytes + if len(data) == 12 { number := binary.BigEndian.Uint64(data[:8]) - txIndex := binary.BigEndian.Uint64(data[8:16]) + txIndex := binary.BigEndian.Uint32(data[8:12]) return &number, &txIndex } // Database v6 tx lookup just stores the block number @@ -68,7 +68,7 @@ func DecodeTxLookupEntry(data []byte, db ethdb.Reader) (*uint64, *uint64) { // hash to allow retrieving the transaction or receipt by hash. It returns the block // number and optionally the transaction index within the block (if available in the // database format). -func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) (*uint64, *uint64) { +func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) (*uint64, *uint32) { data, _ := db.Get(txLookupKey(hash)) if len(data) == 0 { return nil, nil @@ -78,10 +78,10 @@ func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) (*uint64, *uint64) { // writeTxLookupEntryV7 stores a positional metadata for a transaction in database // v7 format, which includes both the block number and transaction index. -func writeTxLookupEntryV7(db ethdb.KeyValueWriter, hash common.Hash, blockNumber uint64, txIndex uint64) { - var data [16]byte +func writeTxLookupEntryV7(db ethdb.KeyValueWriter, hash common.Hash, blockNumber uint64, txIndex uint32) { + var data [12]byte binary.BigEndian.PutUint64(data[:8], blockNumber) - binary.BigEndian.PutUint64(data[8:16], txIndex) + binary.BigEndian.PutUint32(data[8:12], txIndex) if err := db.Put(txLookupKey(hash), data[:]); err != nil { log.Crit("Failed to store transaction lookup entry", "err", err) } @@ -91,7 +91,7 @@ func writeTxLookupEntryV7(db ethdb.KeyValueWriter, hash common.Hash, blockNumber // hashes list, using the new database v7 format that includes transaction indices. func WriteTxLookupEntries(db ethdb.KeyValueWriter, number uint64, hashes []common.Hash) { for i, hash := range hashes { - writeTxLookupEntryV7(db, hash, number, uint64(i)) + writeTxLookupEntryV7(db, hash, number, uint32(i)) } } @@ -100,7 +100,7 @@ func WriteTxLookupEntries(db ethdb.KeyValueWriter, number uint64, hashes []commo func WriteTxLookupEntriesByBlock(db ethdb.KeyValueWriter, block *types.Block) { number := block.Number().Uint64() for i, tx := range block.Transactions() { - writeTxLookupEntryV7(db, tx.Hash(), number, uint64(i)) + writeTxLookupEntryV7(db, tx.Hash(), number, uint32(i)) } } @@ -149,7 +149,7 @@ func DeleteAllTxLookupEntries(db ethdb.KeyValueStore, condition func(common.Hash // extractTransactionAtIndex extracts a single transaction from the RLP-encoded // block body at the specified index. This is more efficient than findTxInBlockBody // when the transaction index is known, as it avoids hashing all transactions. -func extractTransactionAtIndex(blockbody rlp.RawValue, targetIndex uint64) (*types.Transaction, error) { +func extractTransactionAtIndex(blockbody rlp.RawValue, targetIndex uint32) (*types.Transaction, error) { txnListRLP, _, err := rlp.SplitList(blockbody) if err != nil { return nil, err @@ -158,7 +158,7 @@ func extractTransactionAtIndex(blockbody rlp.RawValue, targetIndex uint64) (*typ if err != nil { return nil, err } - for i := uint64(0); i < targetIndex; i++ { + for i := uint32(0); i < targetIndex; i++ { if !iter.Next() { return nil, fmt.Errorf("transaction index %d out of bounds", targetIndex) } @@ -190,7 +190,7 @@ func findTxInBlockBody(blockbody rlp.RawValue, target common.Hash) (*types.Trans if err != nil { return nil, 0, err } - txIndex := uint64(0) + txIndex := uint32(0) for iter.Next() { if iter.Err() != nil { return nil, 0, iter.Err() @@ -212,7 +212,7 @@ func findTxInBlockBody(blockbody rlp.RawValue, target common.Hash) (*types.Trans if err := rlp.DecodeBytes(txRLP, &tx); err != nil { return nil, 0, err } - return &tx, txIndex, nil + return &tx, uint64(txIndex), nil } txIndex++ } @@ -242,7 +242,7 @@ func ReadCanonicalTransaction(db ethdb.Reader, hash common.Hash) (*types.Transac log.Error("Transaction not found at index", "number", *blockNumber, "hash", blockHash, "txhash", hash, "index", *txIndex, "err", err) return nil, common.Hash{}, 0, 0 } - return tx, blockHash, *blockNumber, *txIndex + return tx, blockHash, *blockNumber, uint64(*txIndex) } tx, foundIndex, err := findTxInBlockBody(bodyRLP, hash) if err != nil { @@ -271,8 +271,8 @@ func ReadCanonicalReceipt(db ethdb.Reader, hash common.Hash, config *params.Chai } if txIndex != nil { receipts := ReadReceipts(db, blockHash, *blockNumber, blockHeader.Time, config) - if *txIndex < uint64(len(receipts)) { - return receipts[*txIndex], blockHash, *blockNumber, *txIndex + if uint64(*txIndex) < uint64(len(receipts)) { + return receipts[*txIndex], blockHash, *blockNumber, uint64(*txIndex) } log.Error("Receipt index out of bounds", "number", *blockNumber, "hash", blockHash, "txhash", hash, "index", *txIndex) return nil, common.Hash{}, 0, 0 diff --git a/core/rawdb/accessors_indexes_test.go b/core/rawdb/accessors_indexes_test.go index c7b3546f7d7..243bb6a08d8 100644 --- a/core/rawdb/accessors_indexes_test.go +++ b/core/rawdb/accessors_indexes_test.go @@ -347,7 +347,7 @@ func TestExtractTransactionAtIndex(t *testing.T) { bodyRLP := ReadBodyRLP(db, block.Hash(), block.NumberU64()) for i, expectedTx := range txs { - extractedTx, err := extractTransactionAtIndex(bodyRLP, uint64(i)) + extractedTx, err := extractTransactionAtIndex(bodyRLP, uint32(i)) if err != nil { t.Fatalf("Failed to extract transaction at index %d: %v", i, err) } @@ -356,7 +356,7 @@ func TestExtractTransactionAtIndex(t *testing.T) { } } - _, err := extractTransactionAtIndex(bodyRLP, uint64(len(txs))) + _, err := extractTransactionAtIndex(bodyRLP, uint32(len(txs))) if err == nil { t.Fatal("Expected error for out of bounds index, got nil") } @@ -381,14 +381,14 @@ func TestTxLookupV7Encoding(t *testing.T) { testCases := []struct { blockNumber uint64 - txIndex uint64 + txIndex uint32 txHash common.Hash }{ {0, 0, common.BytesToHash([]byte{0x01})}, {1, 0, common.BytesToHash([]byte{0x02})}, {100, 5, common.BytesToHash([]byte{0x03})}, {999999, 199, common.BytesToHash([]byte{0x04})}, - {18446744073709551615, 255, common.BytesToHash([]byte{0x05})}, // max uint64 + {18446744073709551615, 4294967295, common.BytesToHash([]byte{0x05})}, // max uint32 } for _, tc := range testCases { @@ -417,7 +417,7 @@ func TestTxLookupBackwardCompatibility(t *testing.T) { tx := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11}) txHash := tx.Hash() blockNumber := uint64(314) - txIndex := uint64(2) + txIndex := uint32(2) writeTxLookupEntryV7(db, txHash, blockNumber, txIndex) num, idx := ReadTxLookupEntry(db, txHash) @@ -455,7 +455,7 @@ func TestTxLookupBackwardCompatibility(t *testing.T) { entry := LegacyTxLookupEntry{ BlockHash: blockHash, BlockIndex: blockNumber, - Index: txIndex, + Index: uint64(txIndex), } data, _ := rlp.EncodeToBytes(entry) db.Put(txLookupKey(v3Hash), data) @@ -589,7 +589,7 @@ func BenchmarkExtractTransactionAtIndex(b *testing.B) { WriteBlock(db, block) bodyRLP := ReadBodyRLP(db, block.Hash(), block.NumberU64()) - targetIndex := uint64(size - 1) + targetIndex := uint32(size - 1) b.ResetTimer() b.ReportAllocs() diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index 0a92f7770e3..50c70bfa549 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -379,8 +379,8 @@ func PruneTransactionIndex(db ethdb.Database, pruneBlock uint64) { log.Info("Pruning tx index", "count", count, "removed", removed) } var bn uint64 - // Database v7: block number (8 bytes) + tx index (8 bytes) = 16 bytes - if len(v) == 16 { + // Database v7: block number (8 bytes) + tx index (4 bytes) = 12 bytes + if len(v) == 12 { bn = binary.BigEndian.Uint64(v[:8]) } else if len(v) <= 8 { // Database v6 or earlier